From 41a252523e1084ec93f706214d8b1d075885e0a7 Mon Sep 17 00:00:00 2001 From: Syphax Date: Thu, 19 Jun 2025 09:54:28 +0200 Subject: [PATCH 01/19] update Instance model to use Goo not a raw Sparql query --- lib/ontologies_linked_data/models/instance.rb | 181 +++++------------- test/models/test_instances.rb | 86 ++++----- 2 files changed, 81 insertions(+), 186 deletions(-) diff --git a/lib/ontologies_linked_data/models/instance.rb b/lib/ontologies_linked_data/models/instance.rb index 85bdc3f27..ad8a2cd20 100644 --- a/lib/ontologies_linked_data/models/instance.rb +++ b/lib/ontologies_linked_data/models/instance.rb @@ -1,156 +1,69 @@ module LinkedData module Models - class Instance - include LinkedData::Hypermedia::Resource - include LinkedData::HTTPCache::CacheableResource + class Instance < LinkedData::Models::Base - attr_reader :id, :properties - attr_accessor :label - serialize_default :id, :label, :properties + model :named_individual, name_with: :id, collection: :submission, + namespace: :owl, schemaless: :true , rdf_type: lambda { |*x| RDF::OWL[:NamedIndividual]} - # HTTP cache settings. - cache_timeout 14400 + attribute :label, namespace: :rdfs, enforce: [:list] + attribute :prefLabel, namespace: :skos, enforce: [:existence], alias: true - def initialize(id,label,properties) - @id = id - if label.nil? - sep = "/" - if not id.to_s["#"].nil? - sep = "#" - end - label = id.to_s.split(sep).last - end - @label = label - @properties = properties - end + attribute :types, namespace: :rdf, enforce: [:list], property: :type + attribute :submission, collection: lambda { |s| s.resource_id }, namespace: :metadata - def add_property_value(p,o) - ps = p.to_s - if not @properties.include?(ps) - @properties[ps] = [] - end - @properties[ps] << o - end + serialize_never :submission, :id + serialize_methods :properties + + cache_timeout 14400 - def self.type_uri - LinkedData.settings.id_url_prefix+"metadata/Instance" + def properties + self.unmapped end + end end + module InstanceLoader def self.count_instances_by_class(submission_id,class_id) - query = <<-eos -PREFIX owl: -SELECT (count(DISTINCT ?s) as ?c) WHERE - { - GRAPH <#{submission_id.to_s}> { - ?s a owl:NamedIndividual . - ?s a <#{class_id.to_s}> . - } - } -eos - epr = Goo.sparql_query_client(:main) - graphs = [submission_id] - resultset = epr.query(query, graphs: graphs) - resultset.each do |r| - return r[:c].object - end - return 0 + ## TODO: pass directly an LinkedData::Models::OntologySubmission instance in the arguments instead of submission_id + s = LinkedData::Models::OntologySubmission.find(submission_id).first + instances_by_class_where_query(s, class_id: class_id).count end - def self.get_instances_by_class(submission_id,class_id) - query = <<-eos -PREFIX owl: -SELECT ?s ?label WHERE - { - GRAPH <#{submission_id.to_s}> { - ?s a owl:NamedIndividual . - ?s a <#{class_id.to_s}> . - } - } -eos - epr = Goo.sparql_query_client(:main) - graphs = [submission_id] - resultset = epr.query(query, graphs: graphs) - instances = [] - resultset.each do |r| - inst = LinkedData::Models::Instance.new(r[:s],nil,{}) - instances << inst - end - - if instances.empty? - return [] - end - - include_instance_properties(submission_id,instances) - return instances + def self.get_instances_by_class(submission_id, class_id, page_no: nil, size: nil) + ## TODO: pass directly an LinkedData::Models::OntologySubmission instance in the arguments instead of submission_id + s = LinkedData::Models::OntologySubmission.find(submission_id).first + + inst = instances_by_class_where_query(s, class_id: class_id, page_no: page_no, size: size).all + + # TODO test if "include=all" parameter is passed in the request + # For getting all the properties # For getting all the properties + load_unmapped s,inst unless inst.nil? || inst.empty? + inst end - def self.get_instances_by_ontology(submission_id,page_no,size) - query = <<-eos -PREFIX owl: -SELECT ?s ?label WHERE - { - GRAPH <#{submission_id.to_s}> { - ?s a owl:NamedIndividual . - } - } -eos - epr = Goo.sparql_query_client(:main) - graphs = [submission_id] - resultset = epr.query(query, graphs: graphs) - - total_size = resultset.size - range_start = (page_no - 1) * size - range_end = (page_no * size) - 1 - resultset = resultset[range_start..range_end] - - instances = [] - resultset.each do |r| - inst = LinkedData::Models::Instance.new(r[:s],r[:label],{}) - instances << inst - end unless resultset.nil? - - if instances.size > 0 - include_instance_properties(submission_id,instances) - end - - page = Goo::Base::Page.new(page_no,size,total_size,instances) - return page + def self.get_instances_by_ontology(submission_id, page_no: nil, size: nil) + ## TODO: pass directly an LinkedData::Models::OntologySubmission instance in the arguments instead of submission_id + s = LinkedData::Models::OntologySubmission.find(submission_id).first + inst = s.nil? ? [] : instances_by_class_where_query(s, page_no: page_no, size: size).all + + ## TODO test if "include=all" parameter is passed in the request + load_unmapped s, inst unless inst.nil? || inst.empty? # For getting all the properties + inst end - def self.include_instance_properties(submission_id,instances) - index = Hash.new - instances.each do |inst| - index[inst.id.to_s] = inst - end - uris = index.keys.map { |x| x.to_s } - uri_filter = uris.map { |x| "?s = <#{x}>"}.join(" || ") - - query = <<-eos -PREFIX owl: -SELECT ?s ?p ?o WHERE - { - GRAPH <#{submission_id.to_s}> { - ?s ?p ?o . - } - FILTER( #{uri_filter} ) - } -eos - epr = Goo.sparql_query_client(:main) - graphs = [submission_id] - resultset = epr.query(query, graphs: graphs) - resultset.each do |sol| - s = sol[:s] - p = sol[:p] - o = sol[:o] - if not p.to_s["label"].nil? - index[s.to_s].label = o.to_s - else - index[s.to_s].add_property_value(p,o) - end - end + def self.instances_by_class_where_query(submission, class_id: nil, page_no: nil, size: nil) + where_condition = class_id.nil? ? nil : {types: RDF::URI.new(class_id.to_s)} + query = LinkedData::Models::Instance.where(where_condition).in(submission).include(:types, :label, :prefLabel) + query.page(page_no, size) unless page_no.nil? + query + end + + def self.load_unmapped(submission, models) + LinkedData::Models::Instance.where.in(submission).models(models).include(:unmapped).all end + + end end diff --git a/test/models/test_instances.rb b/test/models/test_instances.rb index 0874294d5..2d77621a8 100644 --- a/test/models/test_instances.rb +++ b/test/models/test_instances.rb @@ -1,26 +1,25 @@ -require_relative "./test_ontology_common" -require "logger" +require_relative './test_ontology_common' +require 'logger' class TestInstances < LinkedData::TestOntologyCommon - PROP_TYPE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" - PROP_CLINICAL_MANIFESTATION = "http://www.owl-ontologies.com/OntologyXCT.owl#isClinicalManifestationOf" - PROP_OBSERVABLE_TRAIT = "http://www.owl-ontologies.com/OntologyXCT.owl#isObservableTraitof" - PROP_HAS_OCCURRENCE = "http://www.owl-ontologies.com/OntologyXCT.owl#hasOccurrenceIn" + PROP_TYPE = RDF::URI.new 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type'.freeze + PROP_CLINICAL_MANIFESTATION = RDF::URI.new 'http://www.owl-ontologies.com/OntologyXCT.owl#isClinicalManifestationOf'.freeze + PROP_OBSERVABLE_TRAIT = RDF::URI.new'http://www.owl-ontologies.com/OntologyXCT.owl#isObservableTraitof'.freeze + PROP_HAS_OCCURRENCE = RDF::URI.new'http://www.owl-ontologies.com/OntologyXCT.owl#hasOccurrenceIn'.freeze + def self.before_suite - LinkedData::TestCase.backend_4s_delete + self.new('').submission_parse('TESTINST', 'Testing instances', + 'test/data/ontology_files/XCTontologyvtemp2_vvtemp2.zip', + 12, + masterFileName: 'XCTontologyvtemp2/XCTontologyvtemp2.owl', + process_rdf: true, extract_metadata: false, generate_missing_labels: false) end def test_instance_counts_class - submission_parse("TESTINST", "Testing instances", - "test/data/ontology_files/XCTontologyvtemp2_vvtemp2.zip", - 12, - masterFileName: "XCTontologyvtemp2/XCTontologyvtemp2.owl", - process_rdf: true, index_search: false, - run_metrics: false, reasoning: true) - submission_id = LinkedData::Models::OntologySubmission.all.first.id - class_id = RDF::URI.new("http://www.owl-ontologies.com/OntologyXCT.owl#ClinicalManifestation") + submission_id = RDF::URI.new("http://data.bioontology.org/ontologies/TESTINST/submissions/12") + class_id = RDF::URI.new('http://www.owl-ontologies.com/OntologyXCT.owl#ClinicalManifestation') instances = LinkedData::InstanceLoader.get_instances_by_class(submission_id, class_id) assert_equal 385, instances.length @@ -30,56 +29,38 @@ def test_instance_counts_class end def test_instance_counts_ontology - submission_parse("TESTINST", "Testing instances", - "test/data/ontology_files/XCTontologyvtemp2_vvtemp2.zip", - 12, - masterFileName: "XCTontologyvtemp2/XCTontologyvtemp2.owl", - process_rdf: true, index_search: false, - run_metrics: false, reasoning: true) - submission_id = LinkedData::Models::OntologySubmission.all.first.id - instances = LinkedData::InstanceLoader.get_instances_by_ontology(submission_id, 1, 800) + submission_id = RDF::URI.new("http://data.bioontology.org/ontologies/TESTINST/submissions/12") + instances = LinkedData::InstanceLoader.get_instances_by_ontology(submission_id, page_no: 1, size: 800) assert_equal 714, instances.length end - def test_instance_labels - submission_parse("TESTINST", "Testing instances", - "test/data/ontology_files/XCTontologyvtemp2_vvtemp2.zip", - 12, - masterFileName: "XCTontologyvtemp2/XCTontologyvtemp2.owl", - process_rdf: true, index_search: false, - run_metrics: false, reasoning: true) - submission_id = LinkedData::Models::OntologySubmission.all.first.id - class_id = RDF::URI.new("http://www.owl-ontologies.com/OntologyXCT.owl#ClinicalManifestation") + def test_instance_types + submission_id = RDF::URI.new("http://data.bioontology.org/ontologies/TESTINST/submissions/12") + class_id = RDF::URI.new('http://www.owl-ontologies.com/OntologyXCT.owl#ClinicalManifestation') - instances = LinkedData::InstanceLoader.get_instances_by_class(submission_id, class_id) + instances = LinkedData::InstanceLoader.get_instances_by_class(submission_id, class_id) instances.each do |inst| - assert (not inst.label.nil?) + assert (not inst.types.nil?) assert (not inst.id.nil?) end inst1 = instances.find {|inst| inst.id.to_s == 'http://www.owl-ontologies.com/OntologyXCT.owl#PresenceofAbnormalFacialShapeAt46'} - assert (not inst1.nil?) - assert_equal 'PresenceofAbnormalFacialShapeAt46', inst1.label + assert !inst1.nil? + assert_includes inst1.types, class_id inst2 = instances.find {|inst| inst.id.to_s == 'http://www.owl-ontologies.com/OntologyXCT.owl#PresenceofGaitDisturbanceAt50'} - assert (not inst2.nil?) - assert_equal 'PresenceofGaitDisturbanceAt50', inst2.label + assert !inst2.nil? + assert_includes inst2.types, class_id end def test_instance_properties known_properties = [PROP_TYPE, PROP_CLINICAL_MANIFESTATION, PROP_OBSERVABLE_TRAIT, PROP_HAS_OCCURRENCE] - submission_parse("TESTINST", "Testing instances", - "test/data/ontology_files/XCTontologyvtemp2_vvtemp2.zip", - 12, - masterFileName: "XCTontologyvtemp2/XCTontologyvtemp2.owl", - process_rdf: true, index_search: false, - run_metrics: false, reasoning: true) - submission_id = LinkedData::Models::OntologySubmission.all.first.id - class_id = RDF::URI.new("http://www.owl-ontologies.com/OntologyXCT.owl#ClinicalManifestation") + submission_id = RDF::URI.new("http://data.bioontology.org/ontologies/TESTINST/submissions/12") + class_id = RDF::URI.new('http://www.owl-ontologies.com/OntologyXCT.owl#ClinicalManifestation') instances = LinkedData::InstanceLoader.get_instances_by_class(submission_id, class_id) - inst = instances.find {|inst| inst.id.to_s == 'http://www.owl-ontologies.com/OntologyXCT.owl#PresenceofThyroidNoduleAt46'} + inst = instances.find {|inst| inst.id.to_s == 'http://www.owl-ontologies.com/OntologyXCT.owl#PresenceofThyroidNoduleAt46'} assert (not inst.nil?) assert_equal 4, inst.properties.length assert_equal known_properties.sort, inst.properties.keys.sort @@ -87,25 +68,26 @@ def test_instance_properties props = inst.properties known_types = [ - "http://www.owl-ontologies.com/OntologyXCT.owl#ClinicalManifestation", - "http://www.w3.org/2002/07/owl#NamedIndividual" + 'http://www.owl-ontologies.com/OntologyXCT.owl#ClinicalManifestation', + 'http://www.w3.org/2002/07/owl#NamedIndividual' ] + types = props[PROP_TYPE].map { |type| type.to_s } assert_equal 2, types.length assert_equal known_types.sort, types.sort manifestations = props[PROP_CLINICAL_MANIFESTATION] assert_equal 1, manifestations.length - assert_equal "http://www.owl-ontologies.com/OntologyXCT.owl#Patient_11_1", manifestations.first.to_s + assert_equal 'http://www.owl-ontologies.com/OntologyXCT.owl#Patient_11_1', manifestations.first.to_s observables = props[PROP_OBSERVABLE_TRAIT] assert_equal 1, observables.length - assert_equal "http://www.owl-ontologies.com/OntologyXCT.owl#PresenceofThyroidNodule", observables.first.to_s + assert_equal 'http://www.owl-ontologies.com/OntologyXCT.owl#PresenceofThyroidNodule', observables.first.to_s occurrences = props[PROP_HAS_OCCURRENCE] assert_equal 1, occurrences.length assert (occurrences.first.is_a? RDF::Literal) - assert_equal "46", occurrences.first.value + assert_equal '46', occurrences.first.value end end From 89d7675600ec80fba057b00fc720e2d9352753cf Mon Sep 17 00:00:00 2001 From: Syphax Date: Thu, 19 Jun 2025 09:59:25 +0200 Subject: [PATCH 02/19] extract the classes tree and sorting to external modules --- .../concerns/concepts/concept_sort.rb | 55 +++++ .../concerns/concepts/concept_tree.rb | 139 ++++++++++++ lib/ontologies_linked_data/models/class.rb | 202 +++--------------- 3 files changed, 224 insertions(+), 172 deletions(-) create mode 100644 lib/ontologies_linked_data/concerns/concepts/concept_sort.rb create mode 100644 lib/ontologies_linked_data/concerns/concepts/concept_tree.rb diff --git a/lib/ontologies_linked_data/concerns/concepts/concept_sort.rb b/lib/ontologies_linked_data/concerns/concepts/concept_sort.rb new file mode 100644 index 000000000..1c42dcfa2 --- /dev/null +++ b/lib/ontologies_linked_data/concerns/concepts/concept_sort.rb @@ -0,0 +1,55 @@ +module LinkedData + module Concerns + module Concept + module Sort + module ClassMethods + def compare_classes(class_a, class_b) + label_a = "" + label_b = "" + class_a.bring(:prefLabel) if class_a.bring?(:prefLabel) + class_b.bring(:prefLabel) if class_b.bring?(:prefLabel) + + begin + label_a = class_a.prefLabel unless (class_a.prefLabel.nil? || class_a.prefLabel.empty?) + rescue Goo::Base::AttributeNotLoaded + label_a = "" + end + + begin + label_b = class_b.prefLabel unless (class_b.prefLabel.nil? || class_b.prefLabel.empty?) + rescue Goo::Base::AttributeNotLoaded + label_b = "" + end + + label_a = class_a.id if label_a.empty? + label_b = class_b.id if label_b.empty? + + [label_a.downcase] <=> [label_b.downcase] + end + + def sort_classes(classes) + classes.sort { |class_a, class_b| compare_classes(class_a, class_b) } + end + + def sort_tree_children(root_node) + sort_classes!(root_node.children) + root_node.children.each { |ch| sort_tree_children(ch) } + end + + private + + + + def sort_classes!(classes) + classes.sort! { |class_a, class_b| LinkedData::Models::Class.compare_classes(class_a, class_b) } + classes + end + end + + def self.included(base) + base.extend(ClassMethods) + end + end + end + end +end diff --git a/lib/ontologies_linked_data/concerns/concepts/concept_tree.rb b/lib/ontologies_linked_data/concerns/concepts/concept_tree.rb new file mode 100644 index 000000000..def296828 --- /dev/null +++ b/lib/ontologies_linked_data/concerns/concepts/concept_tree.rb @@ -0,0 +1,139 @@ +module LinkedData + module Concerns + module Concept + module Tree + def tree(concept_schemes: [], concept_collections: [], roots: nil) + bring(parents: [:prefLabel]) if bring?(:parents) + return self if parents.nil? || parents.empty? + extra_include = [:hasChildren, :isInActiveScheme, :isInActiveCollection] + roots = self.submission.roots( extra_include, concept_schemes:concept_schemes) if roots.nil? + path = path_to_root(roots) + threshold = 99 + + return self if path.nil? + + attrs_to_load = %i[prefLabel synonym obsolete] + attrs_to_load << :subClassOf if submission.hasOntologyLanguage.obo? + attrs_to_load += self.class.concept_is_in_attributes if submission.skos? + self.class.in(submission) + .models(path) + .include(attrs_to_load).all + load_children(path, threshold: threshold) + + path.reverse! + path.last.instance_variable_set("@children", []) + + childrens_hash = {} + path.each do |m| + next if m.id.to_s["#Thing"] + m.children.each do |c| + childrens_hash[c.id.to_s] = c + c.load_computed_attributes(to_load:extra_include , + options: {schemes: concept_schemes, collections: concept_collections}) + end + m.load_computed_attributes(to_load:extra_include , + options: {schemes: concept_schemes, collections: concept_collections}) + end + + load_children(childrens_hash.values, threshold: threshold) + + build_tree(path) + end + + def tree_sorted(concept_schemes: [], concept_collections: [], roots: nil) + tr = tree(concept_schemes: concept_schemes, concept_collections: concept_collections, roots: roots) + self.class.sort_tree_children(tr) + tr + end + + def paths_to_root(tree: false, roots: nil) + bring(parents: [:prefLabel, :synonym, :definition]) if bring?(:parents) + return [] if parents.nil? || parents.empty? + + paths = [[self]] + traverse_path_to_root(self.parents.dup, paths, 0, tree, roots) unless tree_root?(self, roots) + paths.each do |p| + p.reverse! + end + paths + end + + def path_to_root(roots) + paths = [[self]] + paths = paths_to_root(tree: true, roots: roots) + #select one path that gets to root + path = nil + paths.each do |p| + p.reverse! + unless (p.map { |x| x.id.to_s } & roots.map { |x| x.id.to_s }).empty? + path = p + break + end + end + + if path.nil? + # do one more check for root classes that don't get returned by the submission.roots call + paths.each do |p| + root_node = p.last + root_parents = root_node.parents + + if root_parents.empty? + path = p + break + end + end + end + + path + end + + def tree_root?(concept, roots) + (roots &&roots.map{|r| r.id}.include?(concept.id)) || concept.id.to_s["#Thing"] + end + + private + + def load_children(concepts, threshold: 99) + LinkedData::Models::Class + .partially_load_children(concepts, threshold, submission) + end + + def build_tree(path) + root_node = path.first + tree_node = path.first + path.delete_at(0) + while tree_node && + !tree_node.id.to_s["#Thing"] && + !tree_node.children.empty? && (!path.empty?) do + next_tree_node = nil + tree_node.load_has_children + tree_node.children.each_index do |i| + if tree_node.children[i].id.to_s == path.first.id.to_s + next_tree_node = path.first + children = tree_node.children.dup + children[i] = path.first + tree_node.instance_variable_set("@children", children) + children.each do |c| + c.load_has_children + end + else + tree_node.children[i].instance_variable_set("@children", []) + end + end + + if !path.empty? && next_tree_node.nil? + tree_node.children << path.shift + end + tree_node = next_tree_node + path.delete_at(0) + end + + root_node + end + + end + end + + end +end + diff --git a/lib/ontologies_linked_data/models/class.rb b/lib/ontologies_linked_data/models/class.rb index 471a3afb7..556ecfbeb 100644 --- a/lib/ontologies_linked_data/models/class.rb +++ b/lib/ontologies_linked_data/models/class.rb @@ -10,6 +10,8 @@ class ClassAttributeNotLoaded < StandardError end class Class < LinkedData::Models::Base + include LinkedData::Concerns::Concept::Sort + include LinkedData::Concerns::Concept::Tree model :class, name_with: :id, collection: :submission, namespace: :owl, :schemaless => :true, @@ -352,23 +354,13 @@ def properties(*args) properties end - def paths_to_root() - self.bring(parents: [:prefLabel, :synonym, :definition]) if self.bring?(:parents) - return [] if self.parents.nil? or self.parents.length == 0 - paths = [[self]] - traverse_path_to_root(self.parents.dup, paths, 0) - paths.each do |p| - p.reverse! - end - paths - end - def self.partially_load_children(models, threshold, submission) ld = [:prefLabel, :definition, :synonym] ld << :subClassOf if submission.hasOntologyLanguage.obo? + ld += LinkedData::Models::Class.concept_is_in_attributes if submission.skos? + single_load = [] - query = self.in(submission) - .models(models) + query = self.in(submission).models(models) query.aggregate(:count, :children).all models.each do |cls| @@ -377,13 +369,10 @@ def self.partially_load_children(models, threshold, submission) end if cls.aggregates.first.value > threshold #too many load a page - self.in(submission) - .models(single_load) - .include(children: [:prefLabel]).all page_children = LinkedData::Models::Class - .where(parents: cls) - .include(ld) - .in(submission).page(1,threshold).all + .where(parents: cls) + .include(ld) + .in(submission).page(1,threshold).all cls.instance_variable_set("@children",page_children.to_a) cls.loaded_attributes.add(:children) @@ -392,109 +381,17 @@ def self.partially_load_children(models, threshold, submission) end end - if single_load.length > 0 - self.in(submission) - .models(single_load) - .include(ld << {children: [:prefLabel]}).all - end + self.in(submission).models(single_load).include({children: ld}).all if single_load.length > 0 end - def tree() - self.bring(parents: [:prefLabel]) if self.bring?(:parents) - return self if self.parents.nil? or self.parents.length == 0 - paths = [[self]] - traverse_path_to_root(self.parents.dup, paths, 0, tree=true) - roots = self.submission.roots(extra_include=[:hasChildren]) - threshhold = 99 - - #select one path that gets to root - path = nil - paths.each do |p| - if (p.map { |x| x.id.to_s } & roots.map { |x| x.id.to_s }).length > 0 - path = p - break - end - end - - if path.nil? - # do one more check for root classes that don't get returned by the submission.roots call - paths.each do |p| - root_node = p.last - root_parents = root_node.parents - - if root_parents.empty? - path = p - break - end - end - return self if path.nil? - end - - items_hash = {} - path.each do |t| - items_hash[t.id.to_s] = t - end - - attrs_to_load = [:prefLabel,:synonym,:obsolete] - attrs_to_load << :subClassOf if submission.hasOntologyLanguage.obo? - self.class.in(submission) - .models(items_hash.values) - .include(attrs_to_load).all - - LinkedData::Models::Class - .partially_load_children(items_hash.values,threshhold,self.submission) - - path.reverse! - path.last.instance_variable_set("@children",[]) - childrens_hash = {} - path.each do |m| - next if m.id.to_s["#Thing"] - m.children.each do |c| - childrens_hash[c.id.to_s] = c - end - end - - LinkedData::Models::Class.partially_load_children(childrens_hash.values,threshhold, self.submission) - - #build the tree - root_node = path.first - tree_node = path.first - path.delete_at(0) - while tree_node && - !tree_node.id.to_s["#Thing"] && - tree_node.children.length > 0 and path.length > 0 do - - next_tree_node = nil - tree_node.load_has_children - tree_node.children.each_index do |i| - if tree_node.children[i].id.to_s == path.first.id.to_s - next_tree_node = path.first - children = tree_node.children.dup - children[i] = path.first - tree_node.instance_variable_set("@children",children) - children.each do |c| - c.load_has_children - end - else - tree_node.children[i].instance_variable_set("@children",[]) - end - end - - if path.length > 0 && next_tree_node.nil? - tree_node.children << path.shift - end - - tree_node = next_tree_node - path.delete_at(0) - end - - root_node + def load_computed_attributes(to_load:, options:) + self.load_has_children if to_load&.include?(:hasChildren) + self.load_is_in_scheme(options[:schemes]) if to_load&.include?(:isInActiveScheme) + self.load_is_in_collection(options[:collections]) if to_load&.include?(:isInActiveCollection) end - def tree_sorted() - tr = tree - self.class.sort_tree_children(tr) - tr + def self.concept_is_in_attributes + [:inScheme, :isInActiveScheme, :memberOf, :isInActiveCollection] end def retrieve_ancestors() @@ -636,11 +533,11 @@ def append_if_not_there_already(path, r) path << r end - def traverse_path_to_root(parents, paths, path_i, tree=false) - return if (tree and parents.length == 0) + def traverse_path_to_root(parents, paths, path_i, tree = false, roots = nil) + return if (tree && parents.length == 0) + recursions = [path_i] recurse_on_path = [false] - if parents.length > 1 and not tree (parents.length-1).times do paths << paths[path_i].clone @@ -651,7 +548,7 @@ def traverse_path_to_root(parents, paths, path_i, tree=false) parents.each_index do |i| rec_i = recursions[i] recurse_on_path[i] = recurse_on_path[i] || - !append_if_not_there_already(paths[rec_i], parents[i]).nil? + !append_if_not_there_already(paths[rec_i], parents[i]).nil? end else path = paths[path_i] @@ -664,62 +561,23 @@ def traverse_path_to_root(parents, paths, path_i, tree=false) p = path.last next if p.id.to_s["umls/OrphanClass"] - if p.bring?(:parents) - p.bring(parents: [:prefLabel, :synonym, :definition, parents: [:prefLabel, :synonym, :definition]]) - end + if !tree_root?(p, roots) && recurse_on_path[i] + if p.bring?(:parents) + p.bring(parents: [:prefLabel, :synonym, :definition, :inScheme, parents: [:prefLabel, :synonym, :definition, :inScheme]]) + end - if !p.loaded_attributes.include?(:parents) - # fail safely - logger = LinkedData::Parser.logger || Logger.new($stderr) - logger.error("Class #{p.id.to_s} from #{p.submission.id} cannot load parents") - return - end + if !p.loaded_attributes.include?(:parents) + # fail safely + logger = LinkedData::Parser.logger || Logger.new($stderr) + logger.error("Class #{p.id.to_s} from #{p.submission.id} cannot load parents") + return + end - if !p.id.to_s["#Thing"] &&\ - (recurse_on_path[i] && p.parents && p.parents.length > 0) - traverse_path_to_root(p.parents.dup, paths, rec_i, tree=tree) + traverse_path_to_root(p.parents.dup, paths, rec_i, tree=tree, roots=roots) end end end - def self.sort_tree_children(root_node) - self.sort_classes!(root_node.children) - root_node.children.each { |ch| self.sort_tree_children(ch) } - end - - def self.sort_classes(classes) - classes.sort { |class_a, class_b| self.compare_classes(class_a, class_b) } - end - - def self.sort_classes!(classes) - classes.sort! { |class_a, class_b| self.compare_classes(class_a, class_b) } - classes - end - - def self.compare_classes(class_a, class_b) - label_a = "" - label_b = "" - class_a.bring(:prefLabel) if class_a.bring?(:prefLabel) - class_b.bring(:prefLabel) if class_b.bring?(:prefLabel) - - begin - label_a = class_a.prefLabel unless (class_a.prefLabel.nil? || class_a.prefLabel.empty?) - rescue Goo::Base::AttributeNotLoaded - label_a = "" - end - - begin - label_b = class_b.prefLabel unless (class_b.prefLabel.nil? || class_b.prefLabel.empty?) - rescue Goo::Base::AttributeNotLoaded - label_b = "" - end - - label_a = class_a.id if label_a.empty? - label_b = class_b.id if label_b.empty? - - [label_a.downcase] <=> [label_b.downcase] - end - end end end From 7fce23e51a067891d509b019a784ddd9cd17eb90 Mon Sep 17 00:00:00 2001 From: Syphax Date: Thu, 19 Jun 2025 10:01:53 +0200 Subject: [PATCH 03/19] add the skos new models for collection, schemes and skosxl with tests --- .../models/skos/collection.rb | 43 ++ .../models/skos/scheme.rb | 35 ++ .../models/skos/skosxl.rb | 25 ++ test/models/skos/test_collections.rb | 58 +++ test/models/skos/test_schemes.rb | 366 ++++++++++++++++++ test/models/skos/test_skos_xl.rb | 67 ++++ 6 files changed, 594 insertions(+) create mode 100644 lib/ontologies_linked_data/models/skos/collection.rb create mode 100644 lib/ontologies_linked_data/models/skos/scheme.rb create mode 100644 lib/ontologies_linked_data/models/skos/skosxl.rb create mode 100644 test/models/skos/test_collections.rb create mode 100644 test/models/skos/test_schemes.rb create mode 100644 test/models/skos/test_skos_xl.rb diff --git a/lib/ontologies_linked_data/models/skos/collection.rb b/lib/ontologies_linked_data/models/skos/collection.rb new file mode 100644 index 000000000..afc5724e4 --- /dev/null +++ b/lib/ontologies_linked_data/models/skos/collection.rb @@ -0,0 +1,43 @@ +module LinkedData + module Models + module SKOS + class Collection < LinkedData::Models::Base + + model :collection, name_with: :id, collection: :submission, + namespace: :skos, schemaless: :true, rdf_type: ->(*x) { RDF::SKOS[:Collection] } + + attribute :prefLabel, namespace: :skos, enforce: [:existence] + attribute :member, namespace: :skos, enforce: [:list, :class] + attribute :submission, collection: ->(s) { s.resource_id }, namespace: :metadata + + embed :member + serialize_default :prefLabel, :memberCount + serialize_never :submission, :id, :member + serialize_methods :properties, :memberCount + aggregates memberCount: [:count, :member] + + cache_timeout 14400 + + link_to LinkedData::Hypermedia::Link.new('self', + ->(s) { "ontologies/#{s.submission.ontology.acronym}/collections/#{CGI.escape(s.id.to_s)}"}, + self.uri_type), + LinkedData::Hypermedia::Link.new('members', + ->(s) { "ontologies/#{s.submission.ontology.acronym}/collections/#{CGI.escape(s.id.to_s)}/members"}, + Goo.vocabulary(:skos)['Concept']), + LinkedData::Hypermedia::Link.new('ontology', ->(s) { "ontologies/#{s.submission.ontology.acronym}"}, + Goo.vocabulary['Ontology']) + + def properties + self.unmapped + end + + def memberCount + sol = self.class.in(submission).models([self]).aggregate(:count, :member).first + sol.nil? ? 0 : sol.aggregates.first.value + end + + end + end + end + +end diff --git a/lib/ontologies_linked_data/models/skos/scheme.rb b/lib/ontologies_linked_data/models/skos/scheme.rb new file mode 100644 index 000000000..37e041896 --- /dev/null +++ b/lib/ontologies_linked_data/models/skos/scheme.rb @@ -0,0 +1,35 @@ +module LinkedData + module Models + module SKOS + class Scheme < LinkedData::Models::Base + + model :scheme, name_with: :id, collection: :submission, + namespace: :skos, schemaless: :true, rdf_type: ->(*x) { RDF::SKOS[:ConceptScheme] } + + attribute :prefLabel, namespace: :skos, enforce: [:existence] + + attribute :submission, collection: ->(s) { s.resource_id }, namespace: :metadata + + serialize_never :submission, :id + serialize_methods :properties + + cache_timeout 14400 + + link_to LinkedData::Hypermedia::Link.new('self', + ->(s) { "ontologies/#{s.submission.ontology.acronym}/schemes/#{CGI.escape(s.id.to_s)}"}, + self.uri_type), + LinkedData::Hypermedia::Link.new('roots', + ->(s) { "ontologies/#{s.submission.ontology.acronym}/classes/roots?concept_scheme=#{CGI.escape(s.id.to_s)}"}, + Goo.vocabulary(:skos)['Concept']), + LinkedData::Hypermedia::Link.new('ontology', ->(s) { "ontologies/#{s.submission.ontology.acronym}"}, + Goo.vocabulary['Ontology']) + + def properties + self.unmapped + end + + end + end + end + +end diff --git a/lib/ontologies_linked_data/models/skos/skosxl.rb b/lib/ontologies_linked_data/models/skos/skosxl.rb new file mode 100644 index 000000000..9b95b7cc4 --- /dev/null +++ b/lib/ontologies_linked_data/models/skos/skosxl.rb @@ -0,0 +1,25 @@ +module LinkedData + module Models + module SKOS + class Label < LinkedData::Models::Base + + model :label, name_with: :id, collection: :submission, + namespace: :skos, rdf_type: ->(*x) { RDF::URI.new('http://www.w3.org/2008/05/skos-xl#Label') } + + attribute :literalForm, namespace: :skosxl, enforce: [:existence] + attribute :submission, collection: ->(s) { s.resource_id }, namespace: :metadata + + serialize_never :submission, :id + serialize_methods :properties + + link_to LinkedData::Hypermedia::Link.new('self', ->(s) { "ontologies/#{s.submission.ontology.acronym}/skos_xl_labels/#{CGI.escape(s.id)}"}, self.uri_type) + + def properties + self.unmapped + end + + end + end + end + +end diff --git a/test/models/skos/test_collections.rb b/test/models/skos/test_collections.rb new file mode 100644 index 000000000..c2c1c3d07 --- /dev/null +++ b/test/models/skos/test_collections.rb @@ -0,0 +1,58 @@ +require_relative '../test_ontology_common' +require 'logger' + +class TestCollections < LinkedData::TestOntologyCommon + + def self.before_suite + LinkedData::TestCase.backend_4s_delete + self.new('').submission_parse('INRAETHES', 'Testing skos', + 'test/data/ontology_files/thesaurusINRAE_nouv_structure.skos', + 1, + process_rdf: true, index_search: false, + run_metrics: false, reasoning: false) + end + + def test_collections_all + ont = 'INRAETHES' + sub = LinkedData::Models::Ontology.find(ont).first.latest_submission + collections = LinkedData::Models::SKOS::Collection.in(sub).include(:members, :prefLabel).all + + assert_equal 2, collections.size + collections_test = test_data + + collections.each_with_index do |x, i| + collection_test = collections_test[i] + assert_equal collection_test[:id], x.id.to_s + assert_equal collection_test[:prefLabel], x.prefLabel + assert_equal collection_test[:memberCount], x.memberCount + end + end + + def test_collection_members + ont = 'INRAETHES' + sub = LinkedData::Models::Ontology.find(ont).first.latest_submission + collection_test = test_data.first + collection = LinkedData::Models::SKOS::Collection.find(collection_test[:id]).in(sub).include(:member, :prefLabel).first + + refute_nil collection + members = collection.member + assert_equal collection_test[:memberCount], members.size + end + + private + + def test_data + [ + { + "id": 'http://opendata.inrae.fr/thesaurusINRAE/gr_6c79e7c5', + "prefLabel": 'GR. DEFINED CONCEPTS', + "memberCount": 295 + }, + { + "prefLabel": 'GR. DISCIPLINES', + "memberCount": 233, + "id": 'http://opendata.inrae.fr/thesaurusINRAE/skosCollection_e25f9c62' + } + ] + end +end diff --git a/test/models/skos/test_schemes.rb b/test/models/skos/test_schemes.rb new file mode 100644 index 000000000..41effac6c --- /dev/null +++ b/test/models/skos/test_schemes.rb @@ -0,0 +1,366 @@ +require_relative '../test_ontology_common' +require 'logger' + +class TestSchemes < LinkedData::TestOntologyCommon + + def self.before_suite + LinkedData::TestCase.backend_4s_delete + self.new('').submission_parse('INRAETHES', 'Testing skos', + 'test/data/ontology_files/thesaurusINRAE_nouv_structure.skos', + 1, + process_rdf: true, index_search: false, + run_metrics: false, reasoning: false) + end + + def test_schemes_all + ont = 'INRAETHES' + sub = LinkedData::Models::Ontology.find(ont).first.latest_submission + schemes = LinkedData::Models::SKOS::Scheme.in(sub).include(:prefLabel).all + + assert_equal 66, schemes.size + schemes_test = test_data + + schemes.each_with_index do |x, i| + scheme_test = schemes_test[i] + assert_equal scheme_test[:id], x.id.to_s + assert_equal scheme_test[:prefLabel], x.prefLabel + end + end + + private + + def test_data + [ + { + "prefLabel": 'BIO neurosciences', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_74', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'INRAE domains', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/domainesINRAE', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'EAR meteorology and climatology', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_107', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'HEA prevention and therapy', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_77', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'PHY materials sciences', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_85', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'BIO cell biology', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_64', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'SSH human geography', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_20661', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'BIO immunology', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_75', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'APP variables, parameters and data', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_23256', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'SSH information and communication', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_20962', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'BIO diet and nutrition', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_23276', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'SSH research and education', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_20150', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'CON processing technology and equipment', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_54', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'BIO molecular biology and biochemistry', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_65', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'EAR soil sciences', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_105', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'BIO toxicology and ecotoxicology', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_68', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'AGR farms and farming systems', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_44', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'AGR plant cultural practices and experimentations', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_47', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'CHE chemical and physicochemical analysis', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_23260', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'CON quality of processed products', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_55', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'EAR geology and geomorphology', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_104', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'APP research methods', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_98', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'SSH laws and standards', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_21670', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'BIO general biology', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_26224', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'BIO ethology', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_69', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'PHY energy and thermodynamics', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_86', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'PHY mechanics and robotics', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_88', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'HEA health and welfare', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_78', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'ENV environment and natural resources', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_14', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'PHY civil engineering', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_89', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'HEA diseases, disorders and symptoms', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_76', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'HEA disease vectors', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_79', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'ENV natural and technological hazards', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_17', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'CHE chemical compounds and elements', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_100', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'ENV waste', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_15', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'CON supply chain management', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_56', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'AGR animal husbandry and breeding', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_48', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'ENV water management', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_18', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'AGR agricultural products', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_46', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'MAT computer sciences and artificial intelligence', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_91', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'AGR agricultural machinery and equipment', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_49', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'BIO microbiology', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_71', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'ENV pollution', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_16', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'BIO physiology', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_72', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'SSH culture and humanities', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_26297', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'MAT mathematics and statistics', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_90', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": nil, + "id": 'http://opendata.inrae.fr/thesaurusINRAE/thesaurusINRAE', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'SSH politics and administration', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_22445', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'EAR hydrology', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_106', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'AGR agricultural management', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_26298', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'SSH management sciences', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_21074', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'SSH economics', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_20544', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'BIO anatomy and body fluids', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_63', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'ORG taxonomic classification of organisms', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_26190', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'CHE chemical reactions and physicochemical phenomena', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_102', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'AGR hunting and fishing', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_50', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'CON processed biobased products', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_53', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'BIO genetics', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_70', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'ORG organisms related notions', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_26191', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'SSH sociology and psychology', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_20262', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'APP research equipment', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_97', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'CHE chemical and physicochemical properties', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_101', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'BIO ecology', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_67', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'PHY physical properties of matter', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_84', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'EAR physical geography', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_103', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'PHY hydraulics and aeraulics', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_87', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + } + ] + end +end diff --git a/test/models/skos/test_skos_xl.rb b/test/models/skos/test_skos_xl.rb new file mode 100644 index 000000000..5b05b8063 --- /dev/null +++ b/test/models/skos/test_skos_xl.rb @@ -0,0 +1,67 @@ +require_relative '../test_ontology_common' +require 'logger' + +class TestSkosXlLabel < LinkedData::TestOntologyCommon + + def self.before_suite + LinkedData::TestCase.backend_4s_delete + self.new('').submission_parse('INRAETHES', 'Testing skos', + 'test/data/ontology_files/thesaurusINRAE_nouv_structure.skos', + 1, + process_rdf: true, index_search: false, + run_metrics: false, reasoning: false) + end + + def test_skos_xl_label_all + ont = 'INRAETHES' + sub = LinkedData::Models::Ontology.find(ont).first.latest_submission + labels = LinkedData::Models::SKOS::Label.in(sub).include(:literalForm).all + assert_equal 2, labels.size + tests_labels = test_data + labels.each do |label| + test_label = tests_labels.select { |x| x[:id].eql?(label.id.to_s) } + refute_nil test_label.first + label_test(label, test_label.first) + end + end + + def test_class_skos_xl_label + ont = 'INRAETHES' + ont = LinkedData::Models::Ontology.find(ont).first + sub = ont.latest_submission + + sub.bring_remaining + sub.hasOntologyLanguage = LinkedData::Models::OntologyFormat.find('SKOS').first + sub.save + + class_test = LinkedData::Models::Class.find('http://opendata.inrae.fr/thesaurusINRAE/c_16193') + .in(sub).include(:prefLabel, + altLabelXl: [:literalForm], + prefLabelXl: [:literalForm], + hiddenLabelXl: [:literalForm]).first + + refute_nil class_test + assert_equal 1, class_test.altLabelXl.size + assert_equal 1, class_test.prefLabelXl.size + assert_equal 1, class_test.hiddenLabelXl.size + tests_labels = test_data + + label_test(class_test.altLabelXl.first, tests_labels[0]) + label_test(class_test.prefLabelXl.first, tests_labels[1]) + label_test(class_test.hiddenLabelXl.first, tests_labels[1]) + end + + private + + def test_data + [ + { id: 'http://aims.fao.org/aos/agrovoc/xl_tr_1331561625299', literalForm: 'aktivite' }, + { id: 'http://aims.fao.org/aos/agrovoc/xl_en_668053a7', literalForm: 'air-water exchanges' } + ] + end + + def label_test(label, label_test) + assert_equal label_test[:id], label.id.to_s + assert_equal label_test[:literalForm], label.literalForm + end +end \ No newline at end of file From 3d03c1fd95bd29791e89ad1ae75dba88539fbca6 Mon Sep 17 00:00:00 2001 From: Syphax Date: Thu, 19 Jun 2025 10:06:24 +0200 Subject: [PATCH 04/19] add for classes the properties to know on which scheme and collection they belong --- .../concepts/concept_in_collection.rb | 25 ++++++++++++++++++ .../concerns/concepts/concept_in_scheme.rb | 26 +++++++++++++++++++ lib/ontologies_linked_data/models/class.rb | 15 ++++++++--- 3 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 lib/ontologies_linked_data/concerns/concepts/concept_in_collection.rb create mode 100644 lib/ontologies_linked_data/concerns/concepts/concept_in_scheme.rb diff --git a/lib/ontologies_linked_data/concerns/concepts/concept_in_collection.rb b/lib/ontologies_linked_data/concerns/concepts/concept_in_collection.rb new file mode 100644 index 000000000..1707dea35 --- /dev/null +++ b/lib/ontologies_linked_data/concerns/concepts/concept_in_collection.rb @@ -0,0 +1,25 @@ +module LinkedData + module Concerns + module Concept + module InCollection + def self.included(base) + base.serialize_methods :isInActiveCollection + end + + def isInActiveCollection + @isInActiveCollection + end + + def inCollection?(collection) + self.memberOf.include?(collection) + end + + def load_is_in_collection(collections = []) + included = collections.select { |s| inCollection?(s) } + @isInActiveCollection = included + end + + end + end + end +end diff --git a/lib/ontologies_linked_data/concerns/concepts/concept_in_scheme.rb b/lib/ontologies_linked_data/concerns/concepts/concept_in_scheme.rb new file mode 100644 index 000000000..ba3592d2c --- /dev/null +++ b/lib/ontologies_linked_data/concerns/concepts/concept_in_scheme.rb @@ -0,0 +1,26 @@ +module LinkedData + module Concerns + module Concept + module InScheme + def self.included(base) + base.serialize_methods :isInActiveScheme + end + + def isInActiveScheme + @isInActiveScheme + end + + def inScheme?(scheme) + self.inScheme.include?(scheme) + end + + def load_is_in_scheme(schemes = []) + included = schemes.select { |s| inScheme?(s) } + included = [self.submission.get_main_concept_scheme] if included.empty? && schemes&.empty? + @isInActiveScheme = included + end + + end + end + end +end diff --git a/lib/ontologies_linked_data/models/class.rb b/lib/ontologies_linked_data/models/class.rb index 556ecfbeb..c5fb20d90 100644 --- a/lib/ontologies_linked_data/models/class.rb +++ b/lib/ontologies_linked_data/models/class.rb @@ -12,6 +12,8 @@ class ClassAttributeNotLoaded < StandardError class Class < LinkedData::Models::Base include LinkedData::Concerns::Concept::Sort include LinkedData::Concerns::Concept::Tree + include LinkedData::Concerns::Concept::InScheme + include LinkedData::Concerns::Concept::InCollection model :class, name_with: :id, collection: :submission, namespace: :owl, :schemaless => :true, @@ -43,6 +45,9 @@ def self.urn_id(acronym,classId) attribute :label, namespace: :rdfs, enforce: [:list] attribute :prefLabel, namespace: :skos, enforce: [:existence], alias: true + attribute :prefLabelXl, property: :prefLabel, namespace: :skosxl, enforce: [:label, :list], alias: true + attribute :altLabelXl, property: :altLabel, namespace: :skosxl, enforce: [:label, :list], alias: true + attribute :hiddenLabelXl, property: :hiddenLabel, namespace: :skosxl, enforce: [:label, :list], alias: true attribute :synonym, namespace: :skos, enforce: [:list], property: :altLabel, alias: true attribute :definition, namespace: :skos, enforce: [:list], alias: true attribute :obsolete, namespace: :owl, property: :deprecated, alias: true @@ -79,18 +84,22 @@ def self.urn_id(acronym,classId) attribute :notes, inverse: { on: :note, attribute: :relatedClass } + attribute :inScheme, enforce: [:list, :uri], namespace: :skos + attribute :memberOf, namespace: :uneskos, inverse: { on: :collection , :attribute => :member } attribute :created, namespace: :dcterms attribute :modified, namespace: :dcterms # Hypermedia settings - embed :children, :ancestors, :descendants, :parents - serialize_default :prefLabel, :synonym, :definition, :cui, :semanticType, :obsolete, :matchType, :ontologyType, :provisional # an attribute used in Search (not shown out of context) + embed :children, :ancestors, :descendants, :parents, :prefLabelXl, :altLabelXl, :hiddenLabelXl + serialize_default :prefLabel, :synonym, :definition, :cui, :semanticType, :obsolete, :matchType, + :ontologyType, :provisional, # an attribute used in Search (not shown out of context) + :created, :modified, :memberOf, :inScheme serialize_methods :properties, :childrenCount, :hasChildren serialize_never :submissionAcronym, :submissionId, :submission, :descendants aggregates childrenCount: [:count, :children] links_load submission: [ontology: [:acronym]] do_not_load :descendants, :ancestors - prevent_serialize_when_nested :properties, :parents, :children, :ancestors, :descendants + prevent_serialize_when_nested :properties, :parents, :children, :ancestors, :descendants, :memberOf link_to LinkedData::Hypermedia::Link.new("self", lambda {|s| "ontologies/#{s.submission.ontology.acronym}/classes/#{CGI.escape(s.id.to_s)}"}, self.uri_type), LinkedData::Hypermedia::Link.new("ontology", lambda {|s| "ontologies/#{s.submission.ontology.acronym}"}, Goo.vocabulary["Ontology"]), LinkedData::Hypermedia::Link.new("children", lambda {|s| "ontologies/#{s.submission.ontology.acronym}/classes/#{CGI.escape(s.id.to_s)}/children"}, self.uri_type), From bcea2388bfed35a9a2be97b563beddedc87deaa5 Mon Sep 17 00:00:00 2001 From: Syphax Date: Thu, 19 Jun 2025 10:07:07 +0200 Subject: [PATCH 05/19] update the logic of submission roots to filter by schemes or collections --- .../skos/skos_submission_roots.rb | 99 ++++++++++++++ .../skos/skos_submission_schemes.rb | 20 +++ .../models/ontology_submission.rb | 122 ++++++++++-------- 3 files changed, 188 insertions(+), 53 deletions(-) create mode 100644 lib/ontologies_linked_data/concerns/ontology_submissions/skos/skos_submission_roots.rb create mode 100644 lib/ontologies_linked_data/concerns/ontology_submissions/skos/skos_submission_schemes.rb diff --git a/lib/ontologies_linked_data/concerns/ontology_submissions/skos/skos_submission_roots.rb b/lib/ontologies_linked_data/concerns/ontology_submissions/skos/skos_submission_roots.rb new file mode 100644 index 000000000..4a88c12a0 --- /dev/null +++ b/lib/ontologies_linked_data/concerns/ontology_submissions/skos/skos_submission_roots.rb @@ -0,0 +1,99 @@ +module LinkedData + module Models + module SKOS + module RootsFetcher + + def skos_roots(concept_schemes, page, paged, pagesize) + classes = [] + class_ids, count = roots_by_has_top_concept(concept_schemes, page, paged, pagesize) + + class_ids, count = roots_by_top_concept_of(concept_schemes, page, paged, pagesize) if class_ids.empty? + + class_ids.each do |id| + classes << LinkedData::Models::Class.find(id).in(self).disable_rules.first + end + + classes = Goo::Base::Page.new(page, pagesize, count, classes) if paged + classes + end + + private + + def roots_by_query(query_body, page, paged, pagesize) + root_skos = <<-eos + SELECT DISTINCT ?root WHERE { + GRAPH #{self.id.to_ntriples} { + #{query_body} + }} + eos + count = 0 + + count, root_skos = add_pagination(query_body, page, pagesize, root_skos) if paged + + #needs to get cached + class_ids = [] + + Goo.sparql_query_client.query(root_skos, { graphs: [self.id] }).each_solution do |s| + class_ids << s[:root] + end + + [class_ids, count] + end + + def roots_by_has_top_concept(concept_schemes, page, paged, pagesize) + query_body = <<-eos + ?x #{RDF::SKOS[:hasTopConcept].to_ntriples} ?root . + #{concept_schemes_filter(concept_schemes)} + eos + roots_by_query query_body, page, paged, pagesize + end + + def roots_by_top_concept_of(concept_schemes, page, paged, pagesize) + query_body = <<-eos + ?root #{RDF::SKOS[:topConceptOf].to_ntriples} ?x. + #{concept_schemes_filter(concept_schemes)} + eos + roots_by_query query_body, page, paged, pagesize + end + + def add_pagination(query_body, page, pagesize, root_skos) + count = count_roots(query_body) + + offset = (page - 1) * pagesize + root_skos = "#{root_skos} LIMIT #{pagesize} OFFSET #{offset}" + [count, root_skos] + end + + def count_roots(query_body) + query = <<-eos + SELECT (COUNT(?x) as ?count) WHERE { + GRAPH #{self.id.to_ntriples} { + #{query_body} + }} + eos + rs = Goo.sparql_query_client.query(query) + count = 0 + rs.each do |sol| + count = sol[:count].object + end + count + end + + def concept_schemes_filter(concept_schemes) + concept_schemes = current_schemes(concept_schemes) + concept_schemes = concept_schemes.map { |x| RDF::URI.new(x.to_s).to_ntriples } + concept_schemes.empty? ? '' : "FILTER (?x IN (#{concept_schemes.join(',')}))" + end + + def current_schemes(concept_schemes) + if concept_schemes.nil? || concept_schemes.empty? + main_concept_scheme = get_main_concept_scheme + concept_schemes = main_concept_scheme ? [main_concept_scheme] : [] + end + concept_schemes + end + + end + end + end +end diff --git a/lib/ontologies_linked_data/concerns/ontology_submissions/skos/skos_submission_schemes.rb b/lib/ontologies_linked_data/concerns/ontology_submissions/skos/skos_submission_schemes.rb new file mode 100644 index 000000000..b275d4cd4 --- /dev/null +++ b/lib/ontologies_linked_data/concerns/ontology_submissions/skos/skos_submission_schemes.rb @@ -0,0 +1,20 @@ +module LinkedData + module Models + module SKOS + module ConceptSchemes + def get_main_concept_scheme(default_return: ontology_uri) + all = all_concepts_schemes + unless all.nil? + all = all.map { |x| x.id } + return default_return if all.include?(ontology_uri) + end + end + + def all_concepts_schemes + LinkedData::Models::SKOS::Scheme.in(self).all + end + end + end + end +end + diff --git a/lib/ontologies_linked_data/models/ontology_submission.rb b/lib/ontologies_linked_data/models/ontology_submission.rb index b19a18534..0530e15ab 100644 --- a/lib/ontologies_linked_data/models/ontology_submission.rb +++ b/lib/ontologies_linked_data/models/ontology_submission.rb @@ -17,6 +17,9 @@ class OntologySubmission < LinkedData::Models::Base include LinkedData::Concerns::OntologySubmission::Validators extend LinkedData::Concerns::OntologySubmission::DefaultCallbacks + include SKOS::ConceptSchemes + include SKOS::RootsFetcher + FLAT_ROOTS_LIMIT = 1000 # default file permissions for files copied from tempdir REPOSITORY_FILE_MODE = 0o660 # rw-rw---- @@ -218,11 +221,6 @@ def self.embed_values_hash serialize_default :contact, :ontology, :hasOntologyLanguage, :released, :creationDate, :homepage, :publication, :documentation, :version, :description, :status, :submissionId - # Links - links_load :submissionId, ontology: [:acronym] - link_to LinkedData::Hypermedia::Link.new("metrics", lambda {|s| "#{self.ontology_link(s)}/submissions/#{s.submissionId}/metrics"}, self.type_uri) - LinkedData::Hypermedia::Link.new("download", lambda {|s| "#{self.ontology_link(s)}/submissions/#{s.submissionId}/download"}, self.type_uri) - # HTTP Cache settings cache_timeout 3600 cache_segment_instance lambda {|sub| segment_instance(sub)} @@ -230,7 +228,7 @@ def self.embed_values_hash cache_load ontology: [:acronym] # Access control - read_restriction_based_on lambda {|sub| sub.ontology} + read_restriction_based_on lambda { |sub| sub.ontology } access_control_load ontology: [:administeredBy, :acl, :viewingRestriction] def initialize(*args) @@ -676,7 +674,7 @@ def delete(*args) FileUtils.remove_dir(self.data_folder) if Dir.exist?(self.data_folder) end - def roots(extra_include=nil, page=nil, pagesize=nil) + def roots(extra_include = [], page = nil, pagesize = nil, concept_schemes: [], concept_collections: []) self.bring(:ontology) unless self.loaded_attributes.include?(:ontology) self.bring(:hasOntologyLanguage) unless self.loaded_attributes.include?(:hasOntologyLanguage) paged = false @@ -688,46 +686,12 @@ def roots(extra_include=nil, page=nil, pagesize=nil) paged = true end - skos = self.hasOntologyLanguage&.skos? + skos = self.skos? classes = [] if skos - root_skos = < [self.id] }).each_solution do |s| - class_ids << s[:root] - end - - class_ids.each do |id| - classes << LinkedData::Models::Class.find(id).in(self).disable_rules.first - end - - classes = Goo::Base::Page.new(page, pagesize, count, classes) if paged + classes = skos_roots(concept_schemes, page, paged, pagesize) + extra_include += LinkedData::Models::Class.concept_is_in_attributes else self.ontology.bring(:flat) data_query = nil @@ -760,7 +724,7 @@ def roots(extra_include=nil, page=nil, pagesize=nil) where = LinkedData::Models::Class.in(self).models(classes).include(:prefLabel, :definition, :synonym, :obsolete) if extra_include - [:prefLabel, :definition, :synonym, :obsolete, :childrenCount].each do |x| + %i[prefLabel definition synonym obsolete childrenCount].each do |x| extra_include.delete x end end @@ -780,25 +744,77 @@ def roots(extra_include=nil, page=nil, pagesize=nil) load_children = [:children] end - if extra_include.length > 0 - where.include(extra_include) - end + where.include(extra_include) if extra_include.length > 0 end where.all - if load_children.length > 0 - LinkedData::Models::Class.partially_load_children(classes, 99, self) - end + LinkedData::Models::Class.partially_load_children(classes, 99, self) if load_children.length > 0 classes.delete_if { |c| obs = !c.obsolete.nil? && c.obsolete == true - c.load_has_children if extra_include&.include?(:hasChildren) && !obs + if !obs + c.load_computed_attributes(to_load: extra_include, + options: { schemes: current_schemes(concept_schemes), collections: concept_collections }) + end obs } - classes end + def children(cls, includes_param: [], concept_schemes: [], concept_collections: [], page: 1, size: 50) + ld = LinkedData::Models::Class.goo_attrs_to_load(includes_param) + unmapped = ld.delete(:properties) + + ld += LinkedData::Models::Class.concept_is_in_attributes if skos? + + page_data_query = LinkedData::Models::Class.where(parents: cls).in(self).include(ld) + aggregates = LinkedData::Models::Class.goo_aggregates_to_load(ld) + page_data_query.aggregate(*aggregates) unless aggregates.empty? + page_data = page_data_query.page(page, size).all + LinkedData::Models::Class.in(self).models(page_data).include(:unmapped).all if unmapped + + page_data.delete_if { |x| x.id.to_s == cls.id.to_s } + if ld.include?(:hasChildren) || ld.include?(:isInActiveScheme) || ld.include?(:isInActiveCollection) + page_data.each do |c| + c.load_computed_attributes(to_load: ld, + options: { schemes: concept_schemes, collections: concept_collections }) + end + end + + unless concept_schemes.empty? + page_data.delete_if { |c| Array(c.isInActiveScheme).empty? && !c.load_has_children } + if (page_data.size < size) && page_data.next_page + page_data += children(cls, includes_param: includes_param, concept_schemes: concept_schemes, + concept_collections: concept_collections, + page: page_data.next_page, size: size) + end + end + + page_data + end + + def skos? + self.bring :hasOntologyLanguage if bring? :hasOntologyLanguage + self.hasOntologyLanguage&.skos? + end + + def ontology_uri + self.bring(:uri) if self.bring? :uri + RDF::URI.new(self.uri) + end + + + + + + + + + + + + + def roots_sorted(extra_include=nil) classes = roots(extra_include) LinkedData::Models::Class.sort_classes(classes) From 018422499ca0d58069ebf95e1a702edba6af94c5 Mon Sep 17 00:00:00 2001 From: Syphax Date: Thu, 19 Jun 2025 10:08:02 +0200 Subject: [PATCH 06/19] add for the ontology endpoint links to its related schemes, collections and skos xl labels --- lib/ontologies_linked_data.rb | 2 +- lib/ontologies_linked_data/config/config.rb | 8 +++++--- lib/ontologies_linked_data/models/ontology.rb | 6 ++++++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/lib/ontologies_linked_data.rb b/lib/ontologies_linked_data.rb index 19ab8637a..ff221e08a 100644 --- a/lib/ontologies_linked_data.rb +++ b/lib/ontologies_linked_data.rb @@ -52,7 +52,7 @@ end # We need to require deterministic - that is why we have the sort. -models = Dir.glob("#{project_root}/ontologies_linked_data/models/concerns//**/*.rb").sort +models = Dir.glob("#{project_root}/ontologies_linked_data/models/concerns/**/*.rb").sort models.each do |m| require m end diff --git a/lib/ontologies_linked_data/config/config.rb b/lib/ontologies_linked_data/config/config.rb index 482cb7995..47686214a 100644 --- a/lib/ontologies_linked_data/config/config.rb +++ b/lib/ontologies_linked_data/config/config.rb @@ -8,6 +8,8 @@ module LinkedData @settings = OpenStruct.new @settings_run = false + DEFAULT_PREFIX = 'http://data.bioontology.org/'.freeze + def config(&block) return if @settings_run @settings_run = true @@ -24,7 +26,7 @@ def config(&block) @settings.search_server_url ||= 'http://localhost:8983/solr/term_search_core1' @settings.property_search_server_url ||= 'http://localhost:8983/solr/prop_search_core1' @settings.repository_folder ||= './test/data/ontology_files/repo' - @settings.rest_url_prefix ||= 'http://data.bioontology.org/' + @settings.rest_url_prefix ||= DEFAULT_PREFIX @settings.enable_security ||= false @settings.enable_slices ||= false @@ -34,7 +36,7 @@ def config(&block) @settings.ui_name ||= 'Bioportal' @settings.ui_host ||= 'bioportal.bioontology.org' @settings.replace_url_prefix ||= false - @settings.id_url_prefix ||= "http://data.bioontology.org/" + @settings.id_url_prefix ||= DEFAULT_PREFIX @settings.queries_debug ||= false @settings.enable_monitoring ||= false @@ -188,7 +190,7 @@ def goo_namespaces conf.add_namespace(:uneskos, RDF::Vocabulary.new("http://purl.org/umu/uneskos#")) - conf.id_prefix = 'http://data.bioontology.org/' + conf.id_prefix = DEFAULT_PREFIX conf.pluralize_models(true) end end diff --git a/lib/ontologies_linked_data/models/ontology.rb b/lib/ontologies_linked_data/models/ontology.rb index 03cf6b5ca..17622262a 100644 --- a/lib/ontologies_linked_data/models/ontology.rb +++ b/lib/ontologies_linked_data/models/ontology.rb @@ -6,6 +6,9 @@ require 'ontologies_linked_data/models/metric' require 'ontologies_linked_data/models/category' require 'ontologies_linked_data/models/project' +require 'ontologies_linked_data/models/skos/scheme' +require 'ontologies_linked_data/models/skos/collection' +require 'ontologies_linked_data/models/skos/skosxl' require 'ontologies_linked_data/models/notes/note' require 'ontologies_linked_data/purl/purl_client' @@ -59,6 +62,9 @@ class OntologyAnalyticsError < StandardError; end LinkedData::Hypermedia::Link.new("classes", lambda {|s| "ontologies/#{s.acronym}/classes"}, LinkedData::Models::Class.uri_type), LinkedData::Hypermedia::Link.new("single_class", lambda {|s| "ontologies/#{s.acronym}/classes/{class_id}"}, LinkedData::Models::Class.uri_type), LinkedData::Hypermedia::Link.new("roots", lambda {|s| "ontologies/#{s.acronym}/classes/roots"}, LinkedData::Models::Class.uri_type), + LinkedData::Hypermedia::Link.new("schemes", lambda {|s| "ontologies/#{s.acronym}/schemes"}, LinkedData::Models::SKOS::Scheme.uri_type), + LinkedData::Hypermedia::Link.new("collections", lambda {|s| "ontologies/#{s.acronym}/collections"}, LinkedData::Models::SKOS::Collection.uri_type), + LinkedData::Hypermedia::Link.new("xl_labels", lambda {|s| "ontologies/#{s.acronym}/skos_xl_labels"}, LinkedData::Models::SKOS::Label.uri_type), LinkedData::Hypermedia::Link.new("instances", lambda {|s| "ontologies/#{s.acronym}/instances"}, Goo.vocabulary["Instance"]), LinkedData::Hypermedia::Link.new("metrics", lambda {|s| "ontologies/#{s.acronym}/metrics"}, LinkedData::Models::Metric.type_uri), LinkedData::Hypermedia::Link.new("reviews", lambda {|s| "ontologies/#{s.acronym}/reviews"}, LinkedData::Models::Review.uri_type), From c39d8331874a5a2b16de7ddd4ae25e5bea2fe315 Mon Sep 17 00:00:00 2001 From: Syphax Date: Thu, 19 Jun 2025 10:08:33 +0200 Subject: [PATCH 07/19] add tests to check the submission roots filtring by scheme or collection --- test/data/ontology_files/efo_gwas.skos.owl | 21 +++- test/models/test_skos_submission.rb | 135 +++++++++++++++++++++ 2 files changed, 153 insertions(+), 3 deletions(-) create mode 100644 test/models/test_skos_submission.rb diff --git a/test/data/ontology_files/efo_gwas.skos.owl b/test/data/ontology_files/efo_gwas.skos.owl index f45a5a06a..dbcba0215 100644 --- a/test/data/ontology_files/efo_gwas.skos.owl +++ b/test/data/ontology_files/efo_gwas.skos.owl @@ -634,6 +634,7 @@ + @@ -710,6 +711,7 @@ + @@ -4258,14 +4260,27 @@ - - - + + + + + + + + + + + + + + + + diff --git a/test/models/test_skos_submission.rb b/test/models/test_skos_submission.rb new file mode 100644 index 000000000..35bfeee29 --- /dev/null +++ b/test/models/test_skos_submission.rb @@ -0,0 +1,135 @@ +require_relative './test_ontology_common' +require 'logger' +require 'rack' + +class TestOntologySubmission < LinkedData::TestOntologyCommon + + def before_suite + submission_parse('SKOS-TEST', + 'SKOS TEST Bla', + './test/data/ontology_files/efo_gwas.skos.owl', 987, + process_rdf: true, index_search: false, + run_metrics: false, reasoning: true) + + sub = LinkedData::Models::OntologySubmission.where(ontology: [acronym: 'SKOS-TEST'], + submissionId: 987) + .first + sub.bring_remaining + sub.uri = RDF::URI.new('http://www.ebi.ac.uk/efo/skos/EFO_GWAS_view') + sub.save + sub + end + + def test_get_main_concept_scheme + sub = before_suite + assert_equal sub.uri, sub.get_main_concept_scheme.to_s + end + + def test_roots_no_main_scheme + + sub = before_suite + sub.uri = nil # no concept scheme as owl:ontology found + sub.save + assert_nil sub.get_main_concept_scheme + # if no main scheme found get all roots (topConcepts) + assert sub.roots.map { |x| x.id.to_s }.sort == ['http://www.ebi.ac.uk/efo/EFO_0000311', + 'http://www.ebi.ac.uk/efo/EFO_0001444', + 'http://www.ifomis.org/bfo/1.1/snap#Disposition', + 'http://www.ebi.ac.uk/chebi/searchId.do?chebiId=CHEBI:37577', + 'http://www.ebi.ac.uk/efo/EFO_0000635', + 'http://www.ebi.ac.uk/efo/EFO_0000324'].sort + roots = sub.roots + LinkedData::Models::Class.in(sub).models(roots).include(:children).all + roots.each do |root| + q_broader = <<-eos +SELECT ?children WHERE { + ?children #{RDF::SKOS[:broader].to_ntriples} #{root.id.to_ntriples} } + eos + children_query = [] + Goo.sparql_query_client.query(q_broader).each_solution do |sol| + children_query << sol[:children].to_s + end + assert root.children.map { |x| x.id.to_s }.sort == children_query.sort + end + end + + def test_roots_main_scheme + sub = before_suite + + roots = sub.roots + assert_equal 4, roots.size + roots.each do |r| + assert_equal r.isInActiveScheme, [sub.get_main_concept_scheme.to_s] + assert_equal r.isInActiveCollection, [] + end + roots = roots.map { |r| r.id.to_s } unless roots.nil? + refute_includes roots, 'http://www.ebi.ac.uk/efo/EFO_0000311' + refute_includes roots, 'http://www.ebi.ac.uk/efo/EFO_0000324' + end + + def test_roots_of_a_scheme + sub = before_suite + concept_schemes = ['http://www.ebi.ac.uk/efo/skos/EFO_GWAS_view_2'] + roots = sub.roots(concept_schemes: concept_schemes) + assert_equal 2, roots.size + roots.each do |r| + assert_includes r.inScheme, concept_schemes.first + assert_equal r.isInActiveScheme, concept_schemes + assert_equal r.isInActiveCollection, [] + end + roots = roots.map { |r| r.id.to_s } unless roots.nil? + assert_includes roots, 'http://www.ebi.ac.uk/efo/EFO_0000311' + assert_includes roots, 'http://www.ebi.ac.uk/efo/EFO_0000324' + end + + def test_roots_of_multiple_scheme + sub = before_suite + + concept_schemes = ['http://www.ebi.ac.uk/efo/skos/EFO_GWAS_view_2', + 'http://www.ebi.ac.uk/efo/skos/EFO_GWAS_view'] + roots = sub.roots(concept_schemes: concept_schemes) + assert_equal 6, roots.size + roots.each do |r| + selected_schemes = r.inScheme.select { |s| concept_schemes.include?(s) } + refute_empty selected_schemes + assert_equal r.isInActiveScheme, selected_schemes + assert_equal r.isInActiveCollection, [] + end + roots = roots.map { |r| r.id.to_s } unless roots.nil? + assert roots.sort == ['http://www.ebi.ac.uk/efo/EFO_0000311', + 'http://www.ebi.ac.uk/efo/EFO_0001444', + 'http://www.ifomis.org/bfo/1.1/snap#Disposition', + 'http://www.ebi.ac.uk/chebi/searchId.do?chebiId=CHEBI:37577', + 'http://www.ebi.ac.uk/efo/EFO_0000635', + 'http://www.ebi.ac.uk/efo/EFO_0000324'].sort + end + + def test_roots_of_scheme_collection + sub = before_suite + + concept_schemes = ['http://www.ebi.ac.uk/efo/skos/EFO_GWAS_view'] + concept_collection = ['http://www.ebi.ac.uk/efo/skos/collection_1'] + roots = sub.roots(concept_schemes: concept_schemes, concept_collections: concept_collection) + assert_equal 4, roots.size + + roots.each do |r| + assert_equal r.isInActiveCollection, concept_collection if r.memberOf.include?(concept_collection.first) + end + end + + def test_roots_of_scheme_collections + sub = before_suite + + concept_schemes = ['http://www.ebi.ac.uk/efo/skos/EFO_GWAS_view'] + concept_collection = ['http://www.ebi.ac.uk/efo/skos/collection_1', + 'http://www.ebi.ac.uk/efo/skos/collection_2'] + roots = sub.roots(concept_schemes: concept_schemes, concept_collections: concept_collection) + assert_equal 4, roots.size + + roots.each do |r| + selected_collections = r.memberOf.select { |c| concept_collection.include?(c)} + assert_equal r.isInActiveCollection, selected_collections unless selected_collections.empty? + end + end +end + From 6bb341248edc310772d954281ad3b570e8a60f14 Mon Sep 17 00:00:00 2001 From: Syphax Date: Thu, 19 Jun 2025 10:11:19 +0200 Subject: [PATCH 08/19] add internal skos mappings extraction --- .../mappings/mappings.rb | 322 ++++++++++-------- 1 file changed, 180 insertions(+), 142 deletions(-) diff --git a/lib/ontologies_linked_data/mappings/mappings.rb b/lib/ontologies_linked_data/mappings/mappings.rb index 10abc75e1..8ca9219a8 100644 --- a/lib/ontologies_linked_data/mappings/mappings.rb +++ b/lib/ontologies_linked_data/mappings/mappings.rb @@ -5,21 +5,32 @@ module LinkedData module Mappings OUTSTANDING_LIMIT = 30 - def self.mapping_predicates() - predicates = {} - predicates["CUI"] = ["http://bioportal.bioontology.org/ontologies/umls/cui"] - predicates["SAME_URI"] = - ["http://data.bioontology.org/metadata/def/mappingSameURI"] - predicates["LOOM"] = - ["http://data.bioontology.org/metadata/def/mappingLoom"] - predicates["REST"] = - ["http://data.bioontology.org/metadata/def/mappingRest"] - return predicates - end + def self.mapping_predicates + predicates = {} + predicates["CUI"] = ["http://bioportal.bioontology.org/ontologies/umls/cui"] + predicates["SAME_URI"] = + ["http://data.bioontology.org/metadata/def/mappingSameURI"] + predicates["LOOM"] = + ["http://data.bioontology.org/metadata/def/mappingLoom"] + predicates["REST"] = + ["http://data.bioontology.org/metadata/def/mappingRest"] + return predicates + end - def self.handle_triple_store_downtime(logger=nil) - epr = Goo.sparql_query_client(:main) - status = epr.status + def self.internal_mapping_predicates + predicates = {} + predicates["SKOS:EXACT_MATCH"] = ["http://www.w3.org/2004/02/skos/core#exactMatch"] + predicates["SKOS:CLOSE_MATCH"] = ["http://www.w3.org/2004/02/skos/core#closeMatch"] + predicates["SKOS:BROAD_MATH"] = ["http://www.w3.org/2004/02/skos/core#broadMatch"] + predicates["SKOS:NARROW_MATH"] = ["http://www.w3.org/2004/02/skos/core#narrowMatch"] + predicates["SKOS:RELATED_MATH"] = ["http://www.w3.org/2004/02/skos/core#relatedMatch"] + + return predicates + end + + def self.handle_triple_store_downtime(logger = nil) + epr = Goo.sparql_query_client(:main) + status = epr.status if status[:exception] logger.info(status[:exception]) if logger @@ -35,7 +46,7 @@ def self.handle_triple_store_downtime(logger=nil) def self.mapping_counts(enable_debug=false, logger=nil, reload_cache=false, arr_acronyms=[]) logger = nil unless enable_debug t = Time.now - latest = self.retrieve_latest_submissions(options={acronyms:arr_acronyms}) + latest = self.retrieve_latest_submissions(options={ acronyms:arr_acronyms }) counts = {} i = 0 epr = Goo.sparql_query_client(:main) @@ -145,142 +156,59 @@ def self.empty_page(page,size) return p end - def self.mappings_ontologies(sub1,sub2,page,size,classId=nil,reload_cache=false) - union_template = <<-eos -{ - GRAPH <#{sub1.id.to_s}> { - classId ?o . - } - GRAPH graph { - ?s2 ?o . - } - bind -} -eos - blocks = [] - mappings = [] - persistent_count = 0 - acr1 = sub1.id.to_s.split("/")[-3] - - if classId.nil? - acr2 = nil - acr2 = sub2.id.to_s.split("/")[-3] unless sub2.nil? - pcount = LinkedData::Models::MappingCount.where(ontologies: acr1) - pcount = pcount.and(ontologies: acr2) unless acr2.nil? - f = Goo::Filter.new(:pair_count) == (not acr2.nil?) - pcount = pcount.filter(f) - pcount = pcount.include(:count) - pcount_arr = pcount.all - persistent_count = pcount_arr.length == 0 ? 0 : pcount_arr.first.count + def self.mappings_ontologies(sub1, sub2, page, size, classId = nil, reload_cache = false) + sub1, acr1 = extract_acronym(sub1) + sub2, acr2 = extract_acronym(sub2) - return LinkedData::Mappings.empty_page(page,size) if persistent_count == 0 - end + mappings = [] + persistent_count = 0 - if classId.nil? - union_template = union_template.gsub("classId", "?s1") - else - union_template = union_template.gsub("classId", "<#{classId.to_s}>") - end - # latest_sub_ids = self.retrieve_latest_submission_ids - - mapping_predicates().each do |_source,mapping_predicate| - union_block = union_template.gsub("predicate", mapping_predicate[0]) - union_block = union_block.gsub("bind","BIND ('#{_source}' AS ?source)") - - if sub2.nil? - union_block = union_block.gsub("graph","?g") - else - union_block = union_block.gsub("graph","<#{sub2.id.to_s}>") + if classId.nil? + persistent_count = count_mappings(acr1, acr2) + return LinkedData::Mappings.empty_page(page, size) if persistent_count == 0 end - blocks << union_block - end - unions = blocks.join("\nUNION\n") - - mappings_in_ontology = <<-eos -SELECT DISTINCT query_variables -WHERE { -unions -filter -} page_group -eos - query = mappings_in_ontology.gsub("unions", unions) - variables = "?s2 graph ?source ?o" - variables = "?s1 " + variables if classId.nil? - query = query.gsub("query_variables", variables) - filter = classId.nil? ? "FILTER ((?s1 != ?s2) || (?source = 'SAME_URI'))" : '' - if sub2.nil? - query = query.gsub("graph","?g") - ont_id = sub1.id.to_s.split("/")[0..-3].join("/") + query = mappings_ont_build_query(classId, page, size, sub1, sub2) + epr = Goo.sparql_query_client(:main) + graphs = [sub1] + unless sub2.nil? + graphs << sub2 + end + solutions = epr.query(query, graphs: graphs, reload_cache: reload_cache) + s1 = nil + s1 = RDF::URI.new(classId.to_s) unless classId.nil? + + solutions.each do |sol| + graph2 = sub2.nil? ? sol[:g] : sub2 + s1 = sol[:s1] if classId.nil? + backup_mapping = nil + + if sol[:source].to_s == "REST" + backup_mapping = LinkedData::Models::RestBackupMapping + .find(sol[:o]).include(:process, :class_urns).first + backup_mapping.process.bring_remaining + end - # latest_sub_filter_arr = latest_sub_ids.map { |_, id| "?g = <#{id}>" } - # filter += "\nFILTER (#{latest_sub_filter_arr.join(' || ')}) " + classes = get_mapping_classes_instance(s1, sub1, sol[:s2], graph2) - #STRSTARTS is used to not count older graphs - #no need since now we delete older graphs - filter += "\nFILTER (!STRSTARTS(str(?g),'#{ont_id}'))" - else - query = query.gsub("graph", "") - end - query = query.gsub("filter", filter) + mapping = if backup_mapping.nil? + LinkedData::Models::Mapping.new(classes, sol[:source].to_s) + else + LinkedData::Models::Mapping.new( + classes, sol[:source].to_s, + backup_mapping.process, backup_mapping.id) + end - if size > 0 - pagination = "OFFSET offset LIMIT limit" - query = query.gsub("page_group",pagination) - limit = size - offset = (page-1) * size - query = query.gsub("limit", "#{limit}").gsub("offset", "#{offset}") - else - query = query.gsub("page_group","") - end - epr = Goo.sparql_query_client(:main) - graphs = [sub1.id] - unless sub2.nil? - graphs << sub2.id - end - solutions = epr.query(query, graphs: graphs, reload_cache: reload_cache) - s1 = nil - unless classId.nil? - s1 = RDF::URI.new(classId.to_s) - end - solutions.each do |sol| - graph2 = nil - if sub2.nil? - graph2 = sol[:g] - else - graph2 = sub2.id - end - if classId.nil? - s1 = sol[:s1] + mappings << mapping end - classes = [ read_only_class(s1.to_s,sub1.id.to_s), - read_only_class(sol[:s2].to_s,graph2.to_s) ] - - backup_mapping = nil - mapping = nil - if sol[:source].to_s == "REST" - backup_mapping = LinkedData::Models::RestBackupMapping - .find(sol[:o]).include(:process).first - backup_mapping.process.bring_remaining - end - if backup_mapping.nil? - mapping = LinkedData::Models::Mapping.new( - classes,sol[:source].to_s) - else - mapping = LinkedData::Models::Mapping.new( - classes,sol[:source].to_s, - backup_mapping.process,backup_mapping.id) + + if size == 0 + return mappings end - mappings << mapping - end - if size == 0 - return mappings + page = Goo::Base::Page.new(page, size, persistent_count, mappings) + return page end - page = Goo::Base::Page.new(page,size,nil,mappings) - page.aggregate = persistent_count - return page - end def self.mappings_ontology(sub,page,size,classId=nil,reload_cache=false) return self.mappings_ontologies(sub,nil,page,size,classId=classId, @@ -437,7 +365,7 @@ def self.create_rest_mapping(classes,process) graph_insert << [c.id, RDF::URI.new(rest_predicate), backup_mapping.id] Goo.sparql_update_client.insert_data(graph_insert, graph: sub.id) end - mapping = LinkedData::Models::Mapping.new(classes,"REST",process, backup_mapping.id) + mapping = LinkedData::Models::Mapping.new(classes,"REST", process, backup_mapping.id) return mapping end @@ -773,5 +701,115 @@ def self.create_mapping_count_pairs_for_ontologies(logger, arr_acronyms) # fsave.close end + private + + def self.get_mapping_classes_instance(s1, graph1, s2, graph2) + [read_only_class(s1.to_s, graph1.to_s), + read_only_class(s2.to_s, graph2.to_s)] + end + + def self.mappings_ont_build_query(class_id, page, size, sub1, sub2) + blocks = [] + mapping_predicates.each do |_source, mapping_predicate| + blocks << mappings_union_template(class_id, sub1, sub2, + mapping_predicate[0], + "BIND ('#{_source}' AS ?source)") + end + + + + + + + filter = class_id.nil? ? "FILTER ((?s1 != ?s2) || (?source = 'SAME_URI'))" : '' + if sub2.nil? + + class_id_subject = class_id.nil? ? '?s1' : "<#{class_id.to_s}>" + source_graph = sub1.nil? ? '?g' : "<#{sub1.to_s}>" + internal_mapping_predicates.each do |_source, predicate| + blocks << <<-eos + { + GRAPH #{source_graph} { + #{class_id_subject} <#{predicate[0]}> ?s2 . + } + BIND( AS ?g) + BIND(?s2 AS ?o) + BIND ('#{_source}' AS ?source) + } + eos + end + + ont_id = sub1.to_s.split("/")[0..-3].join("/") + #STRSTARTS is used to not count older graphs + #no need since now we delete older graphs + + filter += "\nFILTER (!STRSTARTS(str(?g),'#{ont_id}')" + filter += " || " + internal_mapping_predicates.keys.map{|x| "(?source = '#{x}')"}.join('||') + filter += ")" + end + + variables = "?s2 #{sub2.nil? ? '?g' : ''} ?source ?o" + variables = "?s1 " + variables if class_id.nil? + + pagination = '' + if size > 0 + limit = size + offset = (page - 1) * size + pagination = "OFFSET #{offset} LIMIT #{limit}" + end + + query = <<-eos +SELECT DISTINCT #{variables} +WHERE { + #{blocks.join("\nUNION\n")} + #{filter} +} #{pagination} + eos + + query + end + + def self.mappings_union_template(class_id, sub1, sub2, predicate, bind) + class_id_subject = class_id.nil? ? '?s1' : "<#{class_id.to_s}>" + target_graph = sub2.nil? ? '?g' : "<#{sub2.to_s}>" + union_template = <<-eos +{ + GRAPH <#{sub1.to_s}> { + #{class_id_subject} <#{predicate}> ?o . + } + GRAPH #{target_graph} { + ?s2 <#{predicate}> ?o . + } + #{bind} +} + eos + end + + def self.count_mappings(acr1, acr2) + count = LinkedData::Models::MappingCount.where(ontologies: acr1) + count = count.and(ontologies: acr2) unless acr2.nil? + f = Goo::Filter.new(:pair_count) == (not acr2.nil?) + count = count.filter(f) + count = count.include(:count) + pcount_arr = count.all + pcount_arr.length == 0 ? 0 : pcount_arr.first.count + end + + def self.extract_acronym(submission) + sub = submission + if submission.nil? + acr = nil + elsif submission.respond_to?(:id) + # Case where sub2 is a Submission + sub = submission.id + acr = sub.to_s.split("/")[-3] + else + acr = sub.to_s + end + + return sub, acr + end + + end end -end + From 891cbac4950955fcb88ba1675521173948c73c03 Mon Sep 17 00:00:00 2001 From: Syphax Date: Thu, 19 Jun 2025 11:50:44 +0200 Subject: [PATCH 09/19] fix tests for allegrograph --- dip.yml | 2 +- test/models/skos/test_schemes.rb | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/dip.yml b/dip.yml index 70dc642b2..e82bad7be 100644 --- a/dip.yml +++ b/dip.yml @@ -40,7 +40,7 @@ interaction: test-ag: description: Run minitest unit tests - service: ruby-ag + service: ruby-agraph command: bundle exec rake test diff --git a/test/models/skos/test_schemes.rb b/test/models/skos/test_schemes.rb index 41effac6c..ca1bf686c 100644 --- a/test/models/skos/test_schemes.rb +++ b/test/models/skos/test_schemes.rb @@ -8,8 +8,8 @@ def self.before_suite self.new('').submission_parse('INRAETHES', 'Testing skos', 'test/data/ontology_files/thesaurusINRAE_nouv_structure.skos', 1, - process_rdf: true, index_search: false, - run_metrics: false, reasoning: false) + process_rdf: true, extract_metadata: false, + generate_missing_labels: false) end def test_schemes_all @@ -19,6 +19,8 @@ def test_schemes_all assert_equal 66, schemes.size schemes_test = test_data + schemes_test = schemes_test.sort_by { |x| x[:id] } + schemes = schemes.sort_by { |x| x.id.to_s } schemes.each_with_index do |x, i| scheme_test = schemes_test[i] From ecff71831e27e39e52890b3a883eb37d94ee182c Mon Sep 17 00:00:00 2001 From: mdorf Date: Mon, 30 Jun 2025 13:06:43 -0700 Subject: [PATCH 10/19] unit tests fixed --- bin/owlapi-wrapper-1.4.3.jar | Bin lib/ontologies_linked_data/models/instance.rb | 14 +++++++------- 2 files changed, 7 insertions(+), 7 deletions(-) mode change 100644 => 100755 bin/owlapi-wrapper-1.4.3.jar diff --git a/bin/owlapi-wrapper-1.4.3.jar b/bin/owlapi-wrapper-1.4.3.jar old mode 100644 new mode 100755 diff --git a/lib/ontologies_linked_data/models/instance.rb b/lib/ontologies_linked_data/models/instance.rb index ad8a2cd20..25f26eb97 100644 --- a/lib/ontologies_linked_data/models/instance.rb +++ b/lib/ontologies_linked_data/models/instance.rb @@ -4,7 +4,7 @@ module Models class Instance < LinkedData::Models::Base model :named_individual, name_with: :id, collection: :submission, - namespace: :owl, schemaless: :true , rdf_type: lambda { |*x| RDF::OWL[:NamedIndividual]} + namespace: :owl, schemaless: :true , rdf_type: lambda { |*x| RDF::OWL[:NamedIndividual]} attribute :label, namespace: :rdfs, enforce: [:list] attribute :prefLabel, namespace: :skos, enforce: [:existence], alias: true @@ -54,16 +54,16 @@ def self.get_instances_by_ontology(submission_id, page_no: nil, size: nil) end def self.instances_by_class_where_query(submission, class_id: nil, page_no: nil, size: nil) - where_condition = class_id.nil? ? nil : {types: RDF::URI.new(class_id.to_s)} - query = LinkedData::Models::Instance.where(where_condition).in(submission).include(:types, :label, :prefLabel) - query.page(page_no, size) unless page_no.nil? - query + where_condition = class_id.nil? ? nil : {types: RDF::URI.new(class_id.to_s)} + query = LinkedData::Models::Instance.where(where_condition).in(submission).include(:types, :label, :prefLabel) + query.page(page_no, size) unless page_no.nil? + query end def self.load_unmapped(submission, models) - LinkedData::Models::Instance.where.in(submission).models(models).include(:unmapped).all + LinkedData::Models::Instance.where.in(submission).models(models).include(:unmapped).all end end -end +end \ No newline at end of file From ae10867cc061c7c210e0ccbd34a10e40f3b248ac Mon Sep 17 00:00:00 2001 From: Jennifer Vendetti Date: Mon, 14 Jul 2025 11:37:27 -0700 Subject: [PATCH 11/19] Correct ui.name capitalization --- lib/ontologies_linked_data/config/config.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ontologies_linked_data/config/config.rb b/lib/ontologies_linked_data/config/config.rb index 00e5ee118..54040a7f5 100644 --- a/lib/ontologies_linked_data/config/config.rb +++ b/lib/ontologies_linked_data/config/config.rb @@ -31,7 +31,7 @@ def config(&block) # Java/JVM options @settings.java_max_heap_size ||= '10240M' - @settings.ui_name ||= 'Bioportal' + @settings.ui_name ||= 'BioPortal' @settings.ui_host ||= 'bioportal.bioontology.org' @settings.replace_url_prefix ||= false @settings.id_url_prefix ||= "http://data.bioontology.org/" From 0343de38932cbe2353b88a2e41d74769a2488588 Mon Sep 17 00:00:00 2001 From: Jennifer Vendetti Date: Mon, 14 Jul 2025 16:24:10 -0700 Subject: [PATCH 12/19] Add helper method to render email templates Also updates obofoundry_sync to use the new helper method --- .../utils/notifications.rb | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/lib/ontologies_linked_data/utils/notifications.rb b/lib/ontologies_linked_data/utils/notifications.rb index fc28b21b5..b6b6ef86c 100644 --- a/lib/ontologies_linked_data/utils/notifications.rb +++ b/lib/ontologies_linked_data/utils/notifications.rb @@ -106,16 +106,29 @@ def self.reset_password(user, token) def self.obofoundry_sync(missing_onts, obsolete_onts) ui_name = LinkedData.settings.ui_name + subject = "[#{ui_name}] OBO Foundry synchronization report" + recipients = Notifier.ontoportal_admin_emails + body = render_template('obofoundry_sync.erb', { + ui_name: ui_name, + missing_onts: missing_onts, + obsolete_onts: obsolete_onts, + }) + + Notifier.notify_mails_grouped(subject, body, recipients) + end + + def self.render_template(template_name, locals = {}) + # template_path = File.join(File.dirname(__FILE__), '..', '..', 'views', template_name) gem_path = Gem.loaded_specs['ontologies_linked_data'].full_gem_path - template = File.read(File.join(gem_path, 'views/emails/obofoundry_sync.erb')) + template_path = File.join(gem_path, 'views', 'emails', template_name) + template = File.read(template_path) b = binding - b.local_variable_set(:ui_name, ui_name) - b.local_variable_set(:missing_onts, missing_onts) - b.local_variable_set(:obsolete_onts, obsolete_onts) - body = ERB.new(template).result(b) + locals.each { |k, v| b.local_variable_set(k, v) } - Notifier.notify_ontoportal_admins("[#{ui_name}] OBO Foundry synchronization report", body) + ERB.new(template).result(b) + rescue Errno::ENOENT => e + raise "Template not found: #{template_path}" end NEW_NOTE = < Date: Tue, 15 Jul 2025 15:27:54 -0700 Subject: [PATCH 13/19] Add a notification for Cloudflare Analytics --- .../utils/notifications.rb | 11 +++++++++++ views/emails/cloudflare_analytics.erb | 16 ++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 views/emails/cloudflare_analytics.erb diff --git a/lib/ontologies_linked_data/utils/notifications.rb b/lib/ontologies_linked_data/utils/notifications.rb index b6b6ef86c..cbd822fa0 100644 --- a/lib/ontologies_linked_data/utils/notifications.rb +++ b/lib/ontologies_linked_data/utils/notifications.rb @@ -117,6 +117,17 @@ def self.obofoundry_sync(missing_onts, obsolete_onts) Notifier.notify_mails_grouped(subject, body, recipients) end + def self.cloudflare_analytics(result_data) + ui_name = LinkedData.settings.ui_name + subject = "[#{ui_name}] Cloudflare Analytics daily collection result: #{result_data[:status]}" + recipients = Notifier.ontoportal_admin_emails + body = render_template('cloudflare_analytics.erb', { + result_data: result_data + }) + + Notifier.notify_mails_grouped(subject, body, recipients) + end + def self.render_template(template_name, locals = {}) # template_path = File.join(File.dirname(__FILE__), '..', '..', 'views', template_name) gem_path = Gem.loaded_specs['ontologies_linked_data'].full_gem_path diff --git a/views/emails/cloudflare_analytics.erb b/views/emails/cloudflare_analytics.erb new file mode 100644 index 000000000..3cccc4943 --- /dev/null +++ b/views/emails/cloudflare_analytics.erb @@ -0,0 +1,16 @@ +<% if result_data[:status] == 'success' %> + Cloudflare Analytics job completed successfully

+ + Start time: <%= result_data[:start_time].utc %>
+ End time: <%= result_data[:end_time].utc %>
+ Duration: <%= result_data[:duration] %> seconds

+<% else %> + Cloudflare Analytics job failed!

+ + Start time: <%= result_data[:start_time].utc %>
+ End time: <%= result_data[:end_time].utc %>
+ Duration: <%= result_data[:duration] %> seconds

+ + Error: + 
<%= result_data[:error] %>


+<% end %> \ No newline at end of file From d57b1cf89f3cb2c7d196daa080f033a7bf10fbbd Mon Sep 17 00:00:00 2001 From: Jennifer Vendetti Date: Tue, 15 Jul 2025 15:32:04 -0700 Subject: [PATCH 14/19] Make render_template private --- lib/ontologies_linked_data/utils/notifications.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/ontologies_linked_data/utils/notifications.rb b/lib/ontologies_linked_data/utils/notifications.rb index cbd822fa0..7a7630258 100644 --- a/lib/ontologies_linked_data/utils/notifications.rb +++ b/lib/ontologies_linked_data/utils/notifications.rb @@ -128,8 +128,9 @@ def self.cloudflare_analytics(result_data) Notifier.notify_mails_grouped(subject, body, recipients) end + private + def self.render_template(template_name, locals = {}) - # template_path = File.join(File.dirname(__FILE__), '..', '..', 'views', template_name) gem_path = Gem.loaded_specs['ontologies_linked_data'].full_gem_path template_path = File.join(gem_path, 'views', 'emails', template_name) template = File.read(template_path) From 2e0b6122d11f83123db952312e0c9b60c95a136b Mon Sep 17 00:00:00 2001 From: Jennifer Vendetti Date: Tue, 15 Jul 2025 15:46:27 -0700 Subject: [PATCH 15/19] Fix some RuboCop warnings --- test/util/test_notifications.rb | 67 ++++++++++++++++----------------- 1 file changed, 32 insertions(+), 35 deletions(-) diff --git a/test/util/test_notifications.rb b/test/util/test_notifications.rb index 9db820db7..8b3589e3c 100644 --- a/test/util/test_notifications.rb +++ b/test/util/test_notifications.rb @@ -1,8 +1,7 @@ -require_relative "../test_case" - -require "email_spec" -require "logger" +require_relative '../test_case' +require 'email_spec' +require 'logger' class TestNotifications < LinkedData::TestCase include EmailSpec::Helpers @@ -12,7 +11,7 @@ def self.before_suite @@disable_override = LinkedData.settings.email_disable_override @@old_support_mails = LinkedData.settings.ontoportal_admin_emails if @@old_support_mails.nil? || @@old_support_mails.empty? - LinkedData.settings.ontoportal_admin_emails = ["ontoportal-support@mail.com"] + LinkedData.settings.ontoportal_admin_emails = ['ontoportal-support@mail.com'] end LinkedData.settings.email_disable_override = true LinkedData.settings.enable_notifications = true @@ -21,7 +20,7 @@ def self.before_suite @@ont = LinkedData::SampleData::Ontology.create_ontologies_and_submissions(ont_count: 1, submission_count: 1)[2].first @@ont.bring_remaining @@user = @@ont.administeredBy.first - @@subscription = self.new("before_suite")._subscription(@@ont) + @@subscription = new('before_suite')._subscription(@@ont) @@user.bring_remaining @@user.subscription = [@@subscription] @@user.save @@ -44,14 +43,14 @@ def setup def _subscription(ont) subscription = LinkedData::Models::Users::Subscription.new subscription.ontology = ont - subscription.notification_type = LinkedData::Models::Users::NotificationType.find("ALL").first + subscription.notification_type = LinkedData::Models::Users::NotificationType.find('ALL').first subscription.save end def test_send_notification - recipients = ["test@example.org"] - subject = "Test subject" - body = "My test body" + recipients = ['test@example.org'] + subject = 'Test subject' + body = 'My test body' # Email recipient address will be overridden LinkedData.settings.email_disable_override = false @@ -60,11 +59,7 @@ def test_send_notification # Disable override LinkedData.settings.email_disable_override = true - LinkedData::Utils::Notifier.notify({ - recipients: recipients, - subject: subject, - body: body - }) + LinkedData::Utils::Notifier.notify({ recipients: recipients, subject: subject, body: body }) assert_equal recipients, last_email_sent.to assert_equal [LinkedData.settings.email_sender], last_email_sent.from assert_equal last_email_sent.body.raw_source, body @@ -72,9 +67,9 @@ def test_send_notification end def test_new_note_notification - recipients = ["test@example.org"] - subject = "Test note subject" - body = "Test note body" + recipients = ['test@example.org'] + subject = 'Test note subject' + body = 'Test note body' note = LinkedData::Models::Note.new note.creator = @@user note.subject = subject @@ -84,11 +79,11 @@ def test_new_note_notification assert_match "[#{@@ui_name} Notes]", last_email_sent.subject assert_equal [@@user.email], last_email_sent.to ensure - note.delete if note + note&.delete end def test_processing_complete_notification - options = { ont_count: 1, submission_count: 2, acronym: "NOTIFY" } + options = { ont_count: 1, submission_count: 2, acronym: 'NOTIFY' } ont = LinkedData::SampleData::Ontology.create_ontologies_and_submissions(options)[2].first subscription = _subscription(ont) @@user.subscription = @@user.subscription.dup << subscription @@ -101,27 +96,26 @@ def test_processing_complete_notification first_user = subscription.user.first first_user.bring :email - assert_match "Parsing Success", all_emails.first.subject + assert_match 'Parsing Success', all_emails.first.subject assert_equal [first_user.email], all_emails.first.to - assert_match ("Parsing Success"), all_emails.last.subject + assert_match 'Parsing Success', all_emails.last.subject assert_equal @@support_mails.uniq.sort, all_emails[1].to.sort assert_equal admin_mails.uniq.sort, all_emails.last.to.sort - reset_mailer - sub = ont.submissions.sort_by { |s| s.id}.first - sub.process_submission(Logger.new(TestLogFile.new), {archive: true}) + sub = ont.submissions.sort_by { |s| s.id }.first + sub.process_submission(Logger.new(TestLogFile.new), { archive: true }) assert_empty all_emails ensure - ont.delete if ont - subscription.delete if subscription + ont&.delete + subscription&.delete end def test_disable_administrative_notifications LinkedData.settings.enable_administrative_notifications = false - options = { ont_count: 1, submission_count: 1, acronym: "DONTNOTIFY" } + options = { ont_count: 1, submission_count: 1, acronym: 'DONTNOTIFY' } ont = LinkedData::SampleData::Ontology.create_ontologies_and_submissions(options)[2].first ont.latest_submission(status: :any).process_submission(Logger.new(TestLogFile.new)) admin_mails = LinkedData::Utils::Notifier.ontology_admin_emails(ont) @@ -129,22 +123,25 @@ def test_disable_administrative_notifications refute_match @@support_mails, last_email_sent.to.sort assert_equal admin_mails, last_email_sent.to.sort - assert_match ("Parsing Success"), all_emails.last.subject + assert_match 'Parsing Success', all_emails.last.subject LinkedData.settings.enable_administrative_notifications = true ensure - ont.delete if ont + ont&.delete end def test_remote_ontology_pull_notification - recipients = ["test@example.org"] - ont_count, acronyms, ontologies = LinkedData::SampleData::Ontology.create_ontologies_and_submissions(ont_count: 1, submission_count: 1, process_submission: false) + recipients = ['test@example.org'] + _ont_count, _acronyms, ontologies = LinkedData::SampleData::Ontology.create_ontologies_and_submissions( + ont_count: 1, submission_count: 1, process_submission: false + ) - ont = LinkedData::Models::Ontology.find(ontologies[0].id).include(:acronym, :administeredBy, :name, :submissions).first + ont = LinkedData::Models::Ontology.find(ontologies[0].id) + .include(:acronym, :administeredBy, :name, :submissions).first ont_admins = Array.new(3) { LinkedData::Models::User.new } ont_admins.each_with_index do |user, i| user.username = "Test User #{i}" user.email = "tester_#{i}@example.org" - user.password = "password" + user.password = 'password' user.save assert user.valid?, user.errors end @@ -164,7 +161,7 @@ def test_remote_ontology_pull_notification assert_equal admin_mails, last_email_sent.to.sort ensure ont_admins.each do |user| - user.delete if user + user&.delete end end From ac14d9cf0a139f8bf4e935a208d2680729dcec55 Mon Sep 17 00:00:00 2001 From: Jennifer Vendetti Date: Tue, 15 Jul 2025 16:00:06 -0700 Subject: [PATCH 16/19] Add unit tests for render_template method --- test/util/test_notifications.rb | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/util/test_notifications.rb b/test/util/test_notifications.rb index 8b3589e3c..9a58db444 100644 --- a/test/util/test_notifications.rb +++ b/test/util/test_notifications.rb @@ -2,6 +2,7 @@ require 'email_spec' require 'logger' +require 'mocha/minitest' class TestNotifications < LinkedData::TestCase include EmailSpec::Helpers @@ -165,6 +166,32 @@ def test_remote_ontology_pull_notification end end + def test_render_template + gem_path = "/fake/gem/path" + Gem.loaded_specs.stubs(:[]).with('ontologies_linked_data').returns( + stub(full_gem_path: gem_path) + ) + + template_content = "Hello <%= name %>!" + File.expects(:read).with("#{gem_path}/views/emails/test.erb").returns(template_content) + + result = LinkedData::Utils::Notifications.render_template('test.erb', { name: 'World' }) + assert_equal "Hello World!", result + end + + def test_render_template_file_not_found + gem_path = "/fake/gem/path" + Gem.loaded_specs.stubs(:[]).with('ontologies_linked_data').returns( + stub(full_gem_path: gem_path) + ) + + File.expects(:read).raises(Errno::ENOENT) + + assert_raises(RuntimeError, "Template not found") do + LinkedData::Utils::Notifications.render_template('nonexistent.erb', {}) + end + end + def test_mail_options current_auth_type = LinkedData.settings.smtp_auth_type From d95d004cd36499f5c24fdd69463fc18e6211c90b Mon Sep 17 00:00:00 2001 From: Jennifer Vendetti Date: Tue, 15 Jul 2025 16:29:43 -0700 Subject: [PATCH 17/19] Add unit tests for analytics notifications --- test/util/test_notifications.rb | 37 +++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/test/util/test_notifications.rb b/test/util/test_notifications.rb index 9a58db444..8ad04ca91 100644 --- a/test/util/test_notifications.rb +++ b/test/util/test_notifications.rb @@ -166,6 +166,43 @@ def test_remote_ontology_pull_notification end end + def test_cloudflare_analytics_success_notification + start_time = Time.now - 3600 + end_time = Time.now + result_data = { + start_time: start_time, + end_time: end_time, + duration: 3600, + status: 'success', + error: nil + } + + LinkedData::Utils::Notifications.cloudflare_analytics(result_data) + + assert_equal 1, all_emails.size + assert_match 'success', last_email_sent.subject + assert_includes last_email_sent.body.raw_source, 'completed successfully' + assert_includes last_email_sent.body.raw_source, start_time.to_s + end + + def test_cloudflare_analytics_failure_notification + start_time = Time.now - 3600 + end_time = Time.now + result_data = { + start_time: start_time, + end_time: end_time, + duration: 3600, + status: 'error', + error: 'Connection timeout' + } + + LinkedData::Utils::Notifications.cloudflare_analytics(result_data) + + assert_equal 1, all_emails.size + assert_match 'error', last_email_sent.subject + assert_includes last_email_sent.body.raw_source, 'Connection timeout' + end + def test_render_template gem_path = "/fake/gem/path" Gem.loaded_specs.stubs(:[]).with('ontologies_linked_data').returns( From ec203c7a7d4998ec2b03afa31ca8ff7a1f831b8d Mon Sep 17 00:00:00 2001 From: Jennifer Vendetti Date: Tue, 15 Jul 2025 16:32:50 -0700 Subject: [PATCH 18/19] Fix some RuboCop warnings --- test/util/test_notifications.rb | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test/util/test_notifications.rb b/test/util/test_notifications.rb index 8ad04ca91..7244fcaba 100644 --- a/test/util/test_notifications.rb +++ b/test/util/test_notifications.rb @@ -18,7 +18,8 @@ def self.before_suite LinkedData.settings.enable_notifications = true @@ui_name = LinkedData.settings.ui_name @@support_mails = LinkedData.settings.ontoportal_admin_emails - @@ont = LinkedData::SampleData::Ontology.create_ontologies_and_submissions(ont_count: 1, submission_count: 1)[2].first + @@ont = LinkedData::SampleData::Ontology.create_ontologies_and_submissions(ont_count: 1, + submission_count: 1)[2].first @@ont.bring_remaining @@user = @@ont.administeredBy.first @@subscription = new('before_suite')._subscription(@@ont) @@ -204,27 +205,27 @@ def test_cloudflare_analytics_failure_notification end def test_render_template - gem_path = "/fake/gem/path" + gem_path = '/fake/gem/path' Gem.loaded_specs.stubs(:[]).with('ontologies_linked_data').returns( stub(full_gem_path: gem_path) ) - template_content = "Hello <%= name %>!" + template_content = 'Hello <%= name %>!' File.expects(:read).with("#{gem_path}/views/emails/test.erb").returns(template_content) result = LinkedData::Utils::Notifications.render_template('test.erb', { name: 'World' }) - assert_equal "Hello World!", result + assert_equal 'Hello World!', result end def test_render_template_file_not_found - gem_path = "/fake/gem/path" + gem_path = '/fake/gem/path' Gem.loaded_specs.stubs(:[]).with('ontologies_linked_data').returns( stub(full_gem_path: gem_path) ) File.expects(:read).raises(Errno::ENOENT) - assert_raises(RuntimeError, "Template not found") do + assert_raises(RuntimeError, 'Template not found') do LinkedData::Utils::Notifications.render_template('nonexistent.erb', {}) end end From 3249a037ca2593c65a2cc6454ec936b38844cbde Mon Sep 17 00:00:00 2001 From: Jennifer Vendetti Date: Tue, 15 Jul 2025 16:54:26 -0700 Subject: [PATCH 19/19] Improve setup and teardown --- test/util/test_notifications.rb | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/test/util/test_notifications.rb b/test/util/test_notifications.rb index 7244fcaba..e590fe9cc 100644 --- a/test/util/test_notifications.rb +++ b/test/util/test_notifications.rb @@ -8,17 +8,20 @@ class TestNotifications < LinkedData::TestCase include EmailSpec::Helpers def self.before_suite - @@notifications_enabled = LinkedData.settings.enable_notifications - @@disable_override = LinkedData.settings.email_disable_override - @@old_support_mails = LinkedData.settings.ontoportal_admin_emails - if @@old_support_mails.nil? || @@old_support_mails.empty? - LinkedData.settings.ontoportal_admin_emails = ['ontoportal-support@mail.com'] - end + # Store original settings + @original_settings = { + notifications_enabled: LinkedData.settings.enable_notifications, + disable_override: LinkedData.settings.email_disable_override, + admin_emails: LinkedData.settings.ontoportal_admin_emails + } + LinkedData.settings.email_disable_override = true LinkedData.settings.enable_notifications = true + LinkedData.settings.ontoportal_admin_emails = ['ontoportal-support@mail.com'] + @@ui_name = LinkedData.settings.ui_name @@support_mails = LinkedData.settings.ontoportal_admin_emails - @@ont = LinkedData::SampleData::Ontology.create_ontologies_and_submissions(ont_count: 1, + @@ont = LinkedData::SampleData::Ontology.create_ontologies_and_submissions(ont_count: 1, submission_count: 1)[2].first @@ont.bring_remaining @@user = @@ont.administeredBy.first @@ -29,12 +32,12 @@ def self.before_suite end def self.after_suite - LinkedData.settings.enable_notifications = @@notifications_enabled - LinkedData.settings.email_disable_override = @@disable_override - LinkedData.settings.ontoportal_admin_emails = @@old_support_mails - @@ont.delete if defined?(@@ont) - @@subscription.delete if defined?(@@subscription) - @@user.delete if defined?(@@user) + # Restore original settings + LinkedData.settings.enable_notifications = @original_settings[:notifications_enabled] + LinkedData.settings.email_disable_override = @original_settings[:disable_override] + LinkedData.settings.ontoportal_admin_emails = @original_settings[:admin_emails] + + [@@ont, @@subscription, @@user].each(&:delete) end def setup