diff --git a/bin/owlapi-wrapper-1.5.0.jar b/bin/owlapi-wrapper-1.5.0.jar old mode 100644 new mode 100755 diff --git a/dip.yml b/dip.yml index 70dc642b2..e82bad7be 100644 --- a/dip.yml +++ b/dip.yml @@ -40,7 +40,7 @@ interaction: test-ag: description: Run minitest unit tests - service: ruby-ag + service: ruby-agraph command: bundle exec rake test diff --git a/lib/ontologies_linked_data.rb b/lib/ontologies_linked_data.rb index 19ab8637a..ff221e08a 100644 --- a/lib/ontologies_linked_data.rb +++ b/lib/ontologies_linked_data.rb @@ -52,7 +52,7 @@ end # We need to require deterministic - that is why we have the sort. -models = Dir.glob("#{project_root}/ontologies_linked_data/models/concerns//**/*.rb").sort +models = Dir.glob("#{project_root}/ontologies_linked_data/models/concerns/**/*.rb").sort models.each do |m| require m end diff --git a/lib/ontologies_linked_data/concerns/concepts/concept_in_collection.rb b/lib/ontologies_linked_data/concerns/concepts/concept_in_collection.rb new file mode 100644 index 000000000..1707dea35 --- /dev/null +++ b/lib/ontologies_linked_data/concerns/concepts/concept_in_collection.rb @@ -0,0 +1,25 @@ +module LinkedData + module Concerns + module Concept + module InCollection + def self.included(base) + base.serialize_methods :isInActiveCollection + end + + def isInActiveCollection + @isInActiveCollection + end + + def inCollection?(collection) + self.memberOf.include?(collection) + end + + def load_is_in_collection(collections = []) + included = collections.select { |s| inCollection?(s) } + @isInActiveCollection = included + end + + end + end + end +end diff --git a/lib/ontologies_linked_data/concerns/concepts/concept_in_scheme.rb b/lib/ontologies_linked_data/concerns/concepts/concept_in_scheme.rb new file mode 100644 index 000000000..ba3592d2c --- /dev/null +++ b/lib/ontologies_linked_data/concerns/concepts/concept_in_scheme.rb @@ -0,0 +1,26 @@ +module LinkedData + module Concerns + module Concept + module InScheme + def self.included(base) + base.serialize_methods :isInActiveScheme + end + + def isInActiveScheme + @isInActiveScheme + end + + def inScheme?(scheme) + self.inScheme.include?(scheme) + end + + def load_is_in_scheme(schemes = []) + included = schemes.select { |s| inScheme?(s) } + included = [self.submission.get_main_concept_scheme] if included.empty? && schemes&.empty? + @isInActiveScheme = included + end + + end + end + end +end diff --git a/lib/ontologies_linked_data/concerns/concepts/concept_sort.rb b/lib/ontologies_linked_data/concerns/concepts/concept_sort.rb new file mode 100644 index 000000000..1c42dcfa2 --- /dev/null +++ b/lib/ontologies_linked_data/concerns/concepts/concept_sort.rb @@ -0,0 +1,55 @@ +module LinkedData + module Concerns + module Concept + module Sort + module ClassMethods + def compare_classes(class_a, class_b) + label_a = "" + label_b = "" + class_a.bring(:prefLabel) if class_a.bring?(:prefLabel) + class_b.bring(:prefLabel) if class_b.bring?(:prefLabel) + + begin + label_a = class_a.prefLabel unless (class_a.prefLabel.nil? || class_a.prefLabel.empty?) + rescue Goo::Base::AttributeNotLoaded + label_a = "" + end + + begin + label_b = class_b.prefLabel unless (class_b.prefLabel.nil? || class_b.prefLabel.empty?) + rescue Goo::Base::AttributeNotLoaded + label_b = "" + end + + label_a = class_a.id if label_a.empty? + label_b = class_b.id if label_b.empty? + + [label_a.downcase] <=> [label_b.downcase] + end + + def sort_classes(classes) + classes.sort { |class_a, class_b| compare_classes(class_a, class_b) } + end + + def sort_tree_children(root_node) + sort_classes!(root_node.children) + root_node.children.each { |ch| sort_tree_children(ch) } + end + + private + + + + def sort_classes!(classes) + classes.sort! { |class_a, class_b| LinkedData::Models::Class.compare_classes(class_a, class_b) } + classes + end + end + + def self.included(base) + base.extend(ClassMethods) + end + end + end + end +end diff --git a/lib/ontologies_linked_data/concerns/concepts/concept_tree.rb b/lib/ontologies_linked_data/concerns/concepts/concept_tree.rb new file mode 100644 index 000000000..def296828 --- /dev/null +++ b/lib/ontologies_linked_data/concerns/concepts/concept_tree.rb @@ -0,0 +1,139 @@ +module LinkedData + module Concerns + module Concept + module Tree + def tree(concept_schemes: [], concept_collections: [], roots: nil) + bring(parents: [:prefLabel]) if bring?(:parents) + return self if parents.nil? || parents.empty? + extra_include = [:hasChildren, :isInActiveScheme, :isInActiveCollection] + roots = self.submission.roots( extra_include, concept_schemes:concept_schemes) if roots.nil? + path = path_to_root(roots) + threshold = 99 + + return self if path.nil? + + attrs_to_load = %i[prefLabel synonym obsolete] + attrs_to_load << :subClassOf if submission.hasOntologyLanguage.obo? + attrs_to_load += self.class.concept_is_in_attributes if submission.skos? + self.class.in(submission) + .models(path) + .include(attrs_to_load).all + load_children(path, threshold: threshold) + + path.reverse! + path.last.instance_variable_set("@children", []) + + childrens_hash = {} + path.each do |m| + next if m.id.to_s["#Thing"] + m.children.each do |c| + childrens_hash[c.id.to_s] = c + c.load_computed_attributes(to_load:extra_include , + options: {schemes: concept_schemes, collections: concept_collections}) + end + m.load_computed_attributes(to_load:extra_include , + options: {schemes: concept_schemes, collections: concept_collections}) + end + + load_children(childrens_hash.values, threshold: threshold) + + build_tree(path) + end + + def tree_sorted(concept_schemes: [], concept_collections: [], roots: nil) + tr = tree(concept_schemes: concept_schemes, concept_collections: concept_collections, roots: roots) + self.class.sort_tree_children(tr) + tr + end + + def paths_to_root(tree: false, roots: nil) + bring(parents: [:prefLabel, :synonym, :definition]) if bring?(:parents) + return [] if parents.nil? || parents.empty? + + paths = [[self]] + traverse_path_to_root(self.parents.dup, paths, 0, tree, roots) unless tree_root?(self, roots) + paths.each do |p| + p.reverse! + end + paths + end + + def path_to_root(roots) + paths = [[self]] + paths = paths_to_root(tree: true, roots: roots) + #select one path that gets to root + path = nil + paths.each do |p| + p.reverse! + unless (p.map { |x| x.id.to_s } & roots.map { |x| x.id.to_s }).empty? + path = p + break + end + end + + if path.nil? + # do one more check for root classes that don't get returned by the submission.roots call + paths.each do |p| + root_node = p.last + root_parents = root_node.parents + + if root_parents.empty? + path = p + break + end + end + end + + path + end + + def tree_root?(concept, roots) + (roots &&roots.map{|r| r.id}.include?(concept.id)) || concept.id.to_s["#Thing"] + end + + private + + def load_children(concepts, threshold: 99) + LinkedData::Models::Class + .partially_load_children(concepts, threshold, submission) + end + + def build_tree(path) + root_node = path.first + tree_node = path.first + path.delete_at(0) + while tree_node && + !tree_node.id.to_s["#Thing"] && + !tree_node.children.empty? && (!path.empty?) do + next_tree_node = nil + tree_node.load_has_children + tree_node.children.each_index do |i| + if tree_node.children[i].id.to_s == path.first.id.to_s + next_tree_node = path.first + children = tree_node.children.dup + children[i] = path.first + tree_node.instance_variable_set("@children", children) + children.each do |c| + c.load_has_children + end + else + tree_node.children[i].instance_variable_set("@children", []) + end + end + + if !path.empty? && next_tree_node.nil? + tree_node.children << path.shift + end + tree_node = next_tree_node + path.delete_at(0) + end + + root_node + end + + end + end + + end +end + diff --git a/lib/ontologies_linked_data/concerns/ontology_submissions/skos/skos_submission_roots.rb b/lib/ontologies_linked_data/concerns/ontology_submissions/skos/skos_submission_roots.rb new file mode 100644 index 000000000..4a88c12a0 --- /dev/null +++ b/lib/ontologies_linked_data/concerns/ontology_submissions/skos/skos_submission_roots.rb @@ -0,0 +1,99 @@ +module LinkedData + module Models + module SKOS + module RootsFetcher + + def skos_roots(concept_schemes, page, paged, pagesize) + classes = [] + class_ids, count = roots_by_has_top_concept(concept_schemes, page, paged, pagesize) + + class_ids, count = roots_by_top_concept_of(concept_schemes, page, paged, pagesize) if class_ids.empty? + + class_ids.each do |id| + classes << LinkedData::Models::Class.find(id).in(self).disable_rules.first + end + + classes = Goo::Base::Page.new(page, pagesize, count, classes) if paged + classes + end + + private + + def roots_by_query(query_body, page, paged, pagesize) + root_skos = <<-eos + SELECT DISTINCT ?root WHERE { + GRAPH #{self.id.to_ntriples} { + #{query_body} + }} + eos + count = 0 + + count, root_skos = add_pagination(query_body, page, pagesize, root_skos) if paged + + #needs to get cached + class_ids = [] + + Goo.sparql_query_client.query(root_skos, { graphs: [self.id] }).each_solution do |s| + class_ids << s[:root] + end + + [class_ids, count] + end + + def roots_by_has_top_concept(concept_schemes, page, paged, pagesize) + query_body = <<-eos + ?x #{RDF::SKOS[:hasTopConcept].to_ntriples} ?root . + #{concept_schemes_filter(concept_schemes)} + eos + roots_by_query query_body, page, paged, pagesize + end + + def roots_by_top_concept_of(concept_schemes, page, paged, pagesize) + query_body = <<-eos + ?root #{RDF::SKOS[:topConceptOf].to_ntriples} ?x. + #{concept_schemes_filter(concept_schemes)} + eos + roots_by_query query_body, page, paged, pagesize + end + + def add_pagination(query_body, page, pagesize, root_skos) + count = count_roots(query_body) + + offset = (page - 1) * pagesize + root_skos = "#{root_skos} LIMIT #{pagesize} OFFSET #{offset}" + [count, root_skos] + end + + def count_roots(query_body) + query = <<-eos + SELECT (COUNT(?x) as ?count) WHERE { + GRAPH #{self.id.to_ntriples} { + #{query_body} + }} + eos + rs = Goo.sparql_query_client.query(query) + count = 0 + rs.each do |sol| + count = sol[:count].object + end + count + end + + def concept_schemes_filter(concept_schemes) + concept_schemes = current_schemes(concept_schemes) + concept_schemes = concept_schemes.map { |x| RDF::URI.new(x.to_s).to_ntriples } + concept_schemes.empty? ? '' : "FILTER (?x IN (#{concept_schemes.join(',')}))" + end + + def current_schemes(concept_schemes) + if concept_schemes.nil? || concept_schemes.empty? + main_concept_scheme = get_main_concept_scheme + concept_schemes = main_concept_scheme ? [main_concept_scheme] : [] + end + concept_schemes + end + + end + end + end +end diff --git a/lib/ontologies_linked_data/concerns/ontology_submissions/skos/skos_submission_schemes.rb b/lib/ontologies_linked_data/concerns/ontology_submissions/skos/skos_submission_schemes.rb new file mode 100644 index 000000000..b275d4cd4 --- /dev/null +++ b/lib/ontologies_linked_data/concerns/ontology_submissions/skos/skos_submission_schemes.rb @@ -0,0 +1,20 @@ +module LinkedData + module Models + module SKOS + module ConceptSchemes + def get_main_concept_scheme(default_return: ontology_uri) + all = all_concepts_schemes + unless all.nil? + all = all.map { |x| x.id } + return default_return if all.include?(ontology_uri) + end + end + + def all_concepts_schemes + LinkedData::Models::SKOS::Scheme.in(self).all + end + end + end + end +end + diff --git a/lib/ontologies_linked_data/config/config.rb b/lib/ontologies_linked_data/config/config.rb index 00e5ee118..0c16c88d8 100644 --- a/lib/ontologies_linked_data/config/config.rb +++ b/lib/ontologies_linked_data/config/config.rb @@ -8,6 +8,8 @@ module LinkedData @settings = OpenStruct.new @settings_run = false + DEFAULT_PREFIX = 'http://data.bioontology.org/'.freeze + def config(&block) return if @settings_run @settings_run = true @@ -24,17 +26,17 @@ def config(&block) @settings.search_server_url ||= 'http://localhost:8983/solr/term_search_core1' @settings.property_search_server_url ||= 'http://localhost:8983/solr/prop_search_core1' @settings.repository_folder ||= './test/data/ontology_files/repo' - @settings.rest_url_prefix ||= 'http://data.bioontology.org/' + @settings.rest_url_prefix ||= DEFAULT_PREFIX @settings.enable_security ||= false @settings.enable_slices ||= false # Java/JVM options @settings.java_max_heap_size ||= '10240M' - @settings.ui_name ||= 'Bioportal' + @settings.ui_name ||= 'BioPortal' @settings.ui_host ||= 'bioportal.bioontology.org' @settings.replace_url_prefix ||= false - @settings.id_url_prefix ||= "http://data.bioontology.org/" + @settings.id_url_prefix ||= DEFAULT_PREFIX @settings.queries_debug ||= false @settings.enable_monitoring ||= false @@ -189,7 +191,7 @@ def goo_namespaces conf.add_namespace(:uneskos, RDF::Vocabulary.new("http://purl.org/umu/uneskos#")) - conf.id_prefix = 'http://data.bioontology.org/' + conf.id_prefix = DEFAULT_PREFIX conf.pluralize_models(true) end end diff --git a/lib/ontologies_linked_data/mappings/mappings.rb b/lib/ontologies_linked_data/mappings/mappings.rb index be43e9454..5d0dedb71 100644 --- a/lib/ontologies_linked_data/mappings/mappings.rb +++ b/lib/ontologies_linked_data/mappings/mappings.rb @@ -46,7 +46,7 @@ def self.handle_triple_store_downtime(logger = nil) def self.mapping_counts(enable_debug=false, logger=nil, reload_cache=false, arr_acronyms=[]) logger = nil unless enable_debug t = Time.now - latest = self.retrieve_latest_submissions(options={acronyms:arr_acronyms}) + latest = self.retrieve_latest_submissions(options={ acronyms:arr_acronyms }) counts = {} i = 0 epr = Goo.sparql_query_client(:main) diff --git a/lib/ontologies_linked_data/models/class.rb b/lib/ontologies_linked_data/models/class.rb index 471a3afb7..c5fb20d90 100644 --- a/lib/ontologies_linked_data/models/class.rb +++ b/lib/ontologies_linked_data/models/class.rb @@ -10,6 +10,10 @@ class ClassAttributeNotLoaded < StandardError end class Class < LinkedData::Models::Base + include LinkedData::Concerns::Concept::Sort + include LinkedData::Concerns::Concept::Tree + include LinkedData::Concerns::Concept::InScheme + include LinkedData::Concerns::Concept::InCollection model :class, name_with: :id, collection: :submission, namespace: :owl, :schemaless => :true, @@ -41,6 +45,9 @@ def self.urn_id(acronym,classId) attribute :label, namespace: :rdfs, enforce: [:list] attribute :prefLabel, namespace: :skos, enforce: [:existence], alias: true + attribute :prefLabelXl, property: :prefLabel, namespace: :skosxl, enforce: [:label, :list], alias: true + attribute :altLabelXl, property: :altLabel, namespace: :skosxl, enforce: [:label, :list], alias: true + attribute :hiddenLabelXl, property: :hiddenLabel, namespace: :skosxl, enforce: [:label, :list], alias: true attribute :synonym, namespace: :skos, enforce: [:list], property: :altLabel, alias: true attribute :definition, namespace: :skos, enforce: [:list], alias: true attribute :obsolete, namespace: :owl, property: :deprecated, alias: true @@ -77,18 +84,22 @@ def self.urn_id(acronym,classId) attribute :notes, inverse: { on: :note, attribute: :relatedClass } + attribute :inScheme, enforce: [:list, :uri], namespace: :skos + attribute :memberOf, namespace: :uneskos, inverse: { on: :collection , :attribute => :member } attribute :created, namespace: :dcterms attribute :modified, namespace: :dcterms # Hypermedia settings - embed :children, :ancestors, :descendants, :parents - serialize_default :prefLabel, :synonym, :definition, :cui, :semanticType, :obsolete, :matchType, :ontologyType, :provisional # an attribute used in Search (not shown out of context) + embed :children, :ancestors, :descendants, :parents, :prefLabelXl, :altLabelXl, :hiddenLabelXl + serialize_default :prefLabel, :synonym, :definition, :cui, :semanticType, :obsolete, :matchType, + :ontologyType, :provisional, # an attribute used in Search (not shown out of context) + :created, :modified, :memberOf, :inScheme serialize_methods :properties, :childrenCount, :hasChildren serialize_never :submissionAcronym, :submissionId, :submission, :descendants aggregates childrenCount: [:count, :children] links_load submission: [ontology: [:acronym]] do_not_load :descendants, :ancestors - prevent_serialize_when_nested :properties, :parents, :children, :ancestors, :descendants + prevent_serialize_when_nested :properties, :parents, :children, :ancestors, :descendants, :memberOf link_to LinkedData::Hypermedia::Link.new("self", lambda {|s| "ontologies/#{s.submission.ontology.acronym}/classes/#{CGI.escape(s.id.to_s)}"}, self.uri_type), LinkedData::Hypermedia::Link.new("ontology", lambda {|s| "ontologies/#{s.submission.ontology.acronym}"}, Goo.vocabulary["Ontology"]), LinkedData::Hypermedia::Link.new("children", lambda {|s| "ontologies/#{s.submission.ontology.acronym}/classes/#{CGI.escape(s.id.to_s)}/children"}, self.uri_type), @@ -352,23 +363,13 @@ def properties(*args) properties end - def paths_to_root() - self.bring(parents: [:prefLabel, :synonym, :definition]) if self.bring?(:parents) - return [] if self.parents.nil? or self.parents.length == 0 - paths = [[self]] - traverse_path_to_root(self.parents.dup, paths, 0) - paths.each do |p| - p.reverse! - end - paths - end - def self.partially_load_children(models, threshold, submission) ld = [:prefLabel, :definition, :synonym] ld << :subClassOf if submission.hasOntologyLanguage.obo? + ld += LinkedData::Models::Class.concept_is_in_attributes if submission.skos? + single_load = [] - query = self.in(submission) - .models(models) + query = self.in(submission).models(models) query.aggregate(:count, :children).all models.each do |cls| @@ -377,13 +378,10 @@ def self.partially_load_children(models, threshold, submission) end if cls.aggregates.first.value > threshold #too many load a page - self.in(submission) - .models(single_load) - .include(children: [:prefLabel]).all page_children = LinkedData::Models::Class - .where(parents: cls) - .include(ld) - .in(submission).page(1,threshold).all + .where(parents: cls) + .include(ld) + .in(submission).page(1,threshold).all cls.instance_variable_set("@children",page_children.to_a) cls.loaded_attributes.add(:children) @@ -392,109 +390,17 @@ def self.partially_load_children(models, threshold, submission) end end - if single_load.length > 0 - self.in(submission) - .models(single_load) - .include(ld << {children: [:prefLabel]}).all - end + self.in(submission).models(single_load).include({children: ld}).all if single_load.length > 0 end - def tree() - self.bring(parents: [:prefLabel]) if self.bring?(:parents) - return self if self.parents.nil? or self.parents.length == 0 - paths = [[self]] - traverse_path_to_root(self.parents.dup, paths, 0, tree=true) - roots = self.submission.roots(extra_include=[:hasChildren]) - threshhold = 99 - - #select one path that gets to root - path = nil - paths.each do |p| - if (p.map { |x| x.id.to_s } & roots.map { |x| x.id.to_s }).length > 0 - path = p - break - end - end - - if path.nil? - # do one more check for root classes that don't get returned by the submission.roots call - paths.each do |p| - root_node = p.last - root_parents = root_node.parents - - if root_parents.empty? - path = p - break - end - end - return self if path.nil? - end - - items_hash = {} - path.each do |t| - items_hash[t.id.to_s] = t - end - - attrs_to_load = [:prefLabel,:synonym,:obsolete] - attrs_to_load << :subClassOf if submission.hasOntologyLanguage.obo? - self.class.in(submission) - .models(items_hash.values) - .include(attrs_to_load).all - - LinkedData::Models::Class - .partially_load_children(items_hash.values,threshhold,self.submission) - - path.reverse! - path.last.instance_variable_set("@children",[]) - childrens_hash = {} - path.each do |m| - next if m.id.to_s["#Thing"] - m.children.each do |c| - childrens_hash[c.id.to_s] = c - end - end - - LinkedData::Models::Class.partially_load_children(childrens_hash.values,threshhold, self.submission) - - #build the tree - root_node = path.first - tree_node = path.first - path.delete_at(0) - while tree_node && - !tree_node.id.to_s["#Thing"] && - tree_node.children.length > 0 and path.length > 0 do - - next_tree_node = nil - tree_node.load_has_children - tree_node.children.each_index do |i| - if tree_node.children[i].id.to_s == path.first.id.to_s - next_tree_node = path.first - children = tree_node.children.dup - children[i] = path.first - tree_node.instance_variable_set("@children",children) - children.each do |c| - c.load_has_children - end - else - tree_node.children[i].instance_variable_set("@children",[]) - end - end - - if path.length > 0 && next_tree_node.nil? - tree_node.children << path.shift - end - - tree_node = next_tree_node - path.delete_at(0) - end - - root_node + def load_computed_attributes(to_load:, options:) + self.load_has_children if to_load&.include?(:hasChildren) + self.load_is_in_scheme(options[:schemes]) if to_load&.include?(:isInActiveScheme) + self.load_is_in_collection(options[:collections]) if to_load&.include?(:isInActiveCollection) end - def tree_sorted() - tr = tree - self.class.sort_tree_children(tr) - tr + def self.concept_is_in_attributes + [:inScheme, :isInActiveScheme, :memberOf, :isInActiveCollection] end def retrieve_ancestors() @@ -636,11 +542,11 @@ def append_if_not_there_already(path, r) path << r end - def traverse_path_to_root(parents, paths, path_i, tree=false) - return if (tree and parents.length == 0) + def traverse_path_to_root(parents, paths, path_i, tree = false, roots = nil) + return if (tree && parents.length == 0) + recursions = [path_i] recurse_on_path = [false] - if parents.length > 1 and not tree (parents.length-1).times do paths << paths[path_i].clone @@ -651,7 +557,7 @@ def traverse_path_to_root(parents, paths, path_i, tree=false) parents.each_index do |i| rec_i = recursions[i] recurse_on_path[i] = recurse_on_path[i] || - !append_if_not_there_already(paths[rec_i], parents[i]).nil? + !append_if_not_there_already(paths[rec_i], parents[i]).nil? end else path = paths[path_i] @@ -664,62 +570,23 @@ def traverse_path_to_root(parents, paths, path_i, tree=false) p = path.last next if p.id.to_s["umls/OrphanClass"] - if p.bring?(:parents) - p.bring(parents: [:prefLabel, :synonym, :definition, parents: [:prefLabel, :synonym, :definition]]) - end + if !tree_root?(p, roots) && recurse_on_path[i] + if p.bring?(:parents) + p.bring(parents: [:prefLabel, :synonym, :definition, :inScheme, parents: [:prefLabel, :synonym, :definition, :inScheme]]) + end - if !p.loaded_attributes.include?(:parents) - # fail safely - logger = LinkedData::Parser.logger || Logger.new($stderr) - logger.error("Class #{p.id.to_s} from #{p.submission.id} cannot load parents") - return - end + if !p.loaded_attributes.include?(:parents) + # fail safely + logger = LinkedData::Parser.logger || Logger.new($stderr) + logger.error("Class #{p.id.to_s} from #{p.submission.id} cannot load parents") + return + end - if !p.id.to_s["#Thing"] &&\ - (recurse_on_path[i] && p.parents && p.parents.length > 0) - traverse_path_to_root(p.parents.dup, paths, rec_i, tree=tree) + traverse_path_to_root(p.parents.dup, paths, rec_i, tree=tree, roots=roots) end end end - def self.sort_tree_children(root_node) - self.sort_classes!(root_node.children) - root_node.children.each { |ch| self.sort_tree_children(ch) } - end - - def self.sort_classes(classes) - classes.sort { |class_a, class_b| self.compare_classes(class_a, class_b) } - end - - def self.sort_classes!(classes) - classes.sort! { |class_a, class_b| self.compare_classes(class_a, class_b) } - classes - end - - def self.compare_classes(class_a, class_b) - label_a = "" - label_b = "" - class_a.bring(:prefLabel) if class_a.bring?(:prefLabel) - class_b.bring(:prefLabel) if class_b.bring?(:prefLabel) - - begin - label_a = class_a.prefLabel unless (class_a.prefLabel.nil? || class_a.prefLabel.empty?) - rescue Goo::Base::AttributeNotLoaded - label_a = "" - end - - begin - label_b = class_b.prefLabel unless (class_b.prefLabel.nil? || class_b.prefLabel.empty?) - rescue Goo::Base::AttributeNotLoaded - label_b = "" - end - - label_a = class_a.id if label_a.empty? - label_b = class_b.id if label_b.empty? - - [label_a.downcase] <=> [label_b.downcase] - end - end end end diff --git a/lib/ontologies_linked_data/models/instance.rb b/lib/ontologies_linked_data/models/instance.rb index 85bdc3f27..25f26eb97 100644 --- a/lib/ontologies_linked_data/models/instance.rb +++ b/lib/ontologies_linked_data/models/instance.rb @@ -1,156 +1,69 @@ module LinkedData module Models - class Instance - include LinkedData::Hypermedia::Resource - include LinkedData::HTTPCache::CacheableResource + class Instance < LinkedData::Models::Base - attr_reader :id, :properties - attr_accessor :label - serialize_default :id, :label, :properties + model :named_individual, name_with: :id, collection: :submission, + namespace: :owl, schemaless: :true , rdf_type: lambda { |*x| RDF::OWL[:NamedIndividual]} - # HTTP cache settings. - cache_timeout 14400 + attribute :label, namespace: :rdfs, enforce: [:list] + attribute :prefLabel, namespace: :skos, enforce: [:existence], alias: true - def initialize(id,label,properties) - @id = id - if label.nil? - sep = "/" - if not id.to_s["#"].nil? - sep = "#" - end - label = id.to_s.split(sep).last - end - @label = label - @properties = properties - end + attribute :types, namespace: :rdf, enforce: [:list], property: :type + attribute :submission, collection: lambda { |s| s.resource_id }, namespace: :metadata - def add_property_value(p,o) - ps = p.to_s - if not @properties.include?(ps) - @properties[ps] = [] - end - @properties[ps] << o - end + serialize_never :submission, :id + serialize_methods :properties + + cache_timeout 14400 - def self.type_uri - LinkedData.settings.id_url_prefix+"metadata/Instance" + def properties + self.unmapped end + end end + module InstanceLoader def self.count_instances_by_class(submission_id,class_id) - query = <<-eos -PREFIX owl: -SELECT (count(DISTINCT ?s) as ?c) WHERE - { - GRAPH <#{submission_id.to_s}> { - ?s a owl:NamedIndividual . - ?s a <#{class_id.to_s}> . - } - } -eos - epr = Goo.sparql_query_client(:main) - graphs = [submission_id] - resultset = epr.query(query, graphs: graphs) - resultset.each do |r| - return r[:c].object - end - return 0 + ## TODO: pass directly an LinkedData::Models::OntologySubmission instance in the arguments instead of submission_id + s = LinkedData::Models::OntologySubmission.find(submission_id).first + instances_by_class_where_query(s, class_id: class_id).count end - def self.get_instances_by_class(submission_id,class_id) - query = <<-eos -PREFIX owl: -SELECT ?s ?label WHERE - { - GRAPH <#{submission_id.to_s}> { - ?s a owl:NamedIndividual . - ?s a <#{class_id.to_s}> . - } - } -eos - epr = Goo.sparql_query_client(:main) - graphs = [submission_id] - resultset = epr.query(query, graphs: graphs) - instances = [] - resultset.each do |r| - inst = LinkedData::Models::Instance.new(r[:s],nil,{}) - instances << inst - end - - if instances.empty? - return [] - end - - include_instance_properties(submission_id,instances) - return instances + def self.get_instances_by_class(submission_id, class_id, page_no: nil, size: nil) + ## TODO: pass directly an LinkedData::Models::OntologySubmission instance in the arguments instead of submission_id + s = LinkedData::Models::OntologySubmission.find(submission_id).first + + inst = instances_by_class_where_query(s, class_id: class_id, page_no: page_no, size: size).all + + # TODO test if "include=all" parameter is passed in the request + # For getting all the properties # For getting all the properties + load_unmapped s,inst unless inst.nil? || inst.empty? + inst end - def self.get_instances_by_ontology(submission_id,page_no,size) - query = <<-eos -PREFIX owl: -SELECT ?s ?label WHERE - { - GRAPH <#{submission_id.to_s}> { - ?s a owl:NamedIndividual . - } - } -eos - epr = Goo.sparql_query_client(:main) - graphs = [submission_id] - resultset = epr.query(query, graphs: graphs) - - total_size = resultset.size - range_start = (page_no - 1) * size - range_end = (page_no * size) - 1 - resultset = resultset[range_start..range_end] - - instances = [] - resultset.each do |r| - inst = LinkedData::Models::Instance.new(r[:s],r[:label],{}) - instances << inst - end unless resultset.nil? - - if instances.size > 0 - include_instance_properties(submission_id,instances) - end - - page = Goo::Base::Page.new(page_no,size,total_size,instances) - return page + def self.get_instances_by_ontology(submission_id, page_no: nil, size: nil) + ## TODO: pass directly an LinkedData::Models::OntologySubmission instance in the arguments instead of submission_id + s = LinkedData::Models::OntologySubmission.find(submission_id).first + inst = s.nil? ? [] : instances_by_class_where_query(s, page_no: page_no, size: size).all + + ## TODO test if "include=all" parameter is passed in the request + load_unmapped s, inst unless inst.nil? || inst.empty? # For getting all the properties + inst end - def self.include_instance_properties(submission_id,instances) - index = Hash.new - instances.each do |inst| - index[inst.id.to_s] = inst - end - uris = index.keys.map { |x| x.to_s } - uri_filter = uris.map { |x| "?s = <#{x}>"}.join(" || ") - - query = <<-eos -PREFIX owl: -SELECT ?s ?p ?o WHERE - { - GRAPH <#{submission_id.to_s}> { - ?s ?p ?o . - } - FILTER( #{uri_filter} ) - } -eos - epr = Goo.sparql_query_client(:main) - graphs = [submission_id] - resultset = epr.query(query, graphs: graphs) - resultset.each do |sol| - s = sol[:s] - p = sol[:p] - o = sol[:o] - if not p.to_s["label"].nil? - index[s.to_s].label = o.to_s - else - index[s.to_s].add_property_value(p,o) - end - end + def self.instances_by_class_where_query(submission, class_id: nil, page_no: nil, size: nil) + where_condition = class_id.nil? ? nil : {types: RDF::URI.new(class_id.to_s)} + query = LinkedData::Models::Instance.where(where_condition).in(submission).include(:types, :label, :prefLabel) + query.page(page_no, size) unless page_no.nil? + query + end + + def self.load_unmapped(submission, models) + LinkedData::Models::Instance.where.in(submission).models(models).include(:unmapped).all end + + end -end +end \ No newline at end of file diff --git a/lib/ontologies_linked_data/models/ontology.rb b/lib/ontologies_linked_data/models/ontology.rb index 9cc53cf37..456115535 100644 --- a/lib/ontologies_linked_data/models/ontology.rb +++ b/lib/ontologies_linked_data/models/ontology.rb @@ -6,6 +6,9 @@ require 'ontologies_linked_data/models/metric' require 'ontologies_linked_data/models/category' require 'ontologies_linked_data/models/project' +require 'ontologies_linked_data/models/skos/scheme' +require 'ontologies_linked_data/models/skos/collection' +require 'ontologies_linked_data/models/skos/skosxl' require 'ontologies_linked_data/models/notes/note' require 'ontologies_linked_data/purl/purl_client' @@ -58,6 +61,9 @@ class OntologyAnalyticsError < StandardError; end LinkedData::Hypermedia::Link.new("classes", lambda {|s| "ontologies/#{s.acronym}/classes"}, LinkedData::Models::Class.uri_type), LinkedData::Hypermedia::Link.new("single_class", lambda {|s| "ontologies/#{s.acronym}/classes/{class_id}"}, LinkedData::Models::Class.uri_type), LinkedData::Hypermedia::Link.new("roots", lambda {|s| "ontologies/#{s.acronym}/classes/roots"}, LinkedData::Models::Class.uri_type), + LinkedData::Hypermedia::Link.new("schemes", lambda {|s| "ontologies/#{s.acronym}/schemes"}, LinkedData::Models::SKOS::Scheme.uri_type), + LinkedData::Hypermedia::Link.new("collections", lambda {|s| "ontologies/#{s.acronym}/collections"}, LinkedData::Models::SKOS::Collection.uri_type), + LinkedData::Hypermedia::Link.new("xl_labels", lambda {|s| "ontologies/#{s.acronym}/skos_xl_labels"}, LinkedData::Models::SKOS::Label.uri_type), LinkedData::Hypermedia::Link.new("instances", lambda {|s| "ontologies/#{s.acronym}/instances"}, Goo.vocabulary["Instance"]), LinkedData::Hypermedia::Link.new("metrics", lambda {|s| "ontologies/#{s.acronym}/metrics"}, LinkedData::Models::Metric.type_uri), LinkedData::Hypermedia::Link.new("reviews", lambda {|s| "ontologies/#{s.acronym}/reviews"}, LinkedData::Models::Review.uri_type), diff --git a/lib/ontologies_linked_data/models/ontology_submission.rb b/lib/ontologies_linked_data/models/ontology_submission.rb index 25a31a87e..34f5a6ac4 100644 --- a/lib/ontologies_linked_data/models/ontology_submission.rb +++ b/lib/ontologies_linked_data/models/ontology_submission.rb @@ -17,6 +17,9 @@ class OntologySubmission < LinkedData::Models::Base include LinkedData::Concerns::OntologySubmission::Validators extend LinkedData::Concerns::OntologySubmission::DefaultCallbacks + include SKOS::ConceptSchemes + include SKOS::RootsFetcher + FLAT_ROOTS_LIMIT = 1000 # default file permissions for files copied from tempdir REPOSITORY_FILE_MODE = 0o660 # rw-rw---- @@ -218,11 +221,6 @@ def self.embed_values_hash serialize_default :contact, :ontology, :hasOntologyLanguage, :released, :creationDate, :homepage, :publication, :documentation, :version, :description, :status, :submissionId - # Links - links_load :submissionId, ontology: [:acronym] - link_to LinkedData::Hypermedia::Link.new("metrics", lambda {|s| "#{self.ontology_link(s)}/submissions/#{s.submissionId}/metrics"}, self.type_uri) - LinkedData::Hypermedia::Link.new("download", lambda {|s| "#{self.ontology_link(s)}/submissions/#{s.submissionId}/download"}, self.type_uri) - # HTTP Cache settings cache_timeout 3600 cache_segment_instance lambda {|sub| segment_instance(sub)} @@ -230,7 +228,7 @@ def self.embed_values_hash cache_load ontology: [:acronym] # Access control - read_restriction_based_on lambda {|sub| sub.ontology} + read_restriction_based_on lambda { |sub| sub.ontology } access_control_load ontology: [:administeredBy, :acl, :viewingRestriction] def initialize(*args) @@ -682,7 +680,7 @@ def delete(*args) FileUtils.remove_dir(self.data_folder) if Dir.exist?(self.data_folder) end - def roots(extra_include=nil, page=nil, pagesize=nil) + def roots(extra_include = [], page = nil, pagesize = nil, concept_schemes: [], concept_collections: []) self.bring(:ontology) unless self.loaded_attributes.include?(:ontology) self.bring(:hasOntologyLanguage) unless self.loaded_attributes.include?(:hasOntologyLanguage) paged = false @@ -694,46 +692,12 @@ def roots(extra_include=nil, page=nil, pagesize=nil) paged = true end - skos = self.hasOntologyLanguage&.skos? + skos = self.skos? classes = [] if skos - root_skos = < [self.id] }).each_solution do |s| - class_ids << s[:root] - end - - class_ids.each do |id| - classes << LinkedData::Models::Class.find(id).in(self).disable_rules.first - end - - classes = Goo::Base::Page.new(page, pagesize, count, classes) if paged + classes = skos_roots(concept_schemes, page, paged, pagesize) + extra_include += LinkedData::Models::Class.concept_is_in_attributes else self.ontology.bring(:flat) data_query = nil @@ -766,7 +730,7 @@ def roots(extra_include=nil, page=nil, pagesize=nil) where = LinkedData::Models::Class.in(self).models(classes).include(:prefLabel, :definition, :synonym, :obsolete) if extra_include - [:prefLabel, :definition, :synonym, :obsolete, :childrenCount].each do |x| + %i[prefLabel definition synonym obsolete childrenCount].each do |x| extra_include.delete x end end @@ -786,25 +750,77 @@ def roots(extra_include=nil, page=nil, pagesize=nil) load_children = [:children] end - if extra_include.length > 0 - where.include(extra_include) - end + where.include(extra_include) if extra_include.length > 0 end where.all - if load_children.length > 0 - LinkedData::Models::Class.partially_load_children(classes, 99, self) - end + LinkedData::Models::Class.partially_load_children(classes, 99, self) if load_children.length > 0 classes.delete_if { |c| obs = !c.obsolete.nil? && c.obsolete == true - c.load_has_children if extra_include&.include?(:hasChildren) && !obs + if !obs + c.load_computed_attributes(to_load: extra_include, + options: { schemes: current_schemes(concept_schemes), collections: concept_collections }) + end obs } - classes end + def children(cls, includes_param: [], concept_schemes: [], concept_collections: [], page: 1, size: 50) + ld = LinkedData::Models::Class.goo_attrs_to_load(includes_param) + unmapped = ld.delete(:properties) + + ld += LinkedData::Models::Class.concept_is_in_attributes if skos? + + page_data_query = LinkedData::Models::Class.where(parents: cls).in(self).include(ld) + aggregates = LinkedData::Models::Class.goo_aggregates_to_load(ld) + page_data_query.aggregate(*aggregates) unless aggregates.empty? + page_data = page_data_query.page(page, size).all + LinkedData::Models::Class.in(self).models(page_data).include(:unmapped).all if unmapped + + page_data.delete_if { |x| x.id.to_s == cls.id.to_s } + if ld.include?(:hasChildren) || ld.include?(:isInActiveScheme) || ld.include?(:isInActiveCollection) + page_data.each do |c| + c.load_computed_attributes(to_load: ld, + options: { schemes: concept_schemes, collections: concept_collections }) + end + end + + unless concept_schemes.empty? + page_data.delete_if { |c| Array(c.isInActiveScheme).empty? && !c.load_has_children } + if (page_data.size < size) && page_data.next_page + page_data += children(cls, includes_param: includes_param, concept_schemes: concept_schemes, + concept_collections: concept_collections, + page: page_data.next_page, size: size) + end + end + + page_data + end + + def skos? + self.bring :hasOntologyLanguage if bring? :hasOntologyLanguage + self.hasOntologyLanguage&.skos? + end + + def ontology_uri + self.bring(:uri) if self.bring? :uri + RDF::URI.new(self.uri) + end + + + + + + + + + + + + + def roots_sorted(extra_include=nil) classes = roots(extra_include) LinkedData::Models::Class.sort_classes(classes) diff --git a/lib/ontologies_linked_data/models/skos/collection.rb b/lib/ontologies_linked_data/models/skos/collection.rb new file mode 100644 index 000000000..afc5724e4 --- /dev/null +++ b/lib/ontologies_linked_data/models/skos/collection.rb @@ -0,0 +1,43 @@ +module LinkedData + module Models + module SKOS + class Collection < LinkedData::Models::Base + + model :collection, name_with: :id, collection: :submission, + namespace: :skos, schemaless: :true, rdf_type: ->(*x) { RDF::SKOS[:Collection] } + + attribute :prefLabel, namespace: :skos, enforce: [:existence] + attribute :member, namespace: :skos, enforce: [:list, :class] + attribute :submission, collection: ->(s) { s.resource_id }, namespace: :metadata + + embed :member + serialize_default :prefLabel, :memberCount + serialize_never :submission, :id, :member + serialize_methods :properties, :memberCount + aggregates memberCount: [:count, :member] + + cache_timeout 14400 + + link_to LinkedData::Hypermedia::Link.new('self', + ->(s) { "ontologies/#{s.submission.ontology.acronym}/collections/#{CGI.escape(s.id.to_s)}"}, + self.uri_type), + LinkedData::Hypermedia::Link.new('members', + ->(s) { "ontologies/#{s.submission.ontology.acronym}/collections/#{CGI.escape(s.id.to_s)}/members"}, + Goo.vocabulary(:skos)['Concept']), + LinkedData::Hypermedia::Link.new('ontology', ->(s) { "ontologies/#{s.submission.ontology.acronym}"}, + Goo.vocabulary['Ontology']) + + def properties + self.unmapped + end + + def memberCount + sol = self.class.in(submission).models([self]).aggregate(:count, :member).first + sol.nil? ? 0 : sol.aggregates.first.value + end + + end + end + end + +end diff --git a/lib/ontologies_linked_data/models/skos/scheme.rb b/lib/ontologies_linked_data/models/skos/scheme.rb new file mode 100644 index 000000000..37e041896 --- /dev/null +++ b/lib/ontologies_linked_data/models/skos/scheme.rb @@ -0,0 +1,35 @@ +module LinkedData + module Models + module SKOS + class Scheme < LinkedData::Models::Base + + model :scheme, name_with: :id, collection: :submission, + namespace: :skos, schemaless: :true, rdf_type: ->(*x) { RDF::SKOS[:ConceptScheme] } + + attribute :prefLabel, namespace: :skos, enforce: [:existence] + + attribute :submission, collection: ->(s) { s.resource_id }, namespace: :metadata + + serialize_never :submission, :id + serialize_methods :properties + + cache_timeout 14400 + + link_to LinkedData::Hypermedia::Link.new('self', + ->(s) { "ontologies/#{s.submission.ontology.acronym}/schemes/#{CGI.escape(s.id.to_s)}"}, + self.uri_type), + LinkedData::Hypermedia::Link.new('roots', + ->(s) { "ontologies/#{s.submission.ontology.acronym}/classes/roots?concept_scheme=#{CGI.escape(s.id.to_s)}"}, + Goo.vocabulary(:skos)['Concept']), + LinkedData::Hypermedia::Link.new('ontology', ->(s) { "ontologies/#{s.submission.ontology.acronym}"}, + Goo.vocabulary['Ontology']) + + def properties + self.unmapped + end + + end + end + end + +end diff --git a/lib/ontologies_linked_data/models/skos/skosxl.rb b/lib/ontologies_linked_data/models/skos/skosxl.rb new file mode 100644 index 000000000..9b95b7cc4 --- /dev/null +++ b/lib/ontologies_linked_data/models/skos/skosxl.rb @@ -0,0 +1,25 @@ +module LinkedData + module Models + module SKOS + class Label < LinkedData::Models::Base + + model :label, name_with: :id, collection: :submission, + namespace: :skos, rdf_type: ->(*x) { RDF::URI.new('http://www.w3.org/2008/05/skos-xl#Label') } + + attribute :literalForm, namespace: :skosxl, enforce: [:existence] + attribute :submission, collection: ->(s) { s.resource_id }, namespace: :metadata + + serialize_never :submission, :id + serialize_methods :properties + + link_to LinkedData::Hypermedia::Link.new('self', ->(s) { "ontologies/#{s.submission.ontology.acronym}/skos_xl_labels/#{CGI.escape(s.id)}"}, self.uri_type) + + def properties + self.unmapped + end + + end + end + end + +end diff --git a/lib/ontologies_linked_data/utils/notifications.rb b/lib/ontologies_linked_data/utils/notifications.rb index fc28b21b5..7a7630258 100644 --- a/lib/ontologies_linked_data/utils/notifications.rb +++ b/lib/ontologies_linked_data/utils/notifications.rb @@ -106,16 +106,41 @@ def self.reset_password(user, token) def self.obofoundry_sync(missing_onts, obsolete_onts) ui_name = LinkedData.settings.ui_name + subject = "[#{ui_name}] OBO Foundry synchronization report" + recipients = Notifier.ontoportal_admin_emails + body = render_template('obofoundry_sync.erb', { + ui_name: ui_name, + missing_onts: missing_onts, + obsolete_onts: obsolete_onts, + }) + + Notifier.notify_mails_grouped(subject, body, recipients) + end + + def self.cloudflare_analytics(result_data) + ui_name = LinkedData.settings.ui_name + subject = "[#{ui_name}] Cloudflare Analytics daily collection result: #{result_data[:status]}" + recipients = Notifier.ontoportal_admin_emails + body = render_template('cloudflare_analytics.erb', { + result_data: result_data + }) + + Notifier.notify_mails_grouped(subject, body, recipients) + end + + private + + def self.render_template(template_name, locals = {}) gem_path = Gem.loaded_specs['ontologies_linked_data'].full_gem_path - template = File.read(File.join(gem_path, 'views/emails/obofoundry_sync.erb')) + template_path = File.join(gem_path, 'views', 'emails', template_name) + template = File.read(template_path) b = binding - b.local_variable_set(:ui_name, ui_name) - b.local_variable_set(:missing_onts, missing_onts) - b.local_variable_set(:obsolete_onts, obsolete_onts) - body = ERB.new(template).result(b) + locals.each { |k, v| b.local_variable_set(k, v) } - Notifier.notify_ontoportal_admins("[#{ui_name}] OBO Foundry synchronization report", body) + ERB.new(template).result(b) + rescue Errno::ENOENT => e + raise "Template not found: #{template_path}" end NEW_NOTE = < + @@ -710,6 +711,7 @@ + @@ -4258,14 +4260,27 @@ - - - + + + + + + + + + + + + + + + + diff --git a/test/models/skos/test_collections.rb b/test/models/skos/test_collections.rb new file mode 100644 index 000000000..c2c1c3d07 --- /dev/null +++ b/test/models/skos/test_collections.rb @@ -0,0 +1,58 @@ +require_relative '../test_ontology_common' +require 'logger' + +class TestCollections < LinkedData::TestOntologyCommon + + def self.before_suite + LinkedData::TestCase.backend_4s_delete + self.new('').submission_parse('INRAETHES', 'Testing skos', + 'test/data/ontology_files/thesaurusINRAE_nouv_structure.skos', + 1, + process_rdf: true, index_search: false, + run_metrics: false, reasoning: false) + end + + def test_collections_all + ont = 'INRAETHES' + sub = LinkedData::Models::Ontology.find(ont).first.latest_submission + collections = LinkedData::Models::SKOS::Collection.in(sub).include(:members, :prefLabel).all + + assert_equal 2, collections.size + collections_test = test_data + + collections.each_with_index do |x, i| + collection_test = collections_test[i] + assert_equal collection_test[:id], x.id.to_s + assert_equal collection_test[:prefLabel], x.prefLabel + assert_equal collection_test[:memberCount], x.memberCount + end + end + + def test_collection_members + ont = 'INRAETHES' + sub = LinkedData::Models::Ontology.find(ont).first.latest_submission + collection_test = test_data.first + collection = LinkedData::Models::SKOS::Collection.find(collection_test[:id]).in(sub).include(:member, :prefLabel).first + + refute_nil collection + members = collection.member + assert_equal collection_test[:memberCount], members.size + end + + private + + def test_data + [ + { + "id": 'http://opendata.inrae.fr/thesaurusINRAE/gr_6c79e7c5', + "prefLabel": 'GR. DEFINED CONCEPTS', + "memberCount": 295 + }, + { + "prefLabel": 'GR. DISCIPLINES', + "memberCount": 233, + "id": 'http://opendata.inrae.fr/thesaurusINRAE/skosCollection_e25f9c62' + } + ] + end +end diff --git a/test/models/skos/test_schemes.rb b/test/models/skos/test_schemes.rb new file mode 100644 index 000000000..ca1bf686c --- /dev/null +++ b/test/models/skos/test_schemes.rb @@ -0,0 +1,368 @@ +require_relative '../test_ontology_common' +require 'logger' + +class TestSchemes < LinkedData::TestOntologyCommon + + def self.before_suite + LinkedData::TestCase.backend_4s_delete + self.new('').submission_parse('INRAETHES', 'Testing skos', + 'test/data/ontology_files/thesaurusINRAE_nouv_structure.skos', + 1, + process_rdf: true, extract_metadata: false, + generate_missing_labels: false) + end + + def test_schemes_all + ont = 'INRAETHES' + sub = LinkedData::Models::Ontology.find(ont).first.latest_submission + schemes = LinkedData::Models::SKOS::Scheme.in(sub).include(:prefLabel).all + + assert_equal 66, schemes.size + schemes_test = test_data + schemes_test = schemes_test.sort_by { |x| x[:id] } + schemes = schemes.sort_by { |x| x.id.to_s } + + schemes.each_with_index do |x, i| + scheme_test = schemes_test[i] + assert_equal scheme_test[:id], x.id.to_s + assert_equal scheme_test[:prefLabel], x.prefLabel + end + end + + private + + def test_data + [ + { + "prefLabel": 'BIO neurosciences', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_74', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'INRAE domains', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/domainesINRAE', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'EAR meteorology and climatology', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_107', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'HEA prevention and therapy', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_77', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'PHY materials sciences', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_85', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'BIO cell biology', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_64', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'SSH human geography', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_20661', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'BIO immunology', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_75', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'APP variables, parameters and data', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_23256', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'SSH information and communication', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_20962', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'BIO diet and nutrition', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_23276', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'SSH research and education', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_20150', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'CON processing technology and equipment', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_54', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'BIO molecular biology and biochemistry', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_65', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'EAR soil sciences', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_105', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'BIO toxicology and ecotoxicology', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_68', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'AGR farms and farming systems', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_44', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'AGR plant cultural practices and experimentations', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_47', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'CHE chemical and physicochemical analysis', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_23260', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'CON quality of processed products', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_55', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'EAR geology and geomorphology', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_104', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'APP research methods', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_98', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'SSH laws and standards', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_21670', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'BIO general biology', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_26224', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'BIO ethology', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_69', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'PHY energy and thermodynamics', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_86', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'PHY mechanics and robotics', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_88', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'HEA health and welfare', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_78', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'ENV environment and natural resources', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_14', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'PHY civil engineering', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_89', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'HEA diseases, disorders and symptoms', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_76', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'HEA disease vectors', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_79', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'ENV natural and technological hazards', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_17', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'CHE chemical compounds and elements', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_100', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'ENV waste', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_15', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'CON supply chain management', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_56', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'AGR animal husbandry and breeding', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_48', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'ENV water management', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_18', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'AGR agricultural products', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_46', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'MAT computer sciences and artificial intelligence', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_91', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'AGR agricultural machinery and equipment', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_49', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'BIO microbiology', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_71', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'ENV pollution', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_16', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'BIO physiology', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_72', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'SSH culture and humanities', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_26297', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'MAT mathematics and statistics', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_90', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": nil, + "id": 'http://opendata.inrae.fr/thesaurusINRAE/thesaurusINRAE', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'SSH politics and administration', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_22445', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'EAR hydrology', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_106', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'AGR agricultural management', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_26298', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'SSH management sciences', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_21074', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'SSH economics', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_20544', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'BIO anatomy and body fluids', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_63', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'ORG taxonomic classification of organisms', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_26190', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'CHE chemical reactions and physicochemical phenomena', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_102', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'AGR hunting and fishing', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_50', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'CON processed biobased products', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_53', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'BIO genetics', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_70', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'ORG organisms related notions', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_26191', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'SSH sociology and psychology', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_20262', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'APP research equipment', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_97', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'CHE chemical and physicochemical properties', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_101', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'BIO ecology', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_67', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'PHY physical properties of matter', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_84', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'EAR physical geography', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_103', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + }, + { + "prefLabel": 'PHY hydraulics and aeraulics', + "id": 'http://opendata.inrae.fr/thesaurusINRAE/mt_87', + "type": 'http://www.w3.org/2004/02/skos/core#ConceptScheme' + } + ] + end +end diff --git a/test/models/skos/test_skos_xl.rb b/test/models/skos/test_skos_xl.rb new file mode 100644 index 000000000..5b05b8063 --- /dev/null +++ b/test/models/skos/test_skos_xl.rb @@ -0,0 +1,67 @@ +require_relative '../test_ontology_common' +require 'logger' + +class TestSkosXlLabel < LinkedData::TestOntologyCommon + + def self.before_suite + LinkedData::TestCase.backend_4s_delete + self.new('').submission_parse('INRAETHES', 'Testing skos', + 'test/data/ontology_files/thesaurusINRAE_nouv_structure.skos', + 1, + process_rdf: true, index_search: false, + run_metrics: false, reasoning: false) + end + + def test_skos_xl_label_all + ont = 'INRAETHES' + sub = LinkedData::Models::Ontology.find(ont).first.latest_submission + labels = LinkedData::Models::SKOS::Label.in(sub).include(:literalForm).all + assert_equal 2, labels.size + tests_labels = test_data + labels.each do |label| + test_label = tests_labels.select { |x| x[:id].eql?(label.id.to_s) } + refute_nil test_label.first + label_test(label, test_label.first) + end + end + + def test_class_skos_xl_label + ont = 'INRAETHES' + ont = LinkedData::Models::Ontology.find(ont).first + sub = ont.latest_submission + + sub.bring_remaining + sub.hasOntologyLanguage = LinkedData::Models::OntologyFormat.find('SKOS').first + sub.save + + class_test = LinkedData::Models::Class.find('http://opendata.inrae.fr/thesaurusINRAE/c_16193') + .in(sub).include(:prefLabel, + altLabelXl: [:literalForm], + prefLabelXl: [:literalForm], + hiddenLabelXl: [:literalForm]).first + + refute_nil class_test + assert_equal 1, class_test.altLabelXl.size + assert_equal 1, class_test.prefLabelXl.size + assert_equal 1, class_test.hiddenLabelXl.size + tests_labels = test_data + + label_test(class_test.altLabelXl.first, tests_labels[0]) + label_test(class_test.prefLabelXl.first, tests_labels[1]) + label_test(class_test.hiddenLabelXl.first, tests_labels[1]) + end + + private + + def test_data + [ + { id: 'http://aims.fao.org/aos/agrovoc/xl_tr_1331561625299', literalForm: 'aktivite' }, + { id: 'http://aims.fao.org/aos/agrovoc/xl_en_668053a7', literalForm: 'air-water exchanges' } + ] + end + + def label_test(label, label_test) + assert_equal label_test[:id], label.id.to_s + assert_equal label_test[:literalForm], label.literalForm + end +end \ No newline at end of file diff --git a/test/models/test_instances.rb b/test/models/test_instances.rb index 0874294d5..2d77621a8 100644 --- a/test/models/test_instances.rb +++ b/test/models/test_instances.rb @@ -1,26 +1,25 @@ -require_relative "./test_ontology_common" -require "logger" +require_relative './test_ontology_common' +require 'logger' class TestInstances < LinkedData::TestOntologyCommon - PROP_TYPE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" - PROP_CLINICAL_MANIFESTATION = "http://www.owl-ontologies.com/OntologyXCT.owl#isClinicalManifestationOf" - PROP_OBSERVABLE_TRAIT = "http://www.owl-ontologies.com/OntologyXCT.owl#isObservableTraitof" - PROP_HAS_OCCURRENCE = "http://www.owl-ontologies.com/OntologyXCT.owl#hasOccurrenceIn" + PROP_TYPE = RDF::URI.new 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type'.freeze + PROP_CLINICAL_MANIFESTATION = RDF::URI.new 'http://www.owl-ontologies.com/OntologyXCT.owl#isClinicalManifestationOf'.freeze + PROP_OBSERVABLE_TRAIT = RDF::URI.new'http://www.owl-ontologies.com/OntologyXCT.owl#isObservableTraitof'.freeze + PROP_HAS_OCCURRENCE = RDF::URI.new'http://www.owl-ontologies.com/OntologyXCT.owl#hasOccurrenceIn'.freeze + def self.before_suite - LinkedData::TestCase.backend_4s_delete + self.new('').submission_parse('TESTINST', 'Testing instances', + 'test/data/ontology_files/XCTontologyvtemp2_vvtemp2.zip', + 12, + masterFileName: 'XCTontologyvtemp2/XCTontologyvtemp2.owl', + process_rdf: true, extract_metadata: false, generate_missing_labels: false) end def test_instance_counts_class - submission_parse("TESTINST", "Testing instances", - "test/data/ontology_files/XCTontologyvtemp2_vvtemp2.zip", - 12, - masterFileName: "XCTontologyvtemp2/XCTontologyvtemp2.owl", - process_rdf: true, index_search: false, - run_metrics: false, reasoning: true) - submission_id = LinkedData::Models::OntologySubmission.all.first.id - class_id = RDF::URI.new("http://www.owl-ontologies.com/OntologyXCT.owl#ClinicalManifestation") + submission_id = RDF::URI.new("http://data.bioontology.org/ontologies/TESTINST/submissions/12") + class_id = RDF::URI.new('http://www.owl-ontologies.com/OntologyXCT.owl#ClinicalManifestation') instances = LinkedData::InstanceLoader.get_instances_by_class(submission_id, class_id) assert_equal 385, instances.length @@ -30,56 +29,38 @@ def test_instance_counts_class end def test_instance_counts_ontology - submission_parse("TESTINST", "Testing instances", - "test/data/ontology_files/XCTontologyvtemp2_vvtemp2.zip", - 12, - masterFileName: "XCTontologyvtemp2/XCTontologyvtemp2.owl", - process_rdf: true, index_search: false, - run_metrics: false, reasoning: true) - submission_id = LinkedData::Models::OntologySubmission.all.first.id - instances = LinkedData::InstanceLoader.get_instances_by_ontology(submission_id, 1, 800) + submission_id = RDF::URI.new("http://data.bioontology.org/ontologies/TESTINST/submissions/12") + instances = LinkedData::InstanceLoader.get_instances_by_ontology(submission_id, page_no: 1, size: 800) assert_equal 714, instances.length end - def test_instance_labels - submission_parse("TESTINST", "Testing instances", - "test/data/ontology_files/XCTontologyvtemp2_vvtemp2.zip", - 12, - masterFileName: "XCTontologyvtemp2/XCTontologyvtemp2.owl", - process_rdf: true, index_search: false, - run_metrics: false, reasoning: true) - submission_id = LinkedData::Models::OntologySubmission.all.first.id - class_id = RDF::URI.new("http://www.owl-ontologies.com/OntologyXCT.owl#ClinicalManifestation") + def test_instance_types + submission_id = RDF::URI.new("http://data.bioontology.org/ontologies/TESTINST/submissions/12") + class_id = RDF::URI.new('http://www.owl-ontologies.com/OntologyXCT.owl#ClinicalManifestation') - instances = LinkedData::InstanceLoader.get_instances_by_class(submission_id, class_id) + instances = LinkedData::InstanceLoader.get_instances_by_class(submission_id, class_id) instances.each do |inst| - assert (not inst.label.nil?) + assert (not inst.types.nil?) assert (not inst.id.nil?) end inst1 = instances.find {|inst| inst.id.to_s == 'http://www.owl-ontologies.com/OntologyXCT.owl#PresenceofAbnormalFacialShapeAt46'} - assert (not inst1.nil?) - assert_equal 'PresenceofAbnormalFacialShapeAt46', inst1.label + assert !inst1.nil? + assert_includes inst1.types, class_id inst2 = instances.find {|inst| inst.id.to_s == 'http://www.owl-ontologies.com/OntologyXCT.owl#PresenceofGaitDisturbanceAt50'} - assert (not inst2.nil?) - assert_equal 'PresenceofGaitDisturbanceAt50', inst2.label + assert !inst2.nil? + assert_includes inst2.types, class_id end def test_instance_properties known_properties = [PROP_TYPE, PROP_CLINICAL_MANIFESTATION, PROP_OBSERVABLE_TRAIT, PROP_HAS_OCCURRENCE] - submission_parse("TESTINST", "Testing instances", - "test/data/ontology_files/XCTontologyvtemp2_vvtemp2.zip", - 12, - masterFileName: "XCTontologyvtemp2/XCTontologyvtemp2.owl", - process_rdf: true, index_search: false, - run_metrics: false, reasoning: true) - submission_id = LinkedData::Models::OntologySubmission.all.first.id - class_id = RDF::URI.new("http://www.owl-ontologies.com/OntologyXCT.owl#ClinicalManifestation") + submission_id = RDF::URI.new("http://data.bioontology.org/ontologies/TESTINST/submissions/12") + class_id = RDF::URI.new('http://www.owl-ontologies.com/OntologyXCT.owl#ClinicalManifestation') instances = LinkedData::InstanceLoader.get_instances_by_class(submission_id, class_id) - inst = instances.find {|inst| inst.id.to_s == 'http://www.owl-ontologies.com/OntologyXCT.owl#PresenceofThyroidNoduleAt46'} + inst = instances.find {|inst| inst.id.to_s == 'http://www.owl-ontologies.com/OntologyXCT.owl#PresenceofThyroidNoduleAt46'} assert (not inst.nil?) assert_equal 4, inst.properties.length assert_equal known_properties.sort, inst.properties.keys.sort @@ -87,25 +68,26 @@ def test_instance_properties props = inst.properties known_types = [ - "http://www.owl-ontologies.com/OntologyXCT.owl#ClinicalManifestation", - "http://www.w3.org/2002/07/owl#NamedIndividual" + 'http://www.owl-ontologies.com/OntologyXCT.owl#ClinicalManifestation', + 'http://www.w3.org/2002/07/owl#NamedIndividual' ] + types = props[PROP_TYPE].map { |type| type.to_s } assert_equal 2, types.length assert_equal known_types.sort, types.sort manifestations = props[PROP_CLINICAL_MANIFESTATION] assert_equal 1, manifestations.length - assert_equal "http://www.owl-ontologies.com/OntologyXCT.owl#Patient_11_1", manifestations.first.to_s + assert_equal 'http://www.owl-ontologies.com/OntologyXCT.owl#Patient_11_1', manifestations.first.to_s observables = props[PROP_OBSERVABLE_TRAIT] assert_equal 1, observables.length - assert_equal "http://www.owl-ontologies.com/OntologyXCT.owl#PresenceofThyroidNodule", observables.first.to_s + assert_equal 'http://www.owl-ontologies.com/OntologyXCT.owl#PresenceofThyroidNodule', observables.first.to_s occurrences = props[PROP_HAS_OCCURRENCE] assert_equal 1, occurrences.length assert (occurrences.first.is_a? RDF::Literal) - assert_equal "46", occurrences.first.value + assert_equal '46', occurrences.first.value end end diff --git a/test/models/test_skos_submission.rb b/test/models/test_skos_submission.rb new file mode 100644 index 000000000..35bfeee29 --- /dev/null +++ b/test/models/test_skos_submission.rb @@ -0,0 +1,135 @@ +require_relative './test_ontology_common' +require 'logger' +require 'rack' + +class TestOntologySubmission < LinkedData::TestOntologyCommon + + def before_suite + submission_parse('SKOS-TEST', + 'SKOS TEST Bla', + './test/data/ontology_files/efo_gwas.skos.owl', 987, + process_rdf: true, index_search: false, + run_metrics: false, reasoning: true) + + sub = LinkedData::Models::OntologySubmission.where(ontology: [acronym: 'SKOS-TEST'], + submissionId: 987) + .first + sub.bring_remaining + sub.uri = RDF::URI.new('http://www.ebi.ac.uk/efo/skos/EFO_GWAS_view') + sub.save + sub + end + + def test_get_main_concept_scheme + sub = before_suite + assert_equal sub.uri, sub.get_main_concept_scheme.to_s + end + + def test_roots_no_main_scheme + + sub = before_suite + sub.uri = nil # no concept scheme as owl:ontology found + sub.save + assert_nil sub.get_main_concept_scheme + # if no main scheme found get all roots (topConcepts) + assert sub.roots.map { |x| x.id.to_s }.sort == ['http://www.ebi.ac.uk/efo/EFO_0000311', + 'http://www.ebi.ac.uk/efo/EFO_0001444', + 'http://www.ifomis.org/bfo/1.1/snap#Disposition', + 'http://www.ebi.ac.uk/chebi/searchId.do?chebiId=CHEBI:37577', + 'http://www.ebi.ac.uk/efo/EFO_0000635', + 'http://www.ebi.ac.uk/efo/EFO_0000324'].sort + roots = sub.roots + LinkedData::Models::Class.in(sub).models(roots).include(:children).all + roots.each do |root| + q_broader = <<-eos +SELECT ?children WHERE { + ?children #{RDF::SKOS[:broader].to_ntriples} #{root.id.to_ntriples} } + eos + children_query = [] + Goo.sparql_query_client.query(q_broader).each_solution do |sol| + children_query << sol[:children].to_s + end + assert root.children.map { |x| x.id.to_s }.sort == children_query.sort + end + end + + def test_roots_main_scheme + sub = before_suite + + roots = sub.roots + assert_equal 4, roots.size + roots.each do |r| + assert_equal r.isInActiveScheme, [sub.get_main_concept_scheme.to_s] + assert_equal r.isInActiveCollection, [] + end + roots = roots.map { |r| r.id.to_s } unless roots.nil? + refute_includes roots, 'http://www.ebi.ac.uk/efo/EFO_0000311' + refute_includes roots, 'http://www.ebi.ac.uk/efo/EFO_0000324' + end + + def test_roots_of_a_scheme + sub = before_suite + concept_schemes = ['http://www.ebi.ac.uk/efo/skos/EFO_GWAS_view_2'] + roots = sub.roots(concept_schemes: concept_schemes) + assert_equal 2, roots.size + roots.each do |r| + assert_includes r.inScheme, concept_schemes.first + assert_equal r.isInActiveScheme, concept_schemes + assert_equal r.isInActiveCollection, [] + end + roots = roots.map { |r| r.id.to_s } unless roots.nil? + assert_includes roots, 'http://www.ebi.ac.uk/efo/EFO_0000311' + assert_includes roots, 'http://www.ebi.ac.uk/efo/EFO_0000324' + end + + def test_roots_of_multiple_scheme + sub = before_suite + + concept_schemes = ['http://www.ebi.ac.uk/efo/skos/EFO_GWAS_view_2', + 'http://www.ebi.ac.uk/efo/skos/EFO_GWAS_view'] + roots = sub.roots(concept_schemes: concept_schemes) + assert_equal 6, roots.size + roots.each do |r| + selected_schemes = r.inScheme.select { |s| concept_schemes.include?(s) } + refute_empty selected_schemes + assert_equal r.isInActiveScheme, selected_schemes + assert_equal r.isInActiveCollection, [] + end + roots = roots.map { |r| r.id.to_s } unless roots.nil? + assert roots.sort == ['http://www.ebi.ac.uk/efo/EFO_0000311', + 'http://www.ebi.ac.uk/efo/EFO_0001444', + 'http://www.ifomis.org/bfo/1.1/snap#Disposition', + 'http://www.ebi.ac.uk/chebi/searchId.do?chebiId=CHEBI:37577', + 'http://www.ebi.ac.uk/efo/EFO_0000635', + 'http://www.ebi.ac.uk/efo/EFO_0000324'].sort + end + + def test_roots_of_scheme_collection + sub = before_suite + + concept_schemes = ['http://www.ebi.ac.uk/efo/skos/EFO_GWAS_view'] + concept_collection = ['http://www.ebi.ac.uk/efo/skos/collection_1'] + roots = sub.roots(concept_schemes: concept_schemes, concept_collections: concept_collection) + assert_equal 4, roots.size + + roots.each do |r| + assert_equal r.isInActiveCollection, concept_collection if r.memberOf.include?(concept_collection.first) + end + end + + def test_roots_of_scheme_collections + sub = before_suite + + concept_schemes = ['http://www.ebi.ac.uk/efo/skos/EFO_GWAS_view'] + concept_collection = ['http://www.ebi.ac.uk/efo/skos/collection_1', + 'http://www.ebi.ac.uk/efo/skos/collection_2'] + roots = sub.roots(concept_schemes: concept_schemes, concept_collections: concept_collection) + assert_equal 4, roots.size + + roots.each do |r| + selected_collections = r.memberOf.select { |c| concept_collection.include?(c)} + assert_equal r.isInActiveCollection, selected_collections unless selected_collections.empty? + end + end +end + diff --git a/test/util/test_notifications.rb b/test/util/test_notifications.rb index 9db820db7..e590fe9cc 100644 --- a/test/util/test_notifications.rb +++ b/test/util/test_notifications.rb @@ -1,39 +1,43 @@ -require_relative "../test_case" - -require "email_spec" -require "logger" +require_relative '../test_case' +require 'email_spec' +require 'logger' +require 'mocha/minitest' class TestNotifications < LinkedData::TestCase include EmailSpec::Helpers def self.before_suite - @@notifications_enabled = LinkedData.settings.enable_notifications - @@disable_override = LinkedData.settings.email_disable_override - @@old_support_mails = LinkedData.settings.ontoportal_admin_emails - if @@old_support_mails.nil? || @@old_support_mails.empty? - LinkedData.settings.ontoportal_admin_emails = ["ontoportal-support@mail.com"] - end + # Store original settings + @original_settings = { + notifications_enabled: LinkedData.settings.enable_notifications, + disable_override: LinkedData.settings.email_disable_override, + admin_emails: LinkedData.settings.ontoportal_admin_emails + } + LinkedData.settings.email_disable_override = true LinkedData.settings.enable_notifications = true + LinkedData.settings.ontoportal_admin_emails = ['ontoportal-support@mail.com'] + @@ui_name = LinkedData.settings.ui_name @@support_mails = LinkedData.settings.ontoportal_admin_emails - @@ont = LinkedData::SampleData::Ontology.create_ontologies_and_submissions(ont_count: 1, submission_count: 1)[2].first + @@ont = LinkedData::SampleData::Ontology.create_ontologies_and_submissions(ont_count: 1, + submission_count: 1)[2].first @@ont.bring_remaining @@user = @@ont.administeredBy.first - @@subscription = self.new("before_suite")._subscription(@@ont) + @@subscription = new('before_suite')._subscription(@@ont) @@user.bring_remaining @@user.subscription = [@@subscription] @@user.save end def self.after_suite - LinkedData.settings.enable_notifications = @@notifications_enabled - LinkedData.settings.email_disable_override = @@disable_override - LinkedData.settings.ontoportal_admin_emails = @@old_support_mails - @@ont.delete if defined?(@@ont) - @@subscription.delete if defined?(@@subscription) - @@user.delete if defined?(@@user) + # Restore original settings + LinkedData.settings.enable_notifications = @original_settings[:notifications_enabled] + LinkedData.settings.email_disable_override = @original_settings[:disable_override] + LinkedData.settings.ontoportal_admin_emails = @original_settings[:admin_emails] + + [@@ont, @@subscription, @@user].each(&:delete) end def setup @@ -44,14 +48,14 @@ def setup def _subscription(ont) subscription = LinkedData::Models::Users::Subscription.new subscription.ontology = ont - subscription.notification_type = LinkedData::Models::Users::NotificationType.find("ALL").first + subscription.notification_type = LinkedData::Models::Users::NotificationType.find('ALL').first subscription.save end def test_send_notification - recipients = ["test@example.org"] - subject = "Test subject" - body = "My test body" + recipients = ['test@example.org'] + subject = 'Test subject' + body = 'My test body' # Email recipient address will be overridden LinkedData.settings.email_disable_override = false @@ -60,11 +64,7 @@ def test_send_notification # Disable override LinkedData.settings.email_disable_override = true - LinkedData::Utils::Notifier.notify({ - recipients: recipients, - subject: subject, - body: body - }) + LinkedData::Utils::Notifier.notify({ recipients: recipients, subject: subject, body: body }) assert_equal recipients, last_email_sent.to assert_equal [LinkedData.settings.email_sender], last_email_sent.from assert_equal last_email_sent.body.raw_source, body @@ -72,9 +72,9 @@ def test_send_notification end def test_new_note_notification - recipients = ["test@example.org"] - subject = "Test note subject" - body = "Test note body" + recipients = ['test@example.org'] + subject = 'Test note subject' + body = 'Test note body' note = LinkedData::Models::Note.new note.creator = @@user note.subject = subject @@ -84,11 +84,11 @@ def test_new_note_notification assert_match "[#{@@ui_name} Notes]", last_email_sent.subject assert_equal [@@user.email], last_email_sent.to ensure - note.delete if note + note&.delete end def test_processing_complete_notification - options = { ont_count: 1, submission_count: 2, acronym: "NOTIFY" } + options = { ont_count: 1, submission_count: 2, acronym: 'NOTIFY' } ont = LinkedData::SampleData::Ontology.create_ontologies_and_submissions(options)[2].first subscription = _subscription(ont) @@user.subscription = @@user.subscription.dup << subscription @@ -101,27 +101,26 @@ def test_processing_complete_notification first_user = subscription.user.first first_user.bring :email - assert_match "Parsing Success", all_emails.first.subject + assert_match 'Parsing Success', all_emails.first.subject assert_equal [first_user.email], all_emails.first.to - assert_match ("Parsing Success"), all_emails.last.subject + assert_match 'Parsing Success', all_emails.last.subject assert_equal @@support_mails.uniq.sort, all_emails[1].to.sort assert_equal admin_mails.uniq.sort, all_emails.last.to.sort - reset_mailer - sub = ont.submissions.sort_by { |s| s.id}.first - sub.process_submission(Logger.new(TestLogFile.new), {archive: true}) + sub = ont.submissions.sort_by { |s| s.id }.first + sub.process_submission(Logger.new(TestLogFile.new), { archive: true }) assert_empty all_emails ensure - ont.delete if ont - subscription.delete if subscription + ont&.delete + subscription&.delete end def test_disable_administrative_notifications LinkedData.settings.enable_administrative_notifications = false - options = { ont_count: 1, submission_count: 1, acronym: "DONTNOTIFY" } + options = { ont_count: 1, submission_count: 1, acronym: 'DONTNOTIFY' } ont = LinkedData::SampleData::Ontology.create_ontologies_and_submissions(options)[2].first ont.latest_submission(status: :any).process_submission(Logger.new(TestLogFile.new)) admin_mails = LinkedData::Utils::Notifier.ontology_admin_emails(ont) @@ -129,22 +128,25 @@ def test_disable_administrative_notifications refute_match @@support_mails, last_email_sent.to.sort assert_equal admin_mails, last_email_sent.to.sort - assert_match ("Parsing Success"), all_emails.last.subject + assert_match 'Parsing Success', all_emails.last.subject LinkedData.settings.enable_administrative_notifications = true ensure - ont.delete if ont + ont&.delete end def test_remote_ontology_pull_notification - recipients = ["test@example.org"] - ont_count, acronyms, ontologies = LinkedData::SampleData::Ontology.create_ontologies_and_submissions(ont_count: 1, submission_count: 1, process_submission: false) + recipients = ['test@example.org'] + _ont_count, _acronyms, ontologies = LinkedData::SampleData::Ontology.create_ontologies_and_submissions( + ont_count: 1, submission_count: 1, process_submission: false + ) - ont = LinkedData::Models::Ontology.find(ontologies[0].id).include(:acronym, :administeredBy, :name, :submissions).first + ont = LinkedData::Models::Ontology.find(ontologies[0].id) + .include(:acronym, :administeredBy, :name, :submissions).first ont_admins = Array.new(3) { LinkedData::Models::User.new } ont_admins.each_with_index do |user, i| user.username = "Test User #{i}" user.email = "tester_#{i}@example.org" - user.password = "password" + user.password = 'password' user.save assert user.valid?, user.errors end @@ -164,7 +166,70 @@ def test_remote_ontology_pull_notification assert_equal admin_mails, last_email_sent.to.sort ensure ont_admins.each do |user| - user.delete if user + user&.delete + end + end + + def test_cloudflare_analytics_success_notification + start_time = Time.now - 3600 + end_time = Time.now + result_data = { + start_time: start_time, + end_time: end_time, + duration: 3600, + status: 'success', + error: nil + } + + LinkedData::Utils::Notifications.cloudflare_analytics(result_data) + + assert_equal 1, all_emails.size + assert_match 'success', last_email_sent.subject + assert_includes last_email_sent.body.raw_source, 'completed successfully' + assert_includes last_email_sent.body.raw_source, start_time.to_s + end + + def test_cloudflare_analytics_failure_notification + start_time = Time.now - 3600 + end_time = Time.now + result_data = { + start_time: start_time, + end_time: end_time, + duration: 3600, + status: 'error', + error: 'Connection timeout' + } + + LinkedData::Utils::Notifications.cloudflare_analytics(result_data) + + assert_equal 1, all_emails.size + assert_match 'error', last_email_sent.subject + assert_includes last_email_sent.body.raw_source, 'Connection timeout' + end + + def test_render_template + gem_path = '/fake/gem/path' + Gem.loaded_specs.stubs(:[]).with('ontologies_linked_data').returns( + stub(full_gem_path: gem_path) + ) + + template_content = 'Hello <%= name %>!' + File.expects(:read).with("#{gem_path}/views/emails/test.erb").returns(template_content) + + result = LinkedData::Utils::Notifications.render_template('test.erb', { name: 'World' }) + assert_equal 'Hello World!', result + end + + def test_render_template_file_not_found + gem_path = '/fake/gem/path' + Gem.loaded_specs.stubs(:[]).with('ontologies_linked_data').returns( + stub(full_gem_path: gem_path) + ) + + File.expects(:read).raises(Errno::ENOENT) + + assert_raises(RuntimeError, 'Template not found') do + LinkedData::Utils::Notifications.render_template('nonexistent.erb', {}) end end diff --git a/views/emails/cloudflare_analytics.erb b/views/emails/cloudflare_analytics.erb new file mode 100644 index 000000000..3cccc4943 --- /dev/null +++ b/views/emails/cloudflare_analytics.erb @@ -0,0 +1,16 @@ +<% if result_data[:status] == 'success' %> + Cloudflare Analytics job completed successfully

+ + Start time: <%= result_data[:start_time].utc %>
+ End time: <%= result_data[:end_time].utc %>
+ Duration: <%= result_data[:duration] %> seconds

+<% else %> + Cloudflare Analytics job failed!

+ + Start time: <%= result_data[:start_time].utc %>
+ End time: <%= result_data[:end_time].utc %>
+ Duration: <%= result_data[:duration] %> seconds

+ + Error: + 
<%= result_data[:error] %>


+<% end %> \ No newline at end of file