Merge pull request #183 from ncbo/develop

Merge branch 'develop'

Author: alexskr

.github/workflows/ruby-unit-tests.yml

@@ -12,7 +12,7 @@ jobs:
         backend: ['ruby', 'ruby-agraph'] # ruby runs tests with 4store backend and ruby-agraph runs with AllegroGraph backend
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up solr configsets
       run: ./test/solr/generate_ncbo_configsets.sh
     - name: create config.rb file

docker-compose.yml

@@ -59,9 +59,9 @@ services:
       GOO_BACKEND_NAME: ag
       GOO_PORT: 10035
       GOO_HOST: agraph-ut
-      GOO_PATH_QUERY: /repositories/bioportal_test
-      GOO_PATH_DATA: /repositories/bioportal_test/statements
-      GOO_PATH_UPDATE: /repositories/bioportal_test/statements
+      GOO_PATH_QUERY: /repositories/ontoportal_test
+      GOO_PATH_DATA: /repositories/ontoportal_test/statements
+      GOO_PATH_UPDATE: /repositories/ontoportal_test/statements
     profiles:
       - agraph
     depends_on:
@@ -92,7 +92,7 @@ services:
     image: solr:8
     volumes:
       - ./test/solr/configsets:/configsets:ro
-    #ports:
+    # ports:
     #  - "8983:8983"
     command:  ["solr-precreate", "term_search_core1", "/configsets/term_search"]
     healthcheck:
@@ -106,7 +106,7 @@ services:
     image: solr:8
     volumes:
       - ./test/solr/configsets:/configsets:ro
-    #ports:
+    # ports:
     #  - "8984:8983"
     command:  ["solr-precreate", "prop_search_core1", "/configsets/property_search"]
     healthcheck:
@@ -117,26 +117,26 @@ services:
       retries: 5
 
   agraph-ut:
-    image: franzinc/agraph:v8.0.0
+    image: franzinc/agraph:v8.0.1
     platform: linux/amd64
     environment:
       - AGRAPH_SUPER_USER=test
       - AGRAPH_SUPER_PASSWORD=xyzzy
     shm_size: 1g
-    # ports:
-    #  - 10035:10035
+      # ports:
+      #  - 10035:10035
     command: >
        bash -c "/agraph/bin/agraph-control --config /agraph/etc/agraph.cfg start
-       ; agtool repos create bioportal_test
+       ; agtool repos create --supersede ontoportal_test
        ; agtool users add anonymous
-       ; agtool users grant anonymous root:bioportal_test:rw
+       ; agtool users grant anonymous root:ontoportal_test:rw
        ; tail -f /agraph/data/agraph.log"
     healthcheck:
-      test: ["CMD-SHELL", "agtool storage-report bioportal_test || exit 1"]
-      start_period: 10s
-      interval: 60s
-      timeout: 5s
-      retries: 3
+      test: ["CMD-SHELL", "agtool storage-report ontoportal_test || exit 1"]
+      start_period: 30s #AllegroGraph can take a loooooong time to start
+      interval: 20s
+      timeout: 10s
+      retries: 20
     profiles:
       - agraph
 

lib/ontologies_linked_data/metrics/metrics.rb

@@ -35,7 +35,7 @@ def self.metrics_for_submission(submission, logger)
         logger.flush
 
         # re-generate metrics file
-        submission.generate_metrics_file2(cls_metrics[:classes], indiv_count, prop_count, cls_metrics[:maxDepth])
+        submission.generate_metrics_file(cls_metrics[:classes], indiv_count, prop_count, cls_metrics[:maxDepth])
         logger.info("generation of metrics file finished")
         logger.flush
 
@@ -54,10 +54,11 @@ def self.max_depth_fn(submission, logger, is_flat, rdfsSC)
       if (mx_from_file && mx_from_file.length == 2 && mx_from_file[0].length >= 4)
       then
         max_depth = mx_from_file[1][3].to_i
+        logger.info("Metrics max_depth retrieved #{max_depth} from the metrics csv file.")
       else
         logger.info("Unable to find metrics providing max_depth in file for submission #{submission.id.to_s}.  Using ruby calculation of max_depth.")  
         roots = submission.roots
-      
+
         unless is_flat
           depths = []
           roots.each do |root|

lib/ontologies_linked_data/models/ontology_submission.rb

@@ -418,41 +418,43 @@ def metrics_from_file(logger=nil)
         metrics
       end
 
-      def generate_metrics_file(class_count, indiv_count, prop_count)
-        CSV.open(self.metrics_path, "wb") do |csv|
-          csv << ["Class Count", "Individual Count", "Property Count"]
-          csv << [class_count, indiv_count, prop_count]
-        end
-      end
-
-      def generate_metrics_file2(class_count, indiv_count, prop_count, max_depth)
+      def generate_metrics_file(class_count, indiv_count, prop_count, max_depth)
         CSV.open(self.metrics_path, "wb") do |csv|
           csv << ["Class Count", "Individual Count", "Property Count", "Max Depth"]
           csv << [class_count, indiv_count, prop_count, max_depth]
         end
       end
 
       def generate_umls_metrics_file(tr_file_path=nil)
-        tr_file_path ||= self.triples_file_path
+        tr_file_path ||= triples_file_path
         class_count = 0
         indiv_count = 0
         prop_count = 0
+        max_depth = 0
 
         File.foreach(tr_file_path) do |line|
           class_count += 1 if line =~ /owl:Class/
           indiv_count += 1 if line =~ /owl:NamedIndividual/
           prop_count += 1 if line =~ /owl:ObjectProperty/
           prop_count += 1 if line =~ /owl:DatatypeProperty/
         end
-        self.generate_metrics_file(class_count, indiv_count, prop_count)
+
+        # Get max depth from the metrics.csv file which is already generated
+        # by owlapi_wrapper when new submission of UMLS ontology is created.
+        # Ruby code/sparql for calculating max_depth fails for large UMLS
+        # ontologie with AllegroGraph backend
+        metrics_from_owlapi = metrics_from_file
+        max_depth = metrics_from_owlapi[1][3] unless metrics_from_owlapi.empty?
+
+        generate_metrics_file(class_count, indiv_count, prop_count, max_depth)
       end
 
       def generate_rdf(logger, reasoning: true)
         mime_type = nil
 
         if self.hasOntologyLanguage.umls?
           triples_file_path = self.triples_file_path
-          logger.info("Using UMLS turtle file found, skipping OWLAPI parse")
+          logger.info("UMLS turtle file found; doing OWLAPI parse to extract metrics")
           logger.flush
           mime_type = LinkedData::MediaTypes.media_type_from_base(LinkedData::MediaTypes::TURTLE)
           generate_umls_metrics_file(triples_file_path)

run-unit-tests.sh

@@ -11,7 +11,8 @@ DC='docker compose'
 test/solr/generate_ncbo_configsets.sh
 
 # build docker containers
-$DC run --rm ruby bundle exec rake test TESTOPTS='-v'
+#$DC run --rm ruby bundle exec rake test TESTOPTS='-v'
 # run unit test with AG backend
-#$DC run --rm ruby-agraph bundle exec rake test TESTOPTS='-v'
-$DC stop
+$DC run --rm ruby-agraph bundle exec rake test TESTOPTS='-v'
+
+$DC --profile agraph --profile 4store stop

test/models/test_ontology_submission.rb

@@ -603,7 +603,7 @@ def test_umls_metrics_file
     metrics = sub.metrics_from_file(Logger.new(sub.parsing_log_path))
     assert !metrics.nil?, "Metrics is nil: #{metrics}"
     assert !metrics.empty?, "Metrics is empty: #{metrics}"
-    metrics.each { |m| assert_equal 3, m.length }
+    metrics.each { |m| assert_equal 4, m.length }
     assert_equal "Individual Count", metrics[0][1]
     assert_equal 133, metrics[1][0].to_i
   end