Feature group routing for submissions (#2393)

* backend & frontend OK / TODO: site worker and leaderboad

* site worker sending submissions to group queue OK

* leaderboad group feature

* logs removed

* fix json leaderboard

* Clean up leaderboard ordering logic

* competition queue on groups with out queue

* some bugfix

* UI bugfix

* UI bugfix

* leaderboard groups format parentsubID_groupname

* fix conflicts issues

* resolve conflict

* clean site worker

* branch update and linter fix

* linter fix

* linter fix

* linter fix

* bugfix group form

* adding migrations files

* fix logic to fix tests problem

* fix logic to fix tests problem

* E2E test fixed

* E2E test fixed

* E2E test fixed

* E2E test fixed

* Fix queue name in server status

* fix queues visibility

* fix queue visibility for groups

* Flake8

---------

Co-authored-by: didayolo <adrien.pavao@gmail.com>

Author: IdirLISN

package.json

@@ -3,12 +3,12 @@
   "version": "0.0.1",
   "dependencies": {
     "jquery": "^4.0.0",
+    "js-beautify": "^1.15.4",
     "npm-watch": "^0.13.0",
     "riot": "^3.13.2",
     "stylus": "^0.64.0",
     "uglify-js": "^3.19.3"
   },
-  "devDependencies": {},
   "watch": {
     "build-stylus": {
       "patterns": [

src/apps/api/serializers/leaderboards.py

@@ -1,11 +1,11 @@
-from django.db.models import Sum, Q, F
+from django.db.models import Prefetch, Sum, Q, F
 from drf_writable_nested import WritableNestedModelSerializer
 from rest_framework import serializers
 
 from api.serializers.submission_leaderboard import SubmissionLeaderBoardSerializer
 
 from competitions.models import Submission, Phase
-from leaderboards.models import Leaderboard, Column
+from leaderboards.models import Leaderboard, Column, SubmissionScore
 
 from .fields import CharacterSeparatedField
 from .tasks import PhaseTaskInstanceSerializer
@@ -98,38 +98,56 @@ class Meta:
         )
 
     def get_submissions(self, instance):
-        # desc == -colname
-        # asc == colname
         primary_col = instance.columns.get(index=instance.primary_index)
-        # Order first by primary column. Then order by other columns after for tie breakers.
+
         ordering = [
             F('primary_col').desc(nulls_last=True)
             if primary_col.sorting == 'desc'
             else F('primary_col').asc(nulls_last=True)
         ]
-        submissions = (
+
+        submissions_qs = (
             Submission.objects.filter(
                 leaderboard=instance,
                 is_specific_task_re_run=False
             )
-            .select_related('owner')
-            .prefetch_related('scores')
-            .annotate(primary_col=Sum('scores__score', filter=Q(scores__column=primary_col)))
+            .select_related(
+                'owner',
+                'organization',
+                'queue',
+                'parent',
+                'phase',
+                'phase__competition',
+                'phase__competition__queue',
+            )
+            .prefetch_related(
+                Prefetch(
+                    'scores',
+                    queryset=SubmissionScore.objects.select_related(
+                        'column',
+                        'column__leaderboard',
+                    ),
+                )
+            )
+            .annotate(primary_col=Sum(
+                'scores__score',
+                filter=Q(scores__column=primary_col)
+            ))
         )
+
         for column in instance.columns.exclude(id=primary_col.id).order_by('index'):
             col_name = f'col{column.index}'
             ordering.append(
                 F(col_name).desc(nulls_last=True)
                 if column.sorting == 'desc'
                 else F(col_name).asc(nulls_last=True)
             )
-            kwargs = {
+            submissions_qs = submissions_qs.annotate(**{
                 col_name: Sum('scores__score', filter=Q(scores__column__index=column.index))
-            }
-            submissions = submissions.annotate(**kwargs)
+            })
 
-        submissions = submissions.order_by(*ordering, 'created_when')
-        return SubmissionLeaderBoardSerializer(submissions, many=True).data
+        submissions_qs = submissions_qs.order_by(*ordering, 'created_when')
+        return SubmissionLeaderBoardSerializer(submissions_qs, many=True).data
 
 
 class LeaderboardPhaseSerializer(serializers.ModelSerializer):

src/apps/api/serializers/submission_leaderboard.py

@@ -26,11 +26,43 @@ class SubmissionLeaderBoardSerializer(serializers.ModelSerializer):
     slug_url = serializers.CharField(source='owner.slug_url')
     organization = SimpleOrganizationSerializer(allow_null=True)
     created_when = serializers.DateTimeField()
+    queue_id = serializers.SerializerMethodField()
+    queue_name = serializers.SerializerMethodField()
+
+    def _get_effective_queue(self, obj):
+        if obj.queue:
+            return obj.queue
+
+        if obj.parent and obj.parent.queue:
+            return obj.parent.queue
+
+        if obj.phase and obj.phase.competition and obj.phase.competition.queue:
+            return obj.phase.competition.queue
+
+        return None
+
+    def _get_display_queue_name(self, obj):
+        queue = self._get_effective_queue(obj)
+        if not queue:
+            return None
+
+        raw_name = queue.name or ""
+        group_name = raw_name.rsplit("__", 1)[-1]  # comp10__CLB -> CLB, APHP -> APHP
+        submission_parent_id = obj.parent_id or obj.id
+
+        return f"{submission_parent_id}_{group_name}"
+
+    def get_queue_name(self, obj):
+        return self._get_display_queue_name(obj)
+
+    def get_queue_id(self, obj):
+        queue = self._get_effective_queue(obj)
+        return queue.id if queue else None
 
     class Meta:
         model = Submission
         fields = (
             'id', 'parent', 'owner', 'leaderboard_id', 'fact_sheet_answers',
             'task', 'scores', 'display_name', 'slug_url', 'organization',
-            'detailed_result', 'created_when'
+            'detailed_result', 'created_when', 'queue_name', 'queue_id',
         )

src/apps/api/views/competitions.py

@@ -1,7 +1,7 @@
 import zipfile
 import json
 import csv
-from collections import OrderedDict
+from collections import Counter, OrderedDict
 from io import StringIO
 from django.http import HttpResponse
 from tempfile import SpooledTemporaryFile
@@ -424,26 +424,43 @@ def collect_leaderboard_data(self, competition, phase_pk=None):
             phase_id = phases[0].id
 
         leaderboard = Leaderboard.objects.prefetch_related('columns').get(phases=phase_id)
-        leaderboard_titles = {phase['id']: f'{leaderboard.title} - {phase["name"]}({phase["id"]})' for phase in submission_query}
+        leaderboard_titles = {
+            phase['id']: f'{leaderboard.title} - {phase["name"]}({phase["id"]})'
+            for phase in submission_query
+        }
         leaderboard_data = {title: {} for title in leaderboard_titles.values()}
 
         for phase in submission_query:
             generated_columns = OrderedDict()
             for task in phase['tasks']:
                 for col in leaderboard.columns.all():
-                    generated_columns.update({f'{col.key}-{task["id"]}': f'{task["name"]}({task["id"]})-{col.title}'})
+                    generated_columns.update({
+                        f'{col.key}-{task["id"]}': f'{task["name"]}({task["id"]})-{col.title}'
+                    })
+
             for submission in phase['submissions']:
-                submission_key = f'{submission["owner"]}-{submission["parent"] or submission["id"]}'
-                if submission_key not in leaderboard_data[leaderboard_titles[phase['id']]].keys():
-                    leaderboard_data[leaderboard_titles[phase['id']]].update({submission_key: OrderedDict()})
-                    if 'fact_sheet_answers' in submission.keys() and submission['fact_sheet_answers']:
-                        leaderboard_data[leaderboard_titles[phase['id']]][submission_key]\
-                            .update({'fact_sheet_answers': submission['fact_sheet_answers']})
+                queue_name = submission.get('queue_name') or ''
+                submission_key = f'{submission["owner"]}-{submission["id"]}'
+                if queue_name:
+                    submission_key = f'{submission_key}-{queue_name}'
+
+                if submission_key not in leaderboard_data[leaderboard_titles[phase['id']]]:
+                    leaderboard_data[leaderboard_titles[phase['id']]][submission_key] = OrderedDict()
+
+                    if submission.get('fact_sheet_answers'):
+                        leaderboard_data[leaderboard_titles[phase['id']]][submission_key].update({
+                            'fact_sheet_answers': submission['fact_sheet_answers']
+                        })
+
                     for col_title in generated_columns.values():
                         leaderboard_data[leaderboard_titles[phase['id']]][submission_key].update({col_title: ""})
+
                 for score in submission['scores']:
                     score_column = generated_columns[f'{score["column_key"]}-{submission["task"]}']
-                    leaderboard_data[leaderboard_titles[phase['id']]][submission_key].update({score_column: score['score']})
+                    leaderboard_data[leaderboard_titles[phase['id']]][submission_key].update({
+                        score_column: score['score']
+                    })
+
         return leaderboard_data
 
     @action(detail=True, methods=['GET'], renderer_classes=[JSONRenderer, CSVRenderer, ZipRenderer])
@@ -773,6 +790,23 @@ def rerun_submissions(self, request, pk):
     @action(detail=True, methods=['GET'], permission_classes=[AllowAny])
     def get_leaderboard(self, request, pk):
         phase = self.get_object()
+
+        def _clean_group_label(raw_name, submission_parent_id=None):
+            if not raw_name:
+                return None
+
+            label = str(raw_name)
+
+            if submission_parent_id is not None:
+                prefix = f"{submission_parent_id}_"
+                if label.startswith(prefix):
+                    label = label[len(prefix):]
+
+            if "__" in label:
+                label = label.rsplit("__", 1)[1]
+
+            return label or None
+
         if phase.competition.fact_sheet:
             fact_sheet_keys = [
                 (
@@ -792,21 +826,73 @@ def get_leaderboard(self, request, pk):
             'submissions': [],
             'tasks': [],
             'fact_sheet_keys': fact_sheet_keys or None,
-            'primary_index': query['leaderboard']['primary_index']
+            'primary_index': query['leaderboard']['primary_index'],
+            'has_group_queues': False,
         }
 
-        columns = [col for col in query['columns']]
+        columns = list(query['columns'])
         submissions_keys = {}
         submission_detailed_results = {}
 
+        group_name_by_user_queue = {}
+        for group in phase.competition.participant_groups.filter(
+            queue__isnull=False
+        ).select_related('queue').prefetch_related('user_set'):
+            cleaned_group_name = _clean_group_label(group.name)
+            for user in group.user_set.all():
+                group_name_by_user_queue[(user.username, group.queue_id)] = cleaned_group_name
+
+        parent_ids = {
+            s['parent']
+            for s in query['submissions']
+            if s['parent'] is not None
+        }
+        parent_task_counts = Counter(
+            (s['parent'], s['task'])
+            for s in query['submissions']
+            if s['parent'] is not None
+        )
+
         for submission in query['submissions']:
-            submission_key = f"{submission['owner']}{submission['parent'] or submission['id']}"
+            if submission['id'] in parent_ids:
+                continue
+
+            submission_parent_id = submission.get('parent') or submission.get('id')
+            raw_queue_name = submission.get('queue_name') or ''
+            queue_id = submission.get('queue_id')
+
+            group_name = group_name_by_user_queue.get(
+                (submission['owner'], queue_id)
+            ) if queue_id else None
+
+            group_label = _clean_group_label(
+                group_name or raw_queue_name,
+                submission_parent_id=submission_parent_id
+            )
+
+            display_group = (
+                f"{submission_parent_id}_{group_label}"
+                if group_label
+                else None
+            )
+
+            parent_id = submission['parent']
+            task_id = submission.get('task')
+
+            # Cas particulier: plusieurs submissions d'un même parent sans queue explicite
+            is_multi_group_null_queue = (
+                parent_id is not None
+                and not queue_id
+                and parent_task_counts.get((parent_id, task_id), 0) > 1
+            )
+
+            if is_multi_group_null_queue:
+                submission_key = f"{submission['owner']}{parent_id}_{submission['id']}"
+            else:
+                submission_key = f"{submission['owner']}{submission_parent_id}_{group_label or ''}"
+
             # gather detailed result from submissions for each task
-            # detailed_results are gathered based on submission key
-            # `id` is used to fetch the right detailed result in detailed results page
-            # `detailed_result` url is not needed
             submission_detailed_results.setdefault(submission_key, []).append({
-                # 'detailed_result': submission['detailed_result'],
                 'task': submission['task'],
                 'id': submission['id']
             })
@@ -821,23 +907,18 @@ def get_leaderboard(self, request, pk):
                     'fact_sheet_answers': submission['fact_sheet_answers'],
                     'slug_url': submission['slug_url'],
                     'organization': submission['organization'],
-                    'created_when': submission['created_when']
+                    'created_when': submission['created_when'],
+                    'queue_name': display_group,
                 })
 
-            for score in submission['scores']:
+                if queue_id or is_multi_group_null_queue:
+                    response['has_group_queues'] = True
 
-                # to check if a column is found
-                # this is useful because of `hidden` field
-                # if a column is hidden it will not be shown here so
-                # we will not return that score to the front-end
+            for score in submission['scores']:
                 column_found = False
-                # default precision is set to 2
                 precision = 2
-                # default hidden is set to false
                 hidden = False
 
-                # loop over columns to find a column with the same index
-                # replace default precision with column precision
                 for col in columns:
                     if col["index"] == score["index"]:
                         precision = col["precision"]
@@ -847,13 +928,8 @@ def get_leaderboard(self, request, pk):
 
                 tempScore = score
                 tempScore['task_id'] = submission['task']
-                # round the score to 'precision' decimal points
                 tempScore['score'] = str(round(float(tempScore["score"]), precision))
 
-                # only add scores to the scores list
-                # if this column is found
-                # and
-                # column is not hidden
                 if column_found and not hidden:
                     response['submissions'][submissions_keys[submission_key]]['scores'].append(tempScore)
 
@@ -877,7 +953,6 @@ def get_leaderboard(self, request, pk):
         # --- end pagination addition ---
 
         for task in query['tasks']:
-            # This can be used to rendered variable columns on each task
             tempTask = {
                 'name': task['name'],
                 'id': task['id'],