feat: add task to publish nb signalements on datagouv

.env.sample

@@ -36,3 +36,9 @@ PROCONNECT_ENDPOINT=
 # Api INSEE
 INSEE_API_URL=
 INSEE_API_KEY_INTEGRATION=
+
+# API DataGouv
+DATAGOUV_API_URL=
+DATAGOUV_API_KEY=
+DATAGOUV_DATASET_ID=
+DATAGOUV_RESOURCE_ID=

README.md

@@ -96,6 +96,10 @@ Elles peuvent être définies classiquement ou en créant un fichier `.env` sur
 | `PROCONNECT_ENDPOINT`         | Proconnect Endpoint                                                                                         |
 | `INSEE_API_URL`               | Url de l'API INSEE (Sirene)                                                                                 |
 | `INSEE_API_KEY_INTEGRATION`   | Clef de l'API INSEE (Sirene)                                                                                |
+| `DATAGOUV_API_URL`            | URL de l'API DataGouv                                                                                       |
+| `DATAGOUV_API_KEY`            | Clef de l'API DataGouv                                                                                      |
+| `DATAGOUV_DATASET_ID`         | Id du dataset sur DataGouv                                                                                  |
+| `DATAGOUV_RESOURCE_ID`        | Id de la ressource sur DataGouv                                                                             |
 
 ## Licence
 

package.json

@@ -68,7 +68,7 @@
     "@testcontainers/postgresql": "^10.10.0",
     "@types/express": "^4.17.17",
     "@types/geojson-vt": "^3.2.0",
-    "@types/jest": "^29.5.2",
+    "@types/jest": "^30.0.0",
     "@types/node": "^20.3.1",
     "@types/nodemailer": "^6.4.15",
     "@types/supertest": "^6.0.0",

src/modules/datagouv/datagouv.module.ts

@@ -0,0 +1,10 @@
+import { Module } from '@nestjs/common';
+import { ConfigModule } from '@nestjs/config';
+import { DataGouvService } from './datagouv.service';
+
+@Module({
+  imports: [ConfigModule],
+  providers: [DataGouvService],
+  exports: [DataGouvService],
+})
+export class DataGouvModule {}

src/modules/datagouv/datagouv.service.ts

@@ -0,0 +1,44 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import axios from 'axios';
+import FormData = require('form-data');
+
+@Injectable()
+export class DataGouvService {
+  private readonly logger = new Logger(DataGouvService.name);
+
+  constructor(private readonly configService: ConfigService) {}
+
+  async uploadCSVResource(
+    datasetId: string,
+    resourceId: string,
+    csvContent: string,
+    fileName: string,
+  ): Promise<void> {
+    const apiKey = this.configService.get('DATAGOUV_API_KEY');
+    const apiUrl =
+      this.configService.get('DATAGOUV_API_URL') ||
+      'https://www.data.gouv.fr/api/1';
+
+    const form = new FormData();
+    form.append('file', Buffer.from(csvContent, 'utf-8'), {
+      filename: fileName,
+      contentType: 'text/csv',
+    });
+
+    await axios.post(
+      `${apiUrl}/datasets/${datasetId}/resources/${resourceId}/upload/`,
+      form,
+      {
+        headers: {
+          ...form.getHeaders(),
+          'X-API-KEY': apiKey,
+        },
+      },
+    );
+
+    this.logger.log(
+      `CSV uploaded to data.gouv.fr (dataset: ${datasetId}, resource: ${resourceId})`,
+    );
+  }
+}

src/modules/signalement/signalement.service.ts

@@ -17,6 +17,8 @@ import {
   CreateReportDTO,
 } from '../../common/base-report.service';
 import { StatsDTO } from '../stats/stats.dto';
+import { ReportStatusEnum } from '../../common/report-status.enum';
+import { getCommune } from '../../utils/cog.utils';
 
 @Injectable()
 export class SignalementService extends BaseReportService<Signalement> {
@@ -144,4 +146,79 @@ export class SignalementService extends BaseReportService<Signalement> {
       count: Number(count),
     })) as { codeCommune: string; count: number }[];
   }
+
+  async getSignalementCountsByCommune(): Promise<
+    {
+      codeCommune: string;
+      nomCommune: string;
+      pending: number;
+      processed: number;
+      ignored: number;
+      expired: number;
+      total: number;
+    }[]
+  > {
+    const qb = this.repository.createQueryBuilder('signalement');
+
+    const rawResults: {
+      codeCommune: string;
+      status: string;
+      count: string;
+    }[] = await qb
+      .select('signalement.code_commune', 'codeCommune')
+      .addSelect('signalement.status', 'status')
+      .addSelect('COUNT(signalement.id)', 'count')
+      .groupBy('signalement.code_commune')
+      .addGroupBy('signalement.status')
+      .getRawMany();
+
+    const communeMap = new Map<
+      string,
+      {
+        pending: number;
+        processed: number;
+        ignored: number;
+        expired: number;
+        total: number;
+      }
+    >();
+
+    for (const row of rawResults) {
+      if (!communeMap.has(row.codeCommune)) {
+        communeMap.set(row.codeCommune, {
+          pending: 0,
+          processed: 0,
+          ignored: 0,
+          expired: 0,
+          total: 0,
+        });
+      }
+      const entry = communeMap.get(row.codeCommune);
+      const count = Number(row.count);
+
+      switch (row.status) {
+        case ReportStatusEnum.PENDING:
+          entry.pending = count;
+          break;
+        case ReportStatusEnum.PROCESSED:
+          entry.processed = count;
+          break;
+        case ReportStatusEnum.IGNORED:
+          entry.ignored = count;
+          break;
+        case ReportStatusEnum.EXPIRED:
+          entry.expired = count;
+          break;
+      }
+      entry.total += count;
+    }
+
+    return Array.from(communeMap.entries())
+      .map(([codeCommune, counts]) => ({
+        codeCommune,
+        nomCommune: getCommune(codeCommune)?.nom || codeCommune,
+        ...counts,
+      }))
+      .sort((a, b) => b.total - a.total);
+  }
 }

src/modules/task/task.module.ts

@@ -2,12 +2,14 @@ import { Module, forwardRef } from '@nestjs/common';
 import { SignalementModule } from '../signalement/signalement.module';
 import { TaskService } from './task.service';
 import { MesAdressesAPIModule } from '../mes-adresses-api/mes-adresses-api.module';
+import { DataGouvModule } from '../datagouv/datagouv.module';
 import { ConfigModule } from '@nestjs/config';
 
 @Module({
   imports: [
     forwardRef(() => SignalementModule),
     MesAdressesAPIModule,
+    DataGouvModule,
     ConfigModule,
   ],
   providers: [TaskService],

src/modules/task/task.service.ts

@@ -2,6 +2,7 @@ import { Injectable, Logger } from '@nestjs/common';
 import { Cron, CronExpression } from '@nestjs/schedule';
 import { SignalementService } from '../signalement/signalement.service';
 import { MesAdressesAPIService } from '../mes-adresses-api/mes-adresses-api.service';
+import { DataGouvService } from '../datagouv/datagouv.service';
 import { MailerService } from '@nestjs-modules/mailer';
 import { ConfigService } from '@nestjs/config';
 import { getCommune } from '../../utils/cog.utils';
@@ -13,6 +14,7 @@ export class TaskService {
   constructor(
     private readonly signalementService: SignalementService,
     private readonly mesAdressesAPIService: MesAdressesAPIService,
+    private readonly dataGouvService: DataGouvService,
     private readonly mailerService: MailerService,
     private configService: ConfigService,
   ) {}
@@ -98,4 +100,40 @@ export class TaskService {
         ' signalements',
     );
   }
+
+  // Cron job that runs every Tuesday at 17:00 PM
+  @Cron('0 17 * * 2')
+  async weeklyDataGouvCSVExport() {
+    const datasetId = this.configService.get('DATAGOUV_DATASET_ID');
+    const resourceId = this.configService.get('DATAGOUV_RESOURCE_ID');
+
+    if (!datasetId || !resourceId) {
+      this.logger.warn(
+        'Skipping weeklyDataGouvCSVExport: DATAGOUV_DATASET_ID or DATAGOUV_RESOURCE_ID not configured',
+      );
+      return;
+    }
+
+    this.logger.log('Start task : weeklyDataGouvCSVExport');
+
+    const countsByCommune =
+      await this.signalementService.getSignalementCountsByCommune();
+
+    const header =
+      'code_insee,nom_commune,nb_signalements_en_attente,nb_signalements_traites,nb_signalements_ignores,nb_signalements_expires,nb_signalements_total';
+    const rows = countsByCommune.map(
+      (row) =>
+        `${row.codeCommune},${row.nomCommune.includes(',') ? `"${row.nomCommune}"` : row.nomCommune},${row.pending},${row.processed},${row.ignored},${row.expired},${row.total}`,
+    );
+    const csvContent = [header, ...rows].join('\n');
+
+    await this.dataGouvService.uploadCSVResource(
+      datasetId,
+      resourceId,
+      csvContent,
+      'signalements-par-commune.csv',
+    );
+
+    this.logger.log('End task : weeklyDataGouvCSVExport');
+  }
 }

src/tests/proconnect.spec.ts

@@ -31,6 +31,8 @@ const MES_ADRESSES_URL = 'http://localhost:8888';
 
 jest.setTimeout(60000);
 
+jest.spyOn(console, 'error').mockImplementation(() => undefined);
+
 describe('ProConnect module', () => {
   let app: INestApplication;
   let postgresContainer: StartedPostgreSqlContainer;

src/tests/settings.spec.ts

@@ -20,6 +20,9 @@ import {
   EnabledListKeys,
   SignalementSubmissionMode,
 } from '../modules/setting/setting.type';
+import { Logger } from '@nestjs/common';
+
+Logger.overrideLogger(false);
 
 const currentRevisionMock = jest.fn();
 
@@ -34,6 +37,7 @@ const testSource = new Source({
       provide: ApiDepotService,
       useValue: {
         getCurrentRevision: currentRevisionMock,
+        getAllCurrentRevisions: jest.fn().mockResolvedValue([]),
       },
     },
   ],