feat: improve consistency check and dry-run

.gitignore

@@ -15,4 +15,6 @@
 .env
 dist
 test-coverage
-node_modules
+node_modules
+el-stats-par-anciennete.json
+el-stats.json

.gitlab-ci.yml

@@ -7,7 +7,7 @@ before_script:
 update-statistics:
   script:
     - echo "NOTION_TOKEN=$NOTION_TOKEN" > .env
-    - yarn run start
+    - yarn run publish-stats
   rules:
     # This ensures that only pushes to the default branch will trigger
     # a pages deploy

README.md

@@ -5,4 +5,4 @@ This modules computes EL statistiques from Notion Data
 # Setup
 
 - Create file `.env` defining the NOTION_TOKEN variable
-- Run `yarn install &&  yarn run start`
+- Run `yarn dry-run` or `yarn publish-stats`

package.json

@@ -23,7 +23,8 @@
     "typescript-eslint": "^7.11.0"
   },
   "scripts": {
-    "start": "node -r ts-node/register --env-file=.env src/index.ts",
+    "publish-stats": "node -r ts-node/register --env-file=.env src/index.ts",
+    "dry-run": "node -r ts-node/register --env-file=.env src/index.ts --dry-run",
     "test": "jest"
   }
 }

src/data/EvenementFamille.ts

@@ -53,6 +53,7 @@ const categorieEvenement: {
   ["Tribunal de police judiciaire"]: "Procédure Pénale",
   ["Mise en demeure de scolarisation"]: "Procédure Pénale",
   ["Audition gendarmerie / police"]: "Procédure Pénale",
+  ["Gendarmerie/Forces de l'ordre"]: "Procédure Pénale",
   ["Passage police municipale"]: "Procédure Pénale",
   ["Convocation procureur"]: "Procédure Pénale",
   ["Audition procureur"]: "Procédure Pénale",
@@ -71,6 +72,8 @@ const categorieEvenement: {
   ["Refus de contrôle"]: "Autre",
   ["Administrateur AD'HOC"]: "Autre",
   ["Signalement"]: "Autre",
+  ["contrôle URSSAF"]: "Autre",
+  ["contrôle fiscal"]: "Autre",
 };
 
 export type CategorieEvenement =

src/data/TypeEvenement.ts

@@ -28,4 +28,7 @@ export type TypeEvenement =
   | "Rappel à la loi"
   | "Passage police municipale"
   | "Administrateur AD'HOC"
-  | "Validation désobéissance";
+  | "Validation désobéissance"
+  | "contrôle URSSAF"
+  | "contrôle fiscal"
+  | "Gendarmerie/Forces de l'ordre";

src/data/checkDataConsistency.ts

@@ -1,54 +1,82 @@
 import { isValidEvenementFamille } from "./EvenementFamille";
-import { Famille } from "./Famille";
+import { Famille, isExResistant, isResistant } from "./Famille";
 
-export function checkDataConsistency(families: Famille[]): ConsistencyIssue[] {
-  return families.flatMap((family) => {
+export function checkDataConsistency(families: Famille[]): ConsistencyReport {
+  const reports = families.map((family) => {
     return checkFamilyDataConsistency(family);
   });
+  return {
+    errors: reports.flatMap((r) => r.errors),
+    warnings: reports.flatMap((r) => r.warnings),
+  };
 }
 
+export type ConsistencyReport = {
+  warnings: ConsistencyIssue[];
+  errors: ConsistencyIssue[];
+};
 export type ConsistencyIssue = {
   issueType: string;
   familyId: string;
-  canIgnore?: boolean;
 };
-function checkFamilyDataConsistency(family: Famille) {
-  const consistencyIssues: ConsistencyIssue[] = [];
+function checkFamilyDataConsistency(family: Famille): ConsistencyReport {
+  const consistencyErrors: ConsistencyIssue[] = [];
+  const consistencyWarnings: ConsistencyIssue[] = [];
 
   if (family.Statut === "Résistant.e") {
     if (family.Integration === null) {
-      consistencyIssues.push({
+      consistencyErrors.push({
         familyId: family.Titre,
         issueType: "Résistant.e without startResistant",
       });
     }
     if (family.Sortie !== null) {
-      consistencyIssues.push({
+      consistencyErrors.push({
         familyId: family.Titre,
         issueType: "Résistant.e with endResistant!!",
       });
     }
   } else if (family.Statut === "Ex résistant·e·s") {
     if (family.Integration === null) {
-      consistencyIssues.push({
+      consistencyErrors.push({
         familyId: family.Titre,
         issueType: "Ex résistant.e.s without startResistant",
       });
     }
     if (family.Sortie === null) {
-      consistencyIssues.push({
+      consistencyErrors.push({
         familyId: family.Titre,
         issueType: "Ex résistant.e.s without endResistant",
       });
     }
     if (family.Integration! > family.Sortie!) {
-      consistencyIssues.push({
+      consistencyErrors.push({
         familyId: family.Titre,
         issueType: "startResistsant > endResistant ",
       });
     }
   }
-  consistencyIssues.push(
+  if (
+    (isResistant(family) || isExResistant(family)) &&
+    family.Integration !== null
+  ) {
+    const miseEnDemeureBeforeInteg =
+      family.Evenements.find(
+        (e) =>
+          e.Type === "Mise en demeure de scolarisation" &&
+          (e.Date === null || e.Date < family.Integration!)
+      ) !== undefined;
+    if (
+      miseEnDemeureBeforeInteg &&
+      family.ContexteEntree !== "Après mise en demeure"
+    ) {
+      consistencyWarnings.push({
+        familyId: family.Titre,
+        issueType: `ContextEntree incorrect`,
+      });
+    }
+  }
+  consistencyErrors.push(
     ...family.Evenements.filter((e) => !isValidEvenementFamille(e.Type)).map(
       (e) => ({
         familyId: family.Titre,
@@ -56,14 +84,16 @@ function checkFamilyDataConsistency(family: Famille) {
       })
     )
   );
-  consistencyIssues.push(
+  consistencyWarnings.push(
     ...family.Evenements.filter((e) => e.Type !== null && e.Date === null).map(
       (e) => ({
         familyId: family.Titre,
         issueType: `Event ${e.notionId} with non null Type "${e.Type}" but null Date`,
-        canIgnore: true,
       })
     )
   );
-  return consistencyIssues;
+  return {
+    errors: consistencyErrors,
+    warnings: consistencyWarnings,
+  };
 }

src/index.ts

@@ -1,40 +1,71 @@
 import { Client } from "@notionhq/client";
+import { writeFileSync } from "fs";
 import { checkDataConsistency } from "./data/checkDataConsistency";
 import { fetchFamiliesWithEventsFromNotion } from "./notion/fetch/fetchFamiliesWithEventsFromNotion";
 import { publishStatisticsToNotion } from "./notion/publish/publishStatisticsToNotion";
 import { computeELStats } from "./statistiques/computeELStats";
 
-(async function IIFE() {
+type ProcessOptions = {
+  dryRun: boolean;
+  notionApiToken: string;
+};
+
+/**
+ * Build options from process args and env
+ */
+function buildProcessOptions(): ProcessOptions {
   const notionApiToken = process.env.NOTION_TOKEN;
   if (!notionApiToken) {
     throw new Error("process.env.NOTION_TOKEN not defined");
   }
+  const dryRun = process.argv.length >= 2 && process.argv[2] === "--dry-run";
+  return {
+    dryRun: dryRun,
+    notionApiToken: notionApiToken,
+  };
+}
+
+(async function IIFE() {
+  const options = buildProcessOptions();
 
   const notionClient = new Client({
-    auth: notionApiToken,
+    auth: options.notionApiToken,
   });
 
   const doFetch = true;
   console.log("Fetching families...");
-  const families = doFetch
+  const familles = doFetch
     ? await fetchFamiliesWithEventsFromNotion(notionClient)
     : [];
 
   console.log("Checking Data Consistency issues...");
-  const consistencyIssues = checkDataConsistency(families);
-  if (consistencyIssues.length > 0) {
-    console.error("Found consistency issues:");
-    console.error(consistencyIssues);
-    if (consistencyIssues.find((issue) => !issue.canIgnore)) {
-      process.exit(1);
-    }
+  const consistencyReport = checkDataConsistency(familles);
+  if (consistencyReport.errors.length > 0) {
+    console.error(
+      `Found ${consistencyReport.errors.length} consistency error(s) that prevent building stats:`
+    );
+    console.error(consistencyReport.errors);
+    process.exit(1);
+  }
+  if (consistencyReport.warnings.length > 0) {
+    console.warn(
+      `Found ${consistencyReport.warnings.length} non blocking consistency warning(s):`
+    );
+    console.warn(consistencyReport.warnings);
   }
 
   const currentDate = new Date(Date.now());
 
   console.log("Building statistics...");
-  const resistantCountStats = computeELStats(families, currentDate);
+  const elStats = computeELStats(familles, currentDate);
 
-  console.log("Publishing statistics...");
-  publishStatisticsToNotion(resistantCountStats, currentDate, notionClient);
+  if (options.dryRun) {
+    console.log(
+      "Dry run => Skip Publishing. Stats are dumped in file el-stats.json"
+    );
+    writeFileSync("./el-stats.json", JSON.stringify(elStats, null, "  "));
+  } else {
+    console.log("Publishing statistics...");
+    publishStatisticsToNotion(elStats, currentDate, notionClient);
+  }
 })();