From f3764a98f5b652ef3f8b32a472fff5d469764ab5 Mon Sep 17 00:00:00 2001 From: "sebastien.arod@gmail.com" Date: Wed, 31 Jul 2024 14:41:22 +0200 Subject: [PATCH] feat: improve consistency check and dry-run --- .gitignore | 4 ++- .gitlab-ci.yml | 2 +- README.md | 2 +- package.json | 3 +- src/data/EvenementFamille.ts | 3 ++ src/data/TypeEvenement.ts | 5 ++- src/data/checkDataConsistency.ts | 60 ++++++++++++++++++++++++-------- src/index.ts | 57 +++++++++++++++++++++++------- 8 files changed, 103 insertions(+), 33 deletions(-) diff --git a/.gitignore b/.gitignore index 5ab8c1c..1d75daf 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,6 @@ .env dist test-coverage -node_modules \ No newline at end of file +node_modules +el-stats-par-anciennete.json +el-stats.json diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6f07947..dded30d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,7 +7,7 @@ before_script: update-statistics: script: - echo "NOTION_TOKEN=$NOTION_TOKEN" > .env - - yarn run start + - yarn run publish-stats rules: # This ensures that only pushes to the default branch will trigger # a pages deploy diff --git a/README.md b/README.md index 9f2eb8a..d922ac6 100644 --- a/README.md +++ b/README.md @@ -5,4 +5,4 @@ This modules computes EL statistiques from Notion Data # Setup - Create file `.env` defining the NOTION_TOKEN variable -- Run `yarn install && yarn run start` +- Run `yarn dry-run` or `yarn publish-stats` diff --git a/package.json b/package.json index 971af95..fd7ae12 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,8 @@ "typescript-eslint": "^7.11.0" }, "scripts": { - "start": "node -r ts-node/register --env-file=.env src/index.ts", + "publish-stats": "node -r ts-node/register --env-file=.env src/index.ts", + "dry-run": "node -r ts-node/register --env-file=.env src/index.ts --dry-run", "test": "jest" } } diff --git a/src/data/EvenementFamille.ts b/src/data/EvenementFamille.ts index 1b278be..93cc133 100644 --- a/src/data/EvenementFamille.ts +++ b/src/data/EvenementFamille.ts @@ -53,6 +53,7 @@ const categorieEvenement: { ["Tribunal de police judiciaire"]: "Procédure Pénale", ["Mise en demeure de scolarisation"]: "Procédure Pénale", ["Audition gendarmerie / police"]: "Procédure Pénale", + ["Gendarmerie/Forces de l'ordre"]: "Procédure Pénale", ["Passage police municipale"]: "Procédure Pénale", ["Convocation procureur"]: "Procédure Pénale", ["Audition procureur"]: "Procédure Pénale", @@ -71,6 +72,8 @@ const categorieEvenement: { ["Refus de contrôle"]: "Autre", ["Administrateur AD'HOC"]: "Autre", ["Signalement"]: "Autre", + ["contrôle URSSAF"]: "Autre", + ["contrôle fiscal"]: "Autre", }; export type CategorieEvenement = diff --git a/src/data/TypeEvenement.ts b/src/data/TypeEvenement.ts index 01ea612..b53bb54 100644 --- a/src/data/TypeEvenement.ts +++ b/src/data/TypeEvenement.ts @@ -28,4 +28,7 @@ export type TypeEvenement = | "Rappel à la loi" | "Passage police municipale" | "Administrateur AD'HOC" - | "Validation désobéissance"; + | "Validation désobéissance" + | "contrôle URSSAF" + | "contrôle fiscal" + | "Gendarmerie/Forces de l'ordre"; diff --git a/src/data/checkDataConsistency.ts b/src/data/checkDataConsistency.ts index 2297268..af3f9d8 100644 --- a/src/data/checkDataConsistency.ts +++ b/src/data/checkDataConsistency.ts @@ -1,54 +1,82 @@ import { isValidEvenementFamille } from "./EvenementFamille"; -import { Famille } from "./Famille"; +import { Famille, isExResistant, isResistant } from "./Famille"; -export function checkDataConsistency(families: Famille[]): ConsistencyIssue[] { - return families.flatMap((family) => { +export function checkDataConsistency(families: Famille[]): ConsistencyReport { + const reports = families.map((family) => { return checkFamilyDataConsistency(family); }); + return { + errors: reports.flatMap((r) => r.errors), + warnings: reports.flatMap((r) => r.warnings), + }; } +export type ConsistencyReport = { + warnings: ConsistencyIssue[]; + errors: ConsistencyIssue[]; +}; export type ConsistencyIssue = { issueType: string; familyId: string; - canIgnore?: boolean; }; -function checkFamilyDataConsistency(family: Famille) { - const consistencyIssues: ConsistencyIssue[] = []; +function checkFamilyDataConsistency(family: Famille): ConsistencyReport { + const consistencyErrors: ConsistencyIssue[] = []; + const consistencyWarnings: ConsistencyIssue[] = []; if (family.Statut === "Résistant.e") { if (family.Integration === null) { - consistencyIssues.push({ + consistencyErrors.push({ familyId: family.Titre, issueType: "Résistant.e without startResistant", }); } if (family.Sortie !== null) { - consistencyIssues.push({ + consistencyErrors.push({ familyId: family.Titre, issueType: "Résistant.e with endResistant!!", }); } } else if (family.Statut === "Ex résistant·e·s") { if (family.Integration === null) { - consistencyIssues.push({ + consistencyErrors.push({ familyId: family.Titre, issueType: "Ex résistant.e.s without startResistant", }); } if (family.Sortie === null) { - consistencyIssues.push({ + consistencyErrors.push({ familyId: family.Titre, issueType: "Ex résistant.e.s without endResistant", }); } if (family.Integration! > family.Sortie!) { - consistencyIssues.push({ + consistencyErrors.push({ familyId: family.Titre, issueType: "startResistsant > endResistant ", }); } } - consistencyIssues.push( + if ( + (isResistant(family) || isExResistant(family)) && + family.Integration !== null + ) { + const miseEnDemeureBeforeInteg = + family.Evenements.find( + (e) => + e.Type === "Mise en demeure de scolarisation" && + (e.Date === null || e.Date < family.Integration!) + ) !== undefined; + if ( + miseEnDemeureBeforeInteg && + family.ContexteEntree !== "Après mise en demeure" + ) { + consistencyWarnings.push({ + familyId: family.Titre, + issueType: `ContextEntree incorrect`, + }); + } + } + consistencyErrors.push( ...family.Evenements.filter((e) => !isValidEvenementFamille(e.Type)).map( (e) => ({ familyId: family.Titre, @@ -56,14 +84,16 @@ function checkFamilyDataConsistency(family: Famille) { }) ) ); - consistencyIssues.push( + consistencyWarnings.push( ...family.Evenements.filter((e) => e.Type !== null && e.Date === null).map( (e) => ({ familyId: family.Titre, issueType: `Event ${e.notionId} with non null Type "${e.Type}" but null Date`, - canIgnore: true, }) ) ); - return consistencyIssues; + return { + errors: consistencyErrors, + warnings: consistencyWarnings, + }; } diff --git a/src/index.ts b/src/index.ts index c904dac..fb7f90a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,40 +1,71 @@ import { Client } from "@notionhq/client"; +import { writeFileSync } from "fs"; import { checkDataConsistency } from "./data/checkDataConsistency"; import { fetchFamiliesWithEventsFromNotion } from "./notion/fetch/fetchFamiliesWithEventsFromNotion"; import { publishStatisticsToNotion } from "./notion/publish/publishStatisticsToNotion"; import { computeELStats } from "./statistiques/computeELStats"; -(async function IIFE() { +type ProcessOptions = { + dryRun: boolean; + notionApiToken: string; +}; + +/** + * Build options from process args and env + */ +function buildProcessOptions(): ProcessOptions { const notionApiToken = process.env.NOTION_TOKEN; if (!notionApiToken) { throw new Error("process.env.NOTION_TOKEN not defined"); } + const dryRun = process.argv.length >= 2 && process.argv[2] === "--dry-run"; + return { + dryRun: dryRun, + notionApiToken: notionApiToken, + }; +} + +(async function IIFE() { + const options = buildProcessOptions(); const notionClient = new Client({ - auth: notionApiToken, + auth: options.notionApiToken, }); const doFetch = true; console.log("Fetching families..."); - const families = doFetch + const familles = doFetch ? await fetchFamiliesWithEventsFromNotion(notionClient) : []; console.log("Checking Data Consistency issues..."); - const consistencyIssues = checkDataConsistency(families); - if (consistencyIssues.length > 0) { - console.error("Found consistency issues:"); - console.error(consistencyIssues); - if (consistencyIssues.find((issue) => !issue.canIgnore)) { - process.exit(1); - } + const consistencyReport = checkDataConsistency(familles); + if (consistencyReport.errors.length > 0) { + console.error( + `Found ${consistencyReport.errors.length} consistency error(s) that prevent building stats:` + ); + console.error(consistencyReport.errors); + process.exit(1); + } + if (consistencyReport.warnings.length > 0) { + console.warn( + `Found ${consistencyReport.warnings.length} non blocking consistency warning(s):` + ); + console.warn(consistencyReport.warnings); } const currentDate = new Date(Date.now()); console.log("Building statistics..."); - const resistantCountStats = computeELStats(families, currentDate); + const elStats = computeELStats(familles, currentDate); - console.log("Publishing statistics..."); - publishStatisticsToNotion(resistantCountStats, currentDate, notionClient); + if (options.dryRun) { + console.log( + "Dry run => Skip Publishing. Stats are dumped in file el-stats.json" + ); + writeFileSync("./el-stats.json", JSON.stringify(elStats, null, " ")); + } else { + console.log("Publishing statistics..."); + publishStatisticsToNotion(elStats, currentDate, notionClient); + } })();