From 5c0aebbcc3b0d2976c94069bfe61e5a3680e4804 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Arod?= Date: Sat, 12 Oct 2024 18:54:11 +0200 Subject: [PATCH] =?UTF-8?q?refactor:=20revue=20nettoyage=20des=20donn?= =?UTF-8?q?=C3=A9es?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/data/nettoyage/MessageDeNettoyage.ts | 25 +++++++ .../{ => nettoyage}/checkDataConsistency.ts | 2 +- src/data/nettoyage/nettoyerDonneesFamilles.ts | 68 +++++++++++++++++++ src/index.ts | 51 +++++++++----- .../fetchFamiliesWithEventsFromNotion.ts | 4 +- 5 files changed, 130 insertions(+), 20 deletions(-) create mode 100644 src/data/nettoyage/MessageDeNettoyage.ts rename src/data/{ => nettoyage}/checkDataConsistency.ts (97%) create mode 100644 src/data/nettoyage/nettoyerDonneesFamilles.ts diff --git a/src/data/nettoyage/MessageDeNettoyage.ts b/src/data/nettoyage/MessageDeNettoyage.ts new file mode 100644 index 0000000..667bfce --- /dev/null +++ b/src/data/nettoyage/MessageDeNettoyage.ts @@ -0,0 +1,25 @@ +export type MessageDeNettoyage = { + message: string; + severite: + | "Donnée ignorée" + | "Donnée suspecte" + | "Donnée incohérente bloquante"; +}; +export function msgDonneeIgnoree(message: string): MessageDeNettoyage { + return { + message, + severite: "Donnée ignorée", + }; +} +export function msgDonneeSuspecte(message: string): MessageDeNettoyage { + return { + message, + severite: "Donnée suspecte", + }; +} +export function msgDonneeBloquante(message: string): MessageDeNettoyage { + return { + message, + severite: "Donnée incohérente bloquante", + }; +} diff --git a/src/data/checkDataConsistency.ts b/src/data/nettoyage/checkDataConsistency.ts similarity index 97% rename from src/data/checkDataConsistency.ts rename to src/data/nettoyage/checkDataConsistency.ts index 101c3f6..157bce8 100644 --- a/src/data/checkDataConsistency.ts +++ b/src/data/nettoyage/checkDataConsistency.ts @@ -1,4 +1,4 @@ -import { Famille, isExResistant, isResistant } from "./Famille"; +import { Famille, isExResistant, isResistant } from "../Famille"; export function checkDataConsistency(families: Famille[]): ConsistencyReport { const reports = families.map((family) => { diff --git a/src/data/nettoyage/nettoyerDonneesFamilles.ts b/src/data/nettoyage/nettoyerDonneesFamilles.ts new file mode 100644 index 0000000..37f639e --- /dev/null +++ b/src/data/nettoyage/nettoyerDonneesFamilles.ts @@ -0,0 +1,68 @@ +import { Famille } from "../Famille"; +import { checkDataConsistency } from "./checkDataConsistency"; +import { + MessageDeNettoyage, + msgDonneeBloquante, + msgDonneeIgnoree, + msgDonneeSuspecte, +} from "./MessageDeNettoyage"; + +type DonneesNettoyees = { + familles: Famille[]; + messages: MessageDeNettoyage[]; +}; + +export function nettoyerDonneesFamilles( + donneesFamillesBrutes: Famille[] +): DonneesNettoyees { + let familles = donneesFamillesBrutes; + + let messages: MessageDeNettoyage[] = []; + + const output = supprimerLesEntreesVide(familles); + messages = [...messages, ...output.messages]; + familles = output.familles; + + // TODO convert checkDataConsistency to filters + const consistencyReport = checkDataConsistency(familles); + + // Adapte les message + const errorMessages: MessageDeNettoyage[] = consistencyReport.errors.map( + (e) => msgDonneeBloquante(`${e.familyId} - ${e.issueType}`) + ); + const warnings: MessageDeNettoyage[] = consistencyReport.warnings.map((e) => + msgDonneeSuspecte(`${e.familyId} - ${e.issueType}`) + ); + + messages = [...messages, ...errorMessages, ...warnings]; + + return { + familles, + messages, + }; +} + +/** + * Supprime "Famille de résistant" qui sont souvent créé par erreur + */ +function supprimerLesEntreesVide(familles: Famille[]): DonneesNettoyees { + const nettoyees = familles.filter((f) => f.Titre !== "Famille de résistant"); + const ignorees = familles.filter((f) => f.Titre === "Famille de résistant"); + if (ignorees.length > 0) { + return { + familles: nettoyees, + messages: [ + msgDonneeIgnoree( + `${ + nettoyees.length - familles.length + } enregistrement(s) famille "vide" ignoré(s)` + ), + ], + }; + } else { + return { + familles: nettoyees, + messages: [], + }; + } +} diff --git a/src/index.ts b/src/index.ts index 63cc768..e83bfff 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,5 @@ import { Client } from "@notionhq/client"; import { writeFileSync } from "fs"; -import { checkDataConsistency } from "./data/checkDataConsistency"; import { fetchFamiliesWithEventsFromNotion } from "./notion/fetch/fetchFamiliesWithEventsFromNotion"; import { publishStatisticsToNotion } from "./notion/publish/v1/publishStatisticsToNotion"; import { publishStatsToPage } from "./notion/publish/v2/publishStatsToPage"; @@ -17,6 +16,7 @@ import { formatDate } from "date-fns"; import { computeStatsGeneralesMensuelles } from "./statistiques/v2/generales/computeStatsGeneralesMensuelles"; import { mermaidDiagramStatsGeneralesMensuelles } from "./statistiques/v2/generales/mermaidDiagramStatsGeneralesMensuelles"; import { publishStatsGenerales } from "./notion/publish/v2/publishStatsGenerales"; +import { nettoyerDonneesFamilles } from "./data/nettoyage/nettoyerDonneesFamilles"; type ProcessOptions = { dryRun: boolean; @@ -45,35 +45,54 @@ function buildProcessOptions(): ProcessOptions { auth: options.notionApiToken, }); - console.log("Checking DB schemas..."); + console.log("Vérification schéma des bases Notion..."); await checkDbSchemas(notionClient); - console.log("Fetching families..."); + console.log("Téléchargement des familles..."); const doFetch = true; - const familles = doFetch + const donneesFamillesBrutes = doFetch ? await fetchFamiliesWithEventsFromNotion(notionClient) : []; - console.log("Checking Data Consistency issues..."); - const consistencyReport = checkDataConsistency(familles); - if (consistencyReport.errors.length > 0) { + console.log("Nettoyage des données brutes..."); + const { familles, messages } = nettoyerDonneesFamilles(donneesFamillesBrutes); + + const donneesBloquantes = messages.filter( + (m) => m.severite === "Donnée incohérente bloquante" + ); + const donneesSuspectes = messages.filter( + (m) => m.severite === "Donnée suspecte" + ); + const donneesIgnorees = messages.filter( + (m) => m.severite === "Donnée ignorée" + ); + + if (donneesBloquantes.length > 0) { console.error( - `Found ${consistencyReport.errors.length} consistency error(s) that prevent building stats:` + `${donneesBloquantes.length} Données bloquantes empêche de calculer les statistiques:` ); - console.error(consistencyReport.errors); + donneesBloquantes.forEach((e) => { + console.warn(" - " + e.message); + }); process.exit(1); } - if (consistencyReport.warnings.length > 0) { - console.warn( - `Found ${consistencyReport.warnings.length} non blocking consistency warning(s):` - ); - console.warn(consistencyReport.warnings); + if (donneesSuspectes.length > 0) { + console.warn(`${donneesSuspectes.length} Données suspectes trouvées:`); + donneesSuspectes.forEach((e) => { + console.warn(" - " + e.message); + }); + } + if (donneesIgnorees.length > 0) { + console.warn(`${donneesIgnorees.length} Données ignorées:`); + donneesIgnorees.forEach((e) => { + console.warn(" - " + e.message); + }); } const currentDate = new Date(Date.now()); - console.log("Building statistics..."); + console.log("Calcul des statistiques..."); const elStats = computeELStats(familles, currentDate); const statsGenerales = computeStatsGenerales(familles); @@ -93,7 +112,7 @@ function buildProcessOptions(): ProcessOptions { ); if (options.dryRun) { console.log( - "Dry run => Skip Publishing. Stats are dumped in file el-stats-xxx.json" + "Dry run => Pas de publication. Les stats sont écrite localement dans el-stats-xxx" ); writeFileSync("./el-stats-v1.json", JSON.stringify(elStats, null, " ")); diff --git a/src/notion/fetch/fetchFamiliesWithEventsFromNotion.ts b/src/notion/fetch/fetchFamiliesWithEventsFromNotion.ts index 1f70835..8eb38c5 100644 --- a/src/notion/fetch/fetchFamiliesWithEventsFromNotion.ts +++ b/src/notion/fetch/fetchFamiliesWithEventsFromNotion.ts @@ -47,9 +47,7 @@ export async function fetchFamiliesWithEventsFromNotion( return buildFamily(pageObjectResponse, familyEvents); }) ); - // Fix to remove the empty "Famille de résistant" page that is often created by mistake - const filtered = familles.filter((f) => f.Titre !== "Famille de résistant"); - return filtered; + return familles; } function buildFamilyEvent(page: PageObjectResponse): EvenementFamille {