mirror of
https://framagit.org/enfance-libre/statistiques
synced 2025-03-12 09:15:02 +00:00
feat: base infra and first statistics
This commit is contained in:
parent
3315a45326
commit
74b7fa8f0e
35 changed files with 898 additions and 16 deletions
|
@ -1,9 +1,8 @@
|
||||||
# Statistiques
|
# Statistiques
|
||||||
|
|
||||||
|
This modules computes EL statistiques from Notion Data
|
||||||
|
|
||||||
# Setup
|
# Setup
|
||||||
|
|
||||||
- Configure a .env defining the NOTION_TOKEN variable
|
- Create file `.env` defining the NOTION_TOKEN variable
|
||||||
- Run
|
- Run `yarn install && yarn run start`
|
||||||
`yarn install
|
|
||||||
yarn run update-stats
|
|
||||||
`
|
|
||||||
|
|
86
src/data/Family.ts
Normal file
86
src/data/Family.ts
Normal file
|
@ -0,0 +1,86 @@
|
||||||
|
import { arePeriodsOverlaping } from "../period/arePeriodsOverlaping";
|
||||||
|
import { Period } from "../period/Period";
|
||||||
|
|
||||||
|
export type FamilyStatus =
|
||||||
|
| "Résistant.e"
|
||||||
|
| "Ex résistant·e·s"
|
||||||
|
| "À préciser"
|
||||||
|
| "Se questionne"
|
||||||
|
| "Désobéissence décidée"
|
||||||
|
| "Rédaction Déclaration"
|
||||||
|
| "Déclaration Validée - Attente éléments"
|
||||||
|
| "Abdandon"
|
||||||
|
| "Incompatible";
|
||||||
|
export type Family = {
|
||||||
|
notionId: string;
|
||||||
|
title: string;
|
||||||
|
status: FamilyStatus;
|
||||||
|
|
||||||
|
startResistsant: Date | null;
|
||||||
|
endResistant: Date | null;
|
||||||
|
evenements: FamilyEvent[];
|
||||||
|
};
|
||||||
|
|
||||||
|
export function isResistant(family: Family): boolean {
|
||||||
|
return (
|
||||||
|
family.status === "Résistant.e" &&
|
||||||
|
family.startResistsant !== null &&
|
||||||
|
family.endResistant === null
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isExResistant(family: Family): boolean {
|
||||||
|
return (
|
||||||
|
family.status === "Ex résistant·e·s" &&
|
||||||
|
family.startResistsant !== null &&
|
||||||
|
family.endResistant !== null
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isResistantAtDate(family: Family, date: Date): boolean {
|
||||||
|
if (
|
||||||
|
isResistant(family) &&
|
||||||
|
family.startResistsant!.getTime() <= date.getTime()
|
||||||
|
) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
isExResistant(family) &&
|
||||||
|
family.startResistsant!.getTime() <= date.getTime() &&
|
||||||
|
family.endResistant!.getTime() > date.getTime()
|
||||||
|
) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isResistantOverPeriod(family: Family, period: Period): boolean {
|
||||||
|
const familyPeriodResistant: Period | null = periodOfResistance(family);
|
||||||
|
return (
|
||||||
|
familyPeriodResistant !== null &&
|
||||||
|
arePeriodsOverlaping(familyPeriodResistant, period)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function periodOfResistance(family: Family): Period | null {
|
||||||
|
if (isResistant(family)) {
|
||||||
|
const periodResistant: Period = {
|
||||||
|
start: family.startResistsant!,
|
||||||
|
end: new Date(Date.now()),
|
||||||
|
};
|
||||||
|
return periodResistant;
|
||||||
|
}
|
||||||
|
if (isExResistant(family)) {
|
||||||
|
const periodResistant: Period = {
|
||||||
|
start: family.startResistsant!,
|
||||||
|
end: family.endResistant!,
|
||||||
|
};
|
||||||
|
return periodResistant;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type FamilyEvent = {
|
||||||
|
date: Date;
|
||||||
|
type: string;
|
||||||
|
};
|
63
src/data/checkDataConsistency.ts
Normal file
63
src/data/checkDataConsistency.ts
Normal file
|
@ -0,0 +1,63 @@
|
||||||
|
import { Family } from "./Family";
|
||||||
|
|
||||||
|
export function checkDataConsistency(families: Family[]): ConsistencyIssue[] {
|
||||||
|
return families.flatMap((family) => {
|
||||||
|
return checkFamilyDataConsistency(family);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export type ConsistencyIssue = {
|
||||||
|
issueType: string;
|
||||||
|
familyId: string;
|
||||||
|
};
|
||||||
|
function checkFamilyDataConsistency(family: Family) {
|
||||||
|
const consistencyIssues: ConsistencyIssue[] = [];
|
||||||
|
|
||||||
|
if (family.status === "Résistant.e") {
|
||||||
|
if (family.startResistsant === null) {
|
||||||
|
consistencyIssues.push({
|
||||||
|
familyId: family.title,
|
||||||
|
issueType: "Résistant.e without startResistant",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (family.endResistant !== null) {
|
||||||
|
consistencyIssues.push({
|
||||||
|
familyId: family.title,
|
||||||
|
issueType: "Résistant.e with endResistant!!",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} else if (family.status === "Ex résistant·e·s") {
|
||||||
|
if (family.startResistsant === null) {
|
||||||
|
consistencyIssues.push({
|
||||||
|
familyId: family.title,
|
||||||
|
issueType: "Ex résistant.e.s without startResistant",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (family.endResistant === null) {
|
||||||
|
consistencyIssues.push({
|
||||||
|
familyId: family.title,
|
||||||
|
issueType: "Ex résistant.e.s without endResistant",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (family.startResistsant!.getTime() > family.endResistant!.getTime()) {
|
||||||
|
consistencyIssues.push({
|
||||||
|
familyId: family.title,
|
||||||
|
issueType: "startResistsant > endResistant ",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if (family.startResistsant !== null) {
|
||||||
|
consistencyIssues.push({
|
||||||
|
familyId: family.title,
|
||||||
|
issueType: family.status + " with startResistant",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (family.endResistant !== null) {
|
||||||
|
consistencyIssues.push({
|
||||||
|
familyId: family.title,
|
||||||
|
issueType: family.status + " with endResistant",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return consistencyIssues;
|
||||||
|
}
|
|
@ -1,8 +0,0 @@
|
||||||
import { describe, expect, test } from "@jest/globals";
|
|
||||||
import { hello } from ".";
|
|
||||||
|
|
||||||
describe("module", () => {
|
|
||||||
test("test hello", () => {
|
|
||||||
expect(hello()).toBe("hello");
|
|
||||||
});
|
|
||||||
});
|
|
38
src/index.ts
38
src/index.ts
|
@ -1,3 +1,37 @@
|
||||||
export function hello() {
|
import { Client } from "@notionhq/client";
|
||||||
return "hello";
|
import { checkDataConsistency } from "./data/checkDataConsistency";
|
||||||
|
import { fetchFamiliesWithEventsFromNotion } from "./notion/fetch/fetchFamiliesWithEventsFromNotion";
|
||||||
|
import { publishStatisticsToNotion } from "./notion/publish/publishStatisticsToNotion";
|
||||||
|
import { computeELStats } from "./statistiques/computeELStats";
|
||||||
|
|
||||||
|
(async function IIFE() {
|
||||||
|
const notionApiToken = process.env.NOTION_TOKEN;
|
||||||
|
if (!notionApiToken) {
|
||||||
|
throw new Error("process.env.NOTION_TOKEN not defined");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const notionClient = new Client({
|
||||||
|
auth: notionApiToken,
|
||||||
|
});
|
||||||
|
|
||||||
|
const doFetch = true;
|
||||||
|
console.log("Fetching families...");
|
||||||
|
const families = doFetch
|
||||||
|
? await fetchFamiliesWithEventsFromNotion(notionClient)
|
||||||
|
: [];
|
||||||
|
|
||||||
|
console.log("Checking Data Consistency issues...");
|
||||||
|
const consistencyIssues = checkDataConsistency(families);
|
||||||
|
if (consistencyIssues.length > 0) {
|
||||||
|
console.log("Found consistency issues:");
|
||||||
|
console.log(consistencyIssues);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log("Building statistics...");
|
||||||
|
const resistantCountStats = computeELStats(families);
|
||||||
|
|
||||||
|
console.log(resistantCountStats);
|
||||||
|
|
||||||
|
console.log("Publishing statistics...");
|
||||||
|
publishStatisticsToNotion(resistantCountStats, notionClient);
|
||||||
|
})();
|
||||||
|
|
50
src/notion/fetch/fetchFamiliesWithEventsFromNotion.ts
Normal file
50
src/notion/fetch/fetchFamiliesWithEventsFromNotion.ts
Normal file
|
@ -0,0 +1,50 @@
|
||||||
|
import { Client } from "@notionhq/client";
|
||||||
|
import {
|
||||||
|
PageObjectResponse,
|
||||||
|
QueryDatabaseParameters,
|
||||||
|
} from "@notionhq/client/build/src/api-endpoints";
|
||||||
|
import { Family, FamilyStatus } from "../../data/Family";
|
||||||
|
import { datePropertyToDate } from "../utils/properties/datePropertyToDate";
|
||||||
|
import { statusPropertyToText } from "../utils/properties/statusPropertyToText";
|
||||||
|
import { titlePropertyToText } from "../utils/properties/titlePropertyToText copy";
|
||||||
|
import { queryAllDbResults } from "../utils/queryAllDbResults";
|
||||||
|
import { assertFullPage } from "../utils/types/assertFullPage";
|
||||||
|
|
||||||
|
export async function fetchFamiliesWithEventsFromNotion(
|
||||||
|
notionClient: Client
|
||||||
|
): Promise<Family[]> {
|
||||||
|
const familiesDbId: string = "5b69e02b296d4a578f8c8ab7fe8b05da";
|
||||||
|
const dbQuery: QueryDatabaseParameters = {
|
||||||
|
database_id: familiesDbId,
|
||||||
|
/*filter: {
|
||||||
|
property: "Statut",
|
||||||
|
status: {
|
||||||
|
equals: "Résistant.e",
|
||||||
|
},
|
||||||
|
},*/
|
||||||
|
};
|
||||||
|
|
||||||
|
const results = await queryAllDbResults(notionClient, dbQuery);
|
||||||
|
const families: Family[] = await Promise.all(
|
||||||
|
results.map((pageObjectResponse) => {
|
||||||
|
assertFullPage(pageObjectResponse);
|
||||||
|
return buildFamily(pageObjectResponse);
|
||||||
|
})
|
||||||
|
);
|
||||||
|
return families;
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildFamily(page: PageObjectResponse): Family {
|
||||||
|
const pageProperties = page.properties;
|
||||||
|
|
||||||
|
// TODO Fetch Family Events
|
||||||
|
const family: Family = {
|
||||||
|
notionId: page.id,
|
||||||
|
title: titlePropertyToText(pageProperties, ""),
|
||||||
|
status: statusPropertyToText(pageProperties, "Statut") as FamilyStatus,
|
||||||
|
startResistsant: datePropertyToDate(pageProperties, "Intégration"),
|
||||||
|
endResistant: datePropertyToDate(pageProperties, "Sortie"),
|
||||||
|
evenements: [],
|
||||||
|
};
|
||||||
|
return family;
|
||||||
|
}
|
73
src/notion/publish/publishCurrentStats.ts
Normal file
73
src/notion/publish/publishCurrentStats.ts
Normal file
|
@ -0,0 +1,73 @@
|
||||||
|
import { Client, isFullBlock } from "@notionhq/client";
|
||||||
|
import { BlockObjectRequest } from "@notionhq/client/build/src/api-endpoints";
|
||||||
|
import { ELStats } from "../../statistiques/ELStats";
|
||||||
|
import { listAllChildrenBlocks } from "../utils/listAllChildrenBlocks";
|
||||||
|
import { removeBlocks } from "../utils/removeBlocks";
|
||||||
|
import { richTextToPlainText } from "../utils/text/richTextToPlainText";
|
||||||
|
import { currentStatsHeading, statsPageId } from "./publishStatisticsToNotion";
|
||||||
|
|
||||||
|
export async function publishCurrentStats(
|
||||||
|
notionClient: Client,
|
||||||
|
stats: ELStats
|
||||||
|
) {
|
||||||
|
const childrenBlocks = (
|
||||||
|
await listAllChildrenBlocks(notionClient, {
|
||||||
|
block_id: statsPageId,
|
||||||
|
})
|
||||||
|
).filter(isFullBlock);
|
||||||
|
|
||||||
|
const currentStatsHeadingBlockIndex = childrenBlocks.findIndex(
|
||||||
|
(block) =>
|
||||||
|
block.type === "heading_1" &&
|
||||||
|
richTextToPlainText(block.heading_1.rich_text) === currentStatsHeading
|
||||||
|
);
|
||||||
|
const endOfCurrentStats = childrenBlocks.findIndex(
|
||||||
|
(block, index) =>
|
||||||
|
index > currentStatsHeadingBlockIndex && block.type === "heading_1"
|
||||||
|
);
|
||||||
|
if (currentStatsHeadingBlockIndex === -1 || endOfCurrentStats === -1) {
|
||||||
|
throw new Error("Cannot find expected headings in statistics page");
|
||||||
|
}
|
||||||
|
|
||||||
|
const currentStatsHeadingBlock =
|
||||||
|
childrenBlocks[currentStatsHeadingBlockIndex];
|
||||||
|
|
||||||
|
const blocksIdsToRemove = childrenBlocks
|
||||||
|
.slice(currentStatsHeadingBlockIndex + 1, endOfCurrentStats)
|
||||||
|
.map((b) => b.id);
|
||||||
|
await removeBlocks(notionClient, blocksIdsToRemove);
|
||||||
|
|
||||||
|
notionClient.blocks.children.append({
|
||||||
|
block_id: statsPageId,
|
||||||
|
after: currentStatsHeadingBlock.id,
|
||||||
|
children: [
|
||||||
|
currentStatBlock("Nb Famille Résistante", stats.resistantsCount),
|
||||||
|
currentStatBlock(
|
||||||
|
"Nb Famille Résistante ou Ex-Résistante",
|
||||||
|
stats.resistantsOrExCount
|
||||||
|
),
|
||||||
|
],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function currentStatBlock(
|
||||||
|
stateName: string,
|
||||||
|
value: number
|
||||||
|
): BlockObjectRequest {
|
||||||
|
return {
|
||||||
|
paragraph: {
|
||||||
|
rich_text: [
|
||||||
|
{
|
||||||
|
text: {
|
||||||
|
content:
|
||||||
|
stateName +
|
||||||
|
": " +
|
||||||
|
value.toLocaleString("us-FR", {
|
||||||
|
maximumFractionDigits: 0,
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
69
src/notion/publish/publishStatisticsToNotion.ts
Normal file
69
src/notion/publish/publishStatisticsToNotion.ts
Normal file
|
@ -0,0 +1,69 @@
|
||||||
|
import { Client, isFullPage } from "@notionhq/client";
|
||||||
|
import { ELPeriodStats, ELStats } from "../../statistiques/ELStats";
|
||||||
|
import { queryAllDbResults } from "../utils/queryAllDbResults";
|
||||||
|
import { removeBlocks } from "../utils/removeBlocks";
|
||||||
|
import { publishCurrentStats } from "./publishCurrentStats";
|
||||||
|
|
||||||
|
export const statsPageId = "2b91cd90e3694e96bb196d69aeca59b1";
|
||||||
|
export const currentStatsHeading = "Statistiques actuelles";
|
||||||
|
|
||||||
|
const yearStatsDb = "4b19a72aa07840eab948525ea41878ee";
|
||||||
|
const monthStatsDb = "8418a8a4a7544f6a8c54e6003be7efe5";
|
||||||
|
export async function publishStatisticsToNotion(
|
||||||
|
stats: ELStats,
|
||||||
|
notionClient: Client
|
||||||
|
) {
|
||||||
|
await publishCurrentStats(notionClient, stats);
|
||||||
|
|
||||||
|
await publishPeriodStats(notionClient, yearStatsDb, stats.years);
|
||||||
|
|
||||||
|
await publishPeriodStats(notionClient, monthStatsDb, stats.months);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function publishPeriodStats(
|
||||||
|
notionClient: Client,
|
||||||
|
periodStatsDbId: string,
|
||||||
|
stats: ELPeriodStats[]
|
||||||
|
) {
|
||||||
|
const periodRows = (
|
||||||
|
await queryAllDbResults(notionClient, {
|
||||||
|
database_id: periodStatsDbId,
|
||||||
|
})
|
||||||
|
).filter(isFullPage);
|
||||||
|
await removeBlocks(
|
||||||
|
notionClient,
|
||||||
|
periodRows.map((r) => r.id)
|
||||||
|
);
|
||||||
|
|
||||||
|
for (const stat of stats) {
|
||||||
|
await notionClient.pages.create({
|
||||||
|
parent: {
|
||||||
|
database_id: periodStatsDbId,
|
||||||
|
},
|
||||||
|
properties: {
|
||||||
|
Période: {
|
||||||
|
title: [
|
||||||
|
{
|
||||||
|
text: {
|
||||||
|
content: stat.periodId,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"Nb Famille Résistante": numberProp(stat.resistantsCount),
|
||||||
|
"Nb Famille Résistante - Evol": numberProp(stat.resistantsCountEvol),
|
||||||
|
"Nb Famille Résistante - Evol %": numberProp(
|
||||||
|
stat.resistantsCountEvolPercent
|
||||||
|
),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function numberProp(n: number): {
|
||||||
|
number: number;
|
||||||
|
} {
|
||||||
|
return {
|
||||||
|
number: n,
|
||||||
|
};
|
||||||
|
}
|
22
src/notion/utils/listAllChildrenBlocks.ts
Normal file
22
src/notion/utils/listAllChildrenBlocks.ts
Normal file
|
@ -0,0 +1,22 @@
|
||||||
|
import { Client } from "@notionhq/client";
|
||||||
|
import {
|
||||||
|
BlockObjectResponse,
|
||||||
|
ListBlockChildrenParameters,
|
||||||
|
PartialBlockObjectResponse,
|
||||||
|
} from "@notionhq/client/build/src/api-endpoints";
|
||||||
|
|
||||||
|
export async function listAllChildrenBlocks(
|
||||||
|
notion: Client,
|
||||||
|
query: ListBlockChildrenParameters
|
||||||
|
): Promise<Array<PartialBlockObjectResponse | BlockObjectResponse>> {
|
||||||
|
const response = await notion.blocks.children.list(query);
|
||||||
|
if (response.has_more && response.next_cursor) {
|
||||||
|
const moreResults = await listAllChildrenBlocks(notion, {
|
||||||
|
...query,
|
||||||
|
start_cursor: response.next_cursor,
|
||||||
|
});
|
||||||
|
return [...response.results, ...moreResults];
|
||||||
|
} else {
|
||||||
|
return response.results;
|
||||||
|
}
|
||||||
|
}
|
18
src/notion/utils/properties/datePropertyToDate.ts
Normal file
18
src/notion/utils/properties/datePropertyToDate.ts
Normal file
|
@ -0,0 +1,18 @@
|
||||||
|
import { PageProperties } from "../types/PageProperties";
|
||||||
|
import { extractPagePropertyValue } from "./extractPagePropertyValue";
|
||||||
|
|
||||||
|
export function datePropertyToDate(
|
||||||
|
pageProperties: PageProperties,
|
||||||
|
propName: string
|
||||||
|
): Date | null {
|
||||||
|
const propValue = extractPagePropertyValue(pageProperties, propName);
|
||||||
|
if (propValue.type !== "date") {
|
||||||
|
throw new Error(
|
||||||
|
`Property ${propName} was expected to have type "date" but got "${propValue.type}".`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (propValue.date === null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return new Date(Date.parse(propValue.date.start));
|
||||||
|
}
|
13
src/notion/utils/properties/extractPagePropertyValue.ts
Normal file
13
src/notion/utils/properties/extractPagePropertyValue.ts
Normal file
|
@ -0,0 +1,13 @@
|
||||||
|
import { PageProperties } from "../types/PageProperties";
|
||||||
|
import { PagePropertyValue } from "../types/PagePropertyValue";
|
||||||
|
|
||||||
|
export function extractPagePropertyValue(
|
||||||
|
pageProperties: PageProperties,
|
||||||
|
propName: string
|
||||||
|
): PagePropertyValue {
|
||||||
|
const propValue = pageProperties[propName];
|
||||||
|
if (!propValue) {
|
||||||
|
throw new Error(`Prop ${propName} is missing.`);
|
||||||
|
}
|
||||||
|
return propValue;
|
||||||
|
}
|
15
src/notion/utils/properties/statusPropertyToText.ts
Normal file
15
src/notion/utils/properties/statusPropertyToText.ts
Normal file
|
@ -0,0 +1,15 @@
|
||||||
|
import { PageProperties } from "../types/PageProperties";
|
||||||
|
import { extractPagePropertyValue } from "./extractPagePropertyValue";
|
||||||
|
|
||||||
|
export function statusPropertyToText(
|
||||||
|
pageProperties: PageProperties,
|
||||||
|
propName: string
|
||||||
|
): string {
|
||||||
|
const propValue = extractPagePropertyValue(pageProperties, propName);
|
||||||
|
if (propValue.type !== "status") {
|
||||||
|
throw new Error(
|
||||||
|
`Property ${propName} was expected to have type "title" but got "${propValue.type}".`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return propValue.status!.name;
|
||||||
|
}
|
16
src/notion/utils/properties/titlePropertyToText copy.ts
Normal file
16
src/notion/utils/properties/titlePropertyToText copy.ts
Normal file
|
@ -0,0 +1,16 @@
|
||||||
|
import { richTextToPlainText } from "../text/richTextToPlainText";
|
||||||
|
import { PageProperties } from "../types/PageProperties";
|
||||||
|
import { extractPagePropertyValue } from "./extractPagePropertyValue";
|
||||||
|
|
||||||
|
export function titlePropertyToText(
|
||||||
|
pageProperties: PageProperties,
|
||||||
|
propName: string
|
||||||
|
): string {
|
||||||
|
const propValue = extractPagePropertyValue(pageProperties, propName);
|
||||||
|
if (propValue.type !== "title") {
|
||||||
|
throw new Error(
|
||||||
|
`Property ${propName} was expected to have type "title" but got "${propValue.type}".`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return richTextToPlainText(propValue.title);
|
||||||
|
}
|
21
src/notion/utils/queryAllDbResults.ts
Normal file
21
src/notion/utils/queryAllDbResults.ts
Normal file
|
@ -0,0 +1,21 @@
|
||||||
|
import { Client } from "@notionhq/client";
|
||||||
|
import {
|
||||||
|
QueryDatabaseParameters,
|
||||||
|
QueryDatabaseResponse,
|
||||||
|
} from "@notionhq/client/build/src/api-endpoints";
|
||||||
|
|
||||||
|
export async function queryAllDbResults(
|
||||||
|
notion: Client,
|
||||||
|
dbQuery: QueryDatabaseParameters
|
||||||
|
): Promise<QueryDatabaseResponse["results"]> {
|
||||||
|
const dbResponse = await notion.databases.query(dbQuery);
|
||||||
|
if (dbResponse.has_more && dbResponse.next_cursor) {
|
||||||
|
const moreResults = await queryAllDbResults(notion, {
|
||||||
|
...dbQuery,
|
||||||
|
start_cursor: dbResponse.next_cursor,
|
||||||
|
});
|
||||||
|
return [...dbResponse.results, ...moreResults];
|
||||||
|
} else {
|
||||||
|
return dbResponse.results;
|
||||||
|
}
|
||||||
|
}
|
12
src/notion/utils/removeBlocks.ts
Normal file
12
src/notion/utils/removeBlocks.ts
Normal file
|
@ -0,0 +1,12 @@
|
||||||
|
import { Client } from "@notionhq/client";
|
||||||
|
|
||||||
|
export async function removeBlocks(
|
||||||
|
notionClient: Client,
|
||||||
|
blocksIdsToRemove: string[]
|
||||||
|
) {
|
||||||
|
for (const id of blocksIdsToRemove) {
|
||||||
|
await notionClient.blocks.delete({
|
||||||
|
block_id: id,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
19
src/notion/utils/text/paragraphBlockChildrenToPlainText.ts
Normal file
19
src/notion/utils/text/paragraphBlockChildrenToPlainText.ts
Normal file
|
@ -0,0 +1,19 @@
|
||||||
|
import { isFullBlock } from "@notionhq/client";
|
||||||
|
import {
|
||||||
|
ListBlockChildrenResponse,
|
||||||
|
ParagraphBlockObjectResponse,
|
||||||
|
} from "@notionhq/client/build/src/api-endpoints";
|
||||||
|
import { richTextToPlainText } from "./richTextToPlainText";
|
||||||
|
|
||||||
|
export function paragraphBlockChildrenToPlainText(
|
||||||
|
results: ListBlockChildrenResponse["results"]
|
||||||
|
): string {
|
||||||
|
return results
|
||||||
|
.filter(isFullBlock)
|
||||||
|
.filter((b) => b.type === "paragraph")
|
||||||
|
.map((b) => {
|
||||||
|
const p = b as ParagraphBlockObjectResponse;
|
||||||
|
return richTextToPlainText(p.paragraph.rich_text);
|
||||||
|
})
|
||||||
|
.join("\n");
|
||||||
|
}
|
17
src/notion/utils/text/richTextPropertyToPlainText.ts
Normal file
17
src/notion/utils/text/richTextPropertyToPlainText.ts
Normal file
|
@ -0,0 +1,17 @@
|
||||||
|
import { richTextToPlainText } from "./richTextToPlainText";
|
||||||
|
import { extractPagePropertyValue } from "../properties/extractPagePropertyValue";
|
||||||
|
import { PageProperties } from "../types/PageProperties";
|
||||||
|
|
||||||
|
export function richTextPropertyToPlainText(
|
||||||
|
pageProperties: PageProperties,
|
||||||
|
propName: string
|
||||||
|
): string {
|
||||||
|
const propValue = extractPagePropertyValue(pageProperties, propName);
|
||||||
|
if (propValue.type !== "rich_text") {
|
||||||
|
throw new Error(
|
||||||
|
`Property ${propName} was expected to have type "rich_text" but got "${propValue.type}".`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const propText = richTextToPlainText(propValue.rich_text);
|
||||||
|
return propText;
|
||||||
|
}
|
13
src/notion/utils/text/richTextToPlainText.ts
Normal file
13
src/notion/utils/text/richTextToPlainText.ts
Normal file
|
@ -0,0 +1,13 @@
|
||||||
|
import { RichTextItemResponse } from "@notionhq/client/build/src/api-endpoints";
|
||||||
|
|
||||||
|
export function richTextToPlainText(rich_texts: Array<RichTextItemResponse>) {
|
||||||
|
return rich_texts
|
||||||
|
.map((rt) => {
|
||||||
|
if (rt.href) {
|
||||||
|
return rt.plain_text + " (" + rt.href + ")";
|
||||||
|
} else {
|
||||||
|
return rt.plain_text;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.join("");
|
||||||
|
}
|
8
src/notion/utils/types/PageOrDatabaseResponse.ts
Normal file
8
src/notion/utils/types/PageOrDatabaseResponse.ts
Normal file
|
@ -0,0 +1,8 @@
|
||||||
|
import {
|
||||||
|
DatabaseObjectResponse,
|
||||||
|
PageObjectResponse,
|
||||||
|
} from "@notionhq/client/build/src/api-endpoints";
|
||||||
|
|
||||||
|
export type PageOrDatabaseResponse =
|
||||||
|
| PageObjectResponse
|
||||||
|
| DatabaseObjectResponse;
|
3
src/notion/utils/types/PageProperties.ts
Normal file
3
src/notion/utils/types/PageProperties.ts
Normal file
|
@ -0,0 +1,3 @@
|
||||||
|
import { PageObjectResponse } from "@notionhq/client/build/src/api-endpoints";
|
||||||
|
|
||||||
|
export type PageProperties = PageObjectResponse["properties"];
|
3
src/notion/utils/types/PagePropertyValue.ts
Normal file
3
src/notion/utils/types/PagePropertyValue.ts
Normal file
|
@ -0,0 +1,3 @@
|
||||||
|
import { PageProperties } from "./PageProperties";
|
||||||
|
|
||||||
|
export type PagePropertyValue = PageProperties["x"];
|
23
src/notion/utils/types/assertFullPage.ts
Normal file
23
src/notion/utils/types/assertFullPage.ts
Normal file
|
@ -0,0 +1,23 @@
|
||||||
|
import { isFullPage } from "@notionhq/client";
|
||||||
|
import {
|
||||||
|
BlockObjectResponse,
|
||||||
|
DatabaseObjectResponse,
|
||||||
|
PageObjectResponse,
|
||||||
|
PartialBlockObjectResponse,
|
||||||
|
PartialDatabaseObjectResponse,
|
||||||
|
PartialPageObjectResponse,
|
||||||
|
} from "@notionhq/client/build/src/api-endpoints";
|
||||||
|
|
||||||
|
export function assertFullPage(
|
||||||
|
response:
|
||||||
|
| PageObjectResponse
|
||||||
|
| PartialPageObjectResponse
|
||||||
|
| DatabaseObjectResponse
|
||||||
|
| PartialDatabaseObjectResponse
|
||||||
|
| BlockObjectResponse
|
||||||
|
| PartialBlockObjectResponse
|
||||||
|
): asserts response is PageObjectResponse {
|
||||||
|
if (!isFullPage(response)) {
|
||||||
|
throw new Error("Expecting a full page");
|
||||||
|
}
|
||||||
|
}
|
5
src/period/IdentifiedPeriod.ts
Normal file
5
src/period/IdentifiedPeriod.ts
Normal file
|
@ -0,0 +1,5 @@
|
||||||
|
import { Period } from "./Period";
|
||||||
|
|
||||||
|
export type IdentifiedPeriod = {
|
||||||
|
id: string;
|
||||||
|
} & Period;
|
5
src/period/Period.ts
Normal file
5
src/period/Period.ts
Normal file
|
@ -0,0 +1,5 @@
|
||||||
|
export type Period = {
|
||||||
|
start: Date;
|
||||||
|
/** Exclusive */
|
||||||
|
end: Date;
|
||||||
|
};
|
12
src/period/arePeriodsOverlaping.ts
Normal file
12
src/period/arePeriodsOverlaping.ts
Normal file
|
@ -0,0 +1,12 @@
|
||||||
|
import { isPeriodContaining } from "./isPeriodContaining";
|
||||||
|
import { Period } from "./Period";
|
||||||
|
|
||||||
|
export function arePeriodsOverlaping(
|
||||||
|
period1: Period,
|
||||||
|
period2: Period
|
||||||
|
): boolean {
|
||||||
|
return (
|
||||||
|
isPeriodContaining(period1, period2.start) ||
|
||||||
|
isPeriodContaining(period2, period1.start)
|
||||||
|
);
|
||||||
|
}
|
27
src/period/generateConsecutiveIdentifiedPeriods.ts
Normal file
27
src/period/generateConsecutiveIdentifiedPeriods.ts
Normal file
|
@ -0,0 +1,27 @@
|
||||||
|
import generateConsecutivePeriods from "./generateConsecutivePeriods";
|
||||||
|
import { IdentifiedPeriod } from "./IdentifiedPeriod";
|
||||||
|
import { Period } from "./Period";
|
||||||
|
|
||||||
|
export function generateConsecutiveIdentifiedPeriods({
|
||||||
|
start,
|
||||||
|
nextStart,
|
||||||
|
end = new Date(Date.now()),
|
||||||
|
idProvider,
|
||||||
|
}: {
|
||||||
|
start: Date;
|
||||||
|
nextStart: (date: Date) => Date;
|
||||||
|
end?: Date;
|
||||||
|
idProvider: (period: Period, index: number) => string;
|
||||||
|
}): IdentifiedPeriod[] {
|
||||||
|
return generateConsecutivePeriods({
|
||||||
|
start,
|
||||||
|
incrementStart: nextStart,
|
||||||
|
end,
|
||||||
|
}).map((period, index) => {
|
||||||
|
const id = idProvider(period, index);
|
||||||
|
return {
|
||||||
|
id,
|
||||||
|
...period,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
24
src/period/generateConsecutivePeriods.test.ts
Normal file
24
src/period/generateConsecutivePeriods.test.ts
Normal file
|
@ -0,0 +1,24 @@
|
||||||
|
import { describe, expect, test } from "@jest/globals";
|
||||||
|
import { addYears } from "date-fns/fp";
|
||||||
|
import generateConsecutivePeriods from "./generateConsecutivePeriods";
|
||||||
|
|
||||||
|
describe("generateConsecutivePeriods", () => {
|
||||||
|
test("generateYears", () => {
|
||||||
|
const results = generateConsecutivePeriods({
|
||||||
|
start: new Date(Date.UTC(2020, 0, 1)),
|
||||||
|
end: new Date(Date.UTC(2024, 3, 1)),
|
||||||
|
incrementStart: addYears(1),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(results).toHaveLength(5);
|
||||||
|
|
||||||
|
expect(results[0]).toEqual({
|
||||||
|
start: new Date(Date.UTC(2020, 0, 1)),
|
||||||
|
end: new Date(Date.UTC(2021, 0, 1)),
|
||||||
|
});
|
||||||
|
expect(results[4]).toEqual({
|
||||||
|
start: new Date(Date.UTC(2024, 0, 1)),
|
||||||
|
end: new Date(Date.UTC(2025, 0, 1)),
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
30
src/period/generateConsecutivePeriods.ts
Normal file
30
src/period/generateConsecutivePeriods.ts
Normal file
|
@ -0,0 +1,30 @@
|
||||||
|
import { Period } from "./Period";
|
||||||
|
|
||||||
|
export default function generateConsecutivePeriods({
|
||||||
|
start,
|
||||||
|
incrementStart,
|
||||||
|
end = new Date(Date.now()),
|
||||||
|
}: {
|
||||||
|
start: Date;
|
||||||
|
incrementStart: (date: Date) => Date;
|
||||||
|
end?: Date;
|
||||||
|
}): Period[] {
|
||||||
|
const periods: Period[] = [];
|
||||||
|
if (start.getTime() < end.getTime()) {
|
||||||
|
let nextPeriodStart = start;
|
||||||
|
do {
|
||||||
|
const periodStart = nextPeriodStart;
|
||||||
|
const periodEnd = incrementStart(nextPeriodStart);
|
||||||
|
if (periodEnd.getTime() < periodStart.getTime()) {
|
||||||
|
throw new Error("incrementStart call decremented date!!!");
|
||||||
|
}
|
||||||
|
periods.push({
|
||||||
|
start: periodStart,
|
||||||
|
end: periodEnd,
|
||||||
|
});
|
||||||
|
|
||||||
|
nextPeriodStart = periodEnd;
|
||||||
|
} while (nextPeriodStart.getTime() < end.getTime());
|
||||||
|
}
|
||||||
|
return periods;
|
||||||
|
}
|
62
src/period/isPeriodContaining.test.ts
Normal file
62
src/period/isPeriodContaining.test.ts
Normal file
|
@ -0,0 +1,62 @@
|
||||||
|
import { describe, expect, test } from "@jest/globals";
|
||||||
|
import { isPeriodContaining } from "./isPeriodContaining";
|
||||||
|
|
||||||
|
describe("isPeriodContaining", () => {
|
||||||
|
test("period contains start date", () => {
|
||||||
|
expect(
|
||||||
|
isPeriodContaining(
|
||||||
|
{
|
||||||
|
start: new Date(Date.UTC(2024, 0, 1)),
|
||||||
|
end: new Date(Date.UTC(2024, 3, 1)),
|
||||||
|
},
|
||||||
|
new Date(Date.UTC(2024, 0, 1))
|
||||||
|
)
|
||||||
|
).toBe(true);
|
||||||
|
});
|
||||||
|
test("period contains date in period", () => {
|
||||||
|
expect(
|
||||||
|
isPeriodContaining(
|
||||||
|
{
|
||||||
|
start: new Date(Date.UTC(2024, 0, 1)),
|
||||||
|
end: new Date(Date.UTC(2024, 3, 1)),
|
||||||
|
},
|
||||||
|
new Date(Date.UTC(2024, 0, 10))
|
||||||
|
)
|
||||||
|
).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("period does not contain end date", () => {
|
||||||
|
expect(
|
||||||
|
isPeriodContaining(
|
||||||
|
{
|
||||||
|
start: new Date(Date.UTC(2024, 0, 1)),
|
||||||
|
end: new Date(Date.UTC(2024, 3, 1)),
|
||||||
|
},
|
||||||
|
new Date(Date.UTC(2024, 3, 1))
|
||||||
|
)
|
||||||
|
).toBe(false);
|
||||||
|
});
|
||||||
|
test("period does not contain date before", () => {
|
||||||
|
expect(
|
||||||
|
isPeriodContaining(
|
||||||
|
{
|
||||||
|
start: new Date(Date.UTC(2024, 0, 1)),
|
||||||
|
end: new Date(Date.UTC(2024, 3, 1)),
|
||||||
|
},
|
||||||
|
new Date(Date.UTC(2023, 3, 1))
|
||||||
|
)
|
||||||
|
).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("period does not contain date after", () => {
|
||||||
|
expect(
|
||||||
|
isPeriodContaining(
|
||||||
|
{
|
||||||
|
start: new Date(Date.UTC(2024, 0, 1)),
|
||||||
|
end: new Date(Date.UTC(2024, 3, 1)),
|
||||||
|
},
|
||||||
|
new Date(Date.UTC(2025, 3, 1))
|
||||||
|
)
|
||||||
|
).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
18
src/period/isPeriodContaining.ts
Normal file
18
src/period/isPeriodContaining.ts
Normal file
|
@ -0,0 +1,18 @@
|
||||||
|
import { Period } from "./Period";
|
||||||
|
|
||||||
|
export function isPeriodContaining(
|
||||||
|
period: Period,
|
||||||
|
dateOrPeriod: Date | Period
|
||||||
|
): boolean {
|
||||||
|
if (dateOrPeriod instanceof Date) {
|
||||||
|
return (
|
||||||
|
period.start.getTime() <= dateOrPeriod.getTime() &&
|
||||||
|
dateOrPeriod.getTime() < period.end.getTime()
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
return (
|
||||||
|
period.start.getTime() >= dateOrPeriod.start.getTime() &&
|
||||||
|
dateOrPeriod.end.getTime() <= period.end.getTime()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
21
src/statistiques/ELStats.ts
Normal file
21
src/statistiques/ELStats.ts
Normal file
|
@ -0,0 +1,21 @@
|
||||||
|
export type ELStats = {
|
||||||
|
/** Current Resistants Count */
|
||||||
|
resistantsCount: number;
|
||||||
|
/** Includes Ancient resistants */
|
||||||
|
resistantsOrExCount: number;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resistant count per Year
|
||||||
|
* Family Partially resistant over a year are counted in.
|
||||||
|
*/
|
||||||
|
years: ELPeriodStats[];
|
||||||
|
|
||||||
|
months: ELPeriodStats[];
|
||||||
|
};
|
||||||
|
|
||||||
|
export type ELPeriodStats = {
|
||||||
|
periodId: string;
|
||||||
|
resistantsCount: number;
|
||||||
|
resistantsCountEvol: number;
|
||||||
|
resistantsCountEvolPercent: number;
|
||||||
|
};
|
41
src/statistiques/computeELPeriodStats.ts
Normal file
41
src/statistiques/computeELPeriodStats.ts
Normal file
|
@ -0,0 +1,41 @@
|
||||||
|
import { Family, isResistantOverPeriod } from "../data/Family";
|
||||||
|
import { IdentifiedPeriod } from "../period/IdentifiedPeriod";
|
||||||
|
import { ELPeriodStats } from "./ELStats";
|
||||||
|
|
||||||
|
export function computeELPeriodStats(
|
||||||
|
familles: Family[],
|
||||||
|
periods: IdentifiedPeriod[]
|
||||||
|
): ELPeriodStats[] {
|
||||||
|
const periodStats: ELPeriodStats[] = [];
|
||||||
|
let previousELPeriodStats: ELPeriodStats | null = null;
|
||||||
|
for (const period of periods) {
|
||||||
|
const resistantsCount = familles.filter((famille) =>
|
||||||
|
isResistantOverPeriod(famille, period)
|
||||||
|
).length;
|
||||||
|
|
||||||
|
const stats: ELPeriodStats = {
|
||||||
|
periodId: period.id,
|
||||||
|
resistantsCount,
|
||||||
|
resistantsCountEvol: evol(
|
||||||
|
resistantsCount,
|
||||||
|
previousELPeriodStats?.resistantsCount
|
||||||
|
),
|
||||||
|
resistantsCountEvolPercent: evolPercent(
|
||||||
|
resistantsCount,
|
||||||
|
previousELPeriodStats?.resistantsCount
|
||||||
|
),
|
||||||
|
};
|
||||||
|
periodStats.push(stats);
|
||||||
|
previousELPeriodStats = stats;
|
||||||
|
}
|
||||||
|
return periodStats;
|
||||||
|
}
|
||||||
|
|
||||||
|
function evolPercent(current: number, previous: number | undefined): number {
|
||||||
|
if (previous === undefined) return NaN;
|
||||||
|
return (100 * evol(current, previous)) / previous;
|
||||||
|
}
|
||||||
|
function evol(current: number, previous: number | undefined): number {
|
||||||
|
if (previous === undefined) return NaN;
|
||||||
|
return current - previous;
|
||||||
|
}
|
25
src/statistiques/computeELStats.ts
Normal file
25
src/statistiques/computeELStats.ts
Normal file
|
@ -0,0 +1,25 @@
|
||||||
|
import { Family, isExResistant, isResistant } from "../data/Family";
|
||||||
|
import { ELStats } from "./ELStats";
|
||||||
|
import { computeELPeriodStats } from "./computeELPeriodStats";
|
||||||
|
import { generateELMonths } from "./generateELMonths";
|
||||||
|
import { generateELYears } from "./generateELYears";
|
||||||
|
|
||||||
|
export function computeELStats(families: Family[]): ELStats {
|
||||||
|
const resistantsCount = families.filter(isResistant).length;
|
||||||
|
const resistantsOrExCount = families.filter(
|
||||||
|
(f) => isResistant(f) || isExResistant(f)
|
||||||
|
).length;
|
||||||
|
|
||||||
|
const elYears = generateELYears();
|
||||||
|
const yearsStats = computeELPeriodStats(families, elYears);
|
||||||
|
|
||||||
|
const months = generateELMonths();
|
||||||
|
const monthsStats = computeELPeriodStats(families, months);
|
||||||
|
|
||||||
|
return {
|
||||||
|
resistantsCount,
|
||||||
|
resistantsOrExCount,
|
||||||
|
years: yearsStats,
|
||||||
|
months: monthsStats,
|
||||||
|
};
|
||||||
|
}
|
29
src/statistiques/generateELMonths.ts
Normal file
29
src/statistiques/generateELMonths.ts
Normal file
|
@ -0,0 +1,29 @@
|
||||||
|
import { getMonth, getYear, setMonth, setYear } from "date-fns";
|
||||||
|
import { IdentifiedPeriod } from "../period/IdentifiedPeriod";
|
||||||
|
import { generateConsecutiveIdentifiedPeriods } from "../period/generateConsecutiveIdentifiedPeriods";
|
||||||
|
|
||||||
|
export function generateELMonths(): IdentifiedPeriod[] {
|
||||||
|
const months = generateConsecutiveIdentifiedPeriods({
|
||||||
|
// Start on 2022-07-01 see https://www.notion.so/Liste-de-v-ux-Papa-No-l-de-la-statistique-EL-0f1894c8b16a4fb39b9a759c7446b24b?d=72da4dd0897b4140bb9254e1956d5c6f&pvs=4#a1409542e9974ba2add6e6fe58a70f06
|
||||||
|
start: new Date(Date.UTC(2022, 4, 1)),
|
||||||
|
nextStart: (date) => {
|
||||||
|
// Don't use addMonth because it has an issue
|
||||||
|
// (see https://github.com/date-fns/date-fns/issues/571)
|
||||||
|
const month = getMonth(date);
|
||||||
|
if (month === 11) {
|
||||||
|
return setYear(setMonth(date, 0), getYear(date) + 1);
|
||||||
|
} else {
|
||||||
|
return setMonth(date, month + 1);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
end: new Date(Date.now()),
|
||||||
|
idProvider: (period) =>
|
||||||
|
period.start.getFullYear() +
|
||||||
|
"-" +
|
||||||
|
(period.start.getMonth() + 1).toLocaleString("en-US", {
|
||||||
|
minimumIntegerDigits: 2,
|
||||||
|
useGrouping: false,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
return months;
|
||||||
|
}
|
14
src/statistiques/generateELYears.ts
Normal file
14
src/statistiques/generateELYears.ts
Normal file
|
@ -0,0 +1,14 @@
|
||||||
|
import { addYears } from "date-fns/fp";
|
||||||
|
import { IdentifiedPeriod } from "../period/IdentifiedPeriod";
|
||||||
|
import { generateConsecutiveIdentifiedPeriods } from "../period/generateConsecutiveIdentifiedPeriods";
|
||||||
|
|
||||||
|
export function generateELYears(): IdentifiedPeriod[] {
|
||||||
|
return generateConsecutiveIdentifiedPeriods({
|
||||||
|
// Start on 2022-07-01 see https://www.notion.so/Liste-de-v-ux-Papa-No-l-de-la-statistique-EL-0f1894c8b16a4fb39b9a759c7446b24b?d=72da4dd0897b4140bb9254e1956d5c6f&pvs=4#a1409542e9974ba2add6e6fe58a70f06
|
||||||
|
start: new Date(Date.UTC(2022, 6, 1)),
|
||||||
|
nextStart: addYears(1),
|
||||||
|
end: new Date(Date.now()),
|
||||||
|
idProvider: (period) =>
|
||||||
|
period.start.getFullYear() + "-" + period.end.getFullYear(),
|
||||||
|
});
|
||||||
|
}
|
Loading…
Add table
Reference in a new issue