feat: cache basique

.gitignore

@@ -19,3 +19,4 @@ node_modules
 # outptu stats files
 el-stats*.json
 el-stats*.txt
+notion-data-cache

src/notion/utils/queryAllDbResults.ts

@@ -1,21 +1,96 @@
 import { Client } from "@notionhq/client";
+import { createHash } from "crypto";
+import { stat, mkdir, writeFile, readFile } from "fs";
+
 import {
   QueryDatabaseParameters,
   QueryDatabaseResponse,
 } from "@notionhq/client/build/src/api-endpoints";
+import { promisify } from "util";
+import { dirname } from "path";
+
+type QueryExtraOptions = {
+  cache: CacheConfig;
+};
+
+type CacheConfig =
+  | boolean
+  | {
+      ttl: number;
+    };
 
 export async function queryAllDbResults(
   notion: Client,
-  dbQuery: QueryDatabaseParameters
+  dbQuery: QueryDatabaseParameters,
+  { cache: cacheConfig }: QueryExtraOptions = { cache: false }
 ): Promise<QueryDatabaseResponse["results"]> {
+  const queryHash = createHash("sha1")
+    .update(JSON.stringify(dbQuery))
+    .digest("hex");
+  const cacheEntryFileName =
+    "./notion-data-cache/queryAllDbResults/" +
+    dbQuery.database_id +
+    "-" +
+    queryHash +
+    ".json";
+  if (await shouldReadCacheEntry(cacheConfig, cacheEntryFileName)) {
+    console.log("reading from cache entry " + cacheEntryFileName);
+    return await readCacheEntry(cacheEntryFileName);
+  }
   const dbResponse = await notion.databases.query(dbQuery);
+  let fullResponse = [];
   if (dbResponse.has_more && dbResponse.next_cursor) {
     const moreResults = await queryAllDbResults(notion, {
       ...dbQuery,
       start_cursor: dbResponse.next_cursor,
     });
-    return [...dbResponse.results, ...moreResults];
+    fullResponse = [...dbResponse.results, ...moreResults];
   } else {
-    return dbResponse.results;
+    fullResponse = dbResponse.results;
+  }
+  if (await shouldWriteCacheEntry(cacheConfig)) {
+    await writeCacheEntry(cacheEntryFileName, fullResponse);
+  }
+  return fullResponse;
+}
+
+async function shouldReadCacheEntry(
+  cacheConfig: CacheConfig,
+  cacheEntryFileName: string
+): Promise<boolean> {
+  if (cacheConfig === false) return false;
+  try {
+    const fileStat = await promisify(stat)(cacheEntryFileName);
+    if (cacheConfig === true) {
+      return true;
+    }
+    const lastUpdateAge = new Date().getTime() - fileStat.mtime.getTime();
+    return lastUpdateAge < cacheConfig.ttl;
+  } catch (e) {
+    // Cannot get file stat => it probably doesn't exist yet
+    return false;
   }
 }
+
+async function shouldWriteCacheEntry(
+  cacheConfig: CacheConfig
+): Promise<boolean> {
+  return cacheConfig !== false;
+}
+
+async function writeCacheEntry(
+  cacheEntryFileName: string,
+  response: QueryDatabaseResponse["results"]
+): Promise<void> {
+  await promisify(mkdir)(dirname(cacheEntryFileName), { recursive: true });
+
+  await promisify(writeFile)(cacheEntryFileName, JSON.stringify(response));
+}
+
+async function readCacheEntry(
+  cacheEntryFileName: string
+): Promise<QueryDatabaseResponse["results"]> {
+  const fileBuffer: Buffer = await promisify(readFile)(cacheEntryFileName);
+  const parsed = JSON.parse(fileBuffer.toString());
+  return parsed as QueryDatabaseResponse["results"];
+}