test: store nocks in a sqlite database (#374)

* test: store nocks in a sqlite3 database

* chore: remove nock files

* refactor: reuse statement

* Update tests/recordRequests.js

Co-authored-by: Antoine du Hamel <duhamelantoine1995@gmail.com>

* refactor: only create statements when needed

* refactor: move nocks db up one level

* fix: close db on exit

---------

Co-authored-by: Antoine du Hamel <duhamelantoine1995@gmail.com>

.github/workflows/update-nock-files.yml

@@ -23,9 +23,7 @@ jobs:
       - name: Install Node
         uses: actions/setup-node@v4
         with:
-          # Let's use the oldest version supported to be sure the V8
-          # serialization is compatible with all supported versions.
-          node-version: 18.x
+          node-version: lts/*
 
       - name: Get the Yarn cache directory path
         id: yarn-cache-dir-path
@@ -43,7 +41,7 @@ jobs:
       - run: corepack yarn build # We need the stubs to run the tests
 
       - name: Remove old Nock files to avoid conflicts
-        run: rm -r tests/nock
+        run: rm tests/nocks.db
 
       - run: corepack yarn test
         env:
@@ -51,13 +49,13 @@ jobs:
 
       - name: Check if anything has changed
         id: contains-changes
-        run: echo "result=$(git --no-pager diff --quiet -- tests/nock || echo "yes")" >> $GITHUB_OUTPUT
+        run: echo "result=$(git --no-pager diff --quiet -- tests/nocks.db || echo "yes")" >> $GITHUB_OUTPUT
         shell: bash
 
       - name: Commit changes
         if: ${{ steps.contains-changes.outputs.result == 'yes' }}
         run: |
-          git add tests/nock/
+          git add tests/nocks.db
           git config --global user.email "actions@github.com"
           git config --global user.name "GitHub Actions"
           git commit -m "update Nock files"
@@ -72,10 +70,10 @@ jobs:
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Upload `tests/nock` in case of failure
+      - name: Upload `tests/nocks.db` in case of failure
         uses: actions/upload-artifact@v4
         if: ${{ failure() && steps.contains-changes.outputs.result == 'yes' }}
         with:
           name: nock
           path: |
-            tests/nock
+            tests/nocks.db

package.json

@@ -35,6 +35,7 @@
     "@yarnpkg/fslib": "^3.0.0-rc.48",
     "@zkochan/cmd-shim": "^6.0.0",
     "babel-plugin-dynamic-import-node": "^2.3.3",
+    "better-sqlite3": "^9.4.1",
     "clipanion": "^3.0.1",
     "debug": "^4.1.1",
     "esbuild": "0.19.5",

tests/_runCli.ts

@@ -6,8 +6,6 @@ export async function runCli(cwd: PortablePath, argv: Array<string>): Promise<{e
   const err: Array<Buffer> = [];
 
   return new Promise((resolve, reject) => {
-    if (process.env.RUN_CLI_ID)
-      (process.env.RUN_CLI_ID as any)++;
     const child = spawn(process.execPath, [`--no-warnings`, `-r`, require.resolve(`./recordRequests.js`), require.resolve(`../dist/corepack.js`), ...argv], {
       cwd: npath.fromPortablePath(cwd),
       env: process.env,

tests/recordRequests.js

@@ -1,21 +1,48 @@
 "use strict";
-const fs = require(`node:fs`);
 const path = require(`node:path`);
-const v8 = require(`node:v8`);
+const crypto = require(`node:crypto`);
+const SQLite3 = require(`better-sqlite3`);
+
+const db = new SQLite3(path.join(__dirname, `nocks.db`));
+process.once(`exit`, () => {
+  db.close();
+});
+
+db.exec(`CREATE TABLE IF NOT EXISTS nocks (
+  hash BLOB PRIMARY KEY NOT NULL,
+  body BLOB NOT NULL,
+  headers BLOB NOT NULL,
+  status INTEGER NOT NULL
+)`);
+
 
 /**
- * @type {Map<string, {body: string, status:number, headers: Record<string,string>}> | undefined}
+ * @param {string | URL} input
+ * @param {RequestInit | undefined} init
  */
-let mocks;
+function getRequestHash(input, init) {
+  const hash = crypto.createHash(`sha256`);
+  hash.update(`${input}\0`);
 
-const getNockFile = () =>
-  path.join(
-    __dirname,
-    `nock`,
-    `${process.env.NOCK_FILE_NAME}-${process.env.RUN_CLI_ID}.dat`,
-  );
+  if (init) {
+    for (const key in init) {
+      if (init[key] === undefined) continue;
+
+      switch (key) {
+        case `headers`:
+          hash.update(`${JSON.stringify(Object.fromEntries(new Headers(init.headers || {})))}\0`);
+          break;
+        default:
+          throw new Error(`Hashing for "${key}" not implemented`);
+      }
+    }
+  }
+
+  return hash.digest();
+}
 
 if (process.env.NOCK_ENV === `record`) {
+  const insertNockStatement = db.prepare(`INSERT OR REPLACE INTO nocks (hash, body, headers, status) VALUES (?, ?, jsonb(?), ?)`);
   const realFetch = globalThis.fetch;
   globalThis.fetch = async (input, init) => {
     const response = await realFetch(input, init);
@@ -29,70 +56,31 @@ if (process.env.NOCK_ENV === `record`) {
       }
     }
 
-    mocks ??= new Map();
-    mocks.set(input.toString(), {
-      /*
-        Due to a bug *somewhere* `v8.deserialize` fails to deserialize
-        a `v8.serialize` Buffer if body is a Buffer, Uint8Array,
-        ArrayBuffer, or latin1 string on the Windows GitHub Actions
-        runner with the following error:
-          Unable to deserialize cloned data
-
-        base64 strings works so that's what we'll use for now.
-
-        Tested with Node.js 18.19.0, 20.11.0, and 21.6.1.
-
-        Runner Information:
-          Current runner version: '2.312.0'
-          Operating System
-            Microsoft Windows Server 2022
-            10.0.20348
-            Datacenter
-          Runner Image
-            Image: windows-2022
-            Version: 20240204.1.0
-            Included Software: https://github.com/actions/runner-images/blob/win22/20240204.1/images/windows/Windows2022-Readme.md
-            Image Release: https://github.com/actions/runner-images/releases/tag/win22%2F20240204.1
-          Runner Image Provisioner
-            2.0.341.1
-      */
-      body: Buffer.from(data).toString(`base64`),
-      status: response.status,
-      headers: Object.fromEntries(minimalHeaders),
-    });
+    const requestHash = getRequestHash(input, init);
+    insertNockStatement.run(
+      requestHash,
+      Buffer.from(data),
+      JSON.stringify(Object.fromEntries(minimalHeaders)),
+      response.status,
+    );
 
     return new Response(data, {
       status: response.status,
       headers: minimalHeaders,
     });
   };
-
-  process.once(`exit`, () => {
-    if (mocks) {
-      fs.mkdirSync(path.dirname(getNockFile()), {recursive: true});
-      fs.writeFileSync(getNockFile(), v8.serialize(mocks));
-    }
-  });
 } else if (process.env.NOCK_ENV === `replay`) {
-  globalThis.fetch = async (input, init) => {
-    try {
-      mocks ??= v8.deserialize(fs.readFileSync(getNockFile()));
-    } catch (error) {
-      if (error.code === `ENOENT`) {
-        throw new Error(
-          `No nock file found for this test run; run the tests with NOCK_ENV=record to generate one`,
-          {cause: error},
-        );
-      }
-      throw error;
-    }
+  const getNockStatement = db.prepare(`SELECT body, json(headers) as headers, status FROM nocks WHERE hash = ?`);
 
-    const mock = mocks.get(input.toString());
+  globalThis.fetch = async (input, init) => {
+    const requestHash = getRequestHash(input, init);
+
+    const mock = getNockStatement.get(requestHash);
     if (!mock) throw new Error(`No mock found for ${input}; run the tests with NOCK_ENV=record to generate one`);
 
-    return new Response(Buffer.from(mock.body, `base64`), {
+    return new Response(mock.body, {
       status: mock.status,
-      headers: mock.headers,
+      headers: JSON.parse(mock.headers),
     });
   };
 }

tests/setupTests.js

@@ -1,6 +1,5 @@
-/* global jest, expect, beforeEach, afterAll */
+/* global jest, beforeEach, afterAll */
 
-const crypto = require(`crypto`);
 const process = require(`process`);
 
 jest.retryTimes(2, {logErrorsBeforeRetry: true});
@@ -14,31 +13,9 @@ const processEnv = Object.fromEntries(
   ),
 );
 
-switch (process.env.NOCK_ENV || ``) {
-  case `record`:
-  case `replay`:
-    beforeEach(() => {
-      process.env = {
-        ...processEnv,
-        RUN_CLI_ID: `0`,
-        NOCK_FILE_NAME: crypto
-          .createHash(`md5`)
-          .update(expect.getState().currentTestName)
-          .digest(`base64url`),
-      };
-    });
-    break;
-
-  case ``: {
-    beforeEach(() => {
-      process.env = {...processEnv};
-    });
-  } break;
-
-  default: {
-    throw new Error(`Invalid NOCK_ENV variable`);
-  }
-}
+beforeEach(() => {
+  process.env = {...processEnv};
+});
 
 afterAll(() => {
   process.env = OLD_ENV;