feat: add CSV conversion command to ensrainbow CLI

.changeset/brave-kiwis-notice.md

@@ -0,0 +1,5 @@
+---
+"ensrainbow": patch
+---
+
+feat: add CSV conversion command to ensrainbow CLI to convert rainbow tables from CSV format to ensrainbow format

apps/ensrainbow/package.json

@@ -19,6 +19,7 @@
     "validate:lite": "tsx src/cli.ts validate --lite",
     "purge": "tsx src/cli.ts purge",
     "convert": "tsx src/cli.ts convert",
+    "convert-sql": "tsx src/cli.ts convert-sql",
     "test": "vitest",
     "test:coverage": "vitest --coverage",
     "lint": "biome check --write .",
@@ -38,7 +39,8 @@
     "progress": "^2.0.3",
     "protobufjs": "^7.4.0",
     "viem": "catalog:",
-    "yargs": "^17.7.2"
+    "yargs": "^17.7.2",
+    "@fast-csv/parse": "^5.0.0"
   },
   "devDependencies": {
     "@ensnode/shared-configs": "workspace:*",

apps/ensrainbow/src/cli.test.ts

@@ -111,7 +111,7 @@ describe("CLI", () => {
 
         expect(() =>
           cli.parse([
-            "convert",
+            "convert-sql",
             "--input-file",
             sqlInputFile,
             "--output-file",
@@ -122,7 +122,7 @@ describe("CLI", () => {
         // Successful convert with args
         const ingestCli = createCLI({ exitProcess: false });
         await ingestCli.parse([
-          "convert",
+          "convert-sql",
           "--input-file",
           sqlInputFile,
           "--output-file",
@@ -132,7 +132,7 @@ describe("CLI", () => {
           "--label-set-version",
           labelSetVersion.toString(),
         ]);
-        //command: pnpm convert --input-file test/fixtures/test_ens_names.sql.gz --output-file test/fixtures/test_ens_names_0.ensrainbow --label-set-id test-ens-names --label-set-version 0
+        //command: pnpm convert-sql --input-file test/fixtures/test_ens_names.sql.gz --output-file test/fixtures/test_ens_names_0.ensrainbow --label-set-id test-ens-names --label-set-version 0
         //verify that the file is created
 
         await expect(stat(ensrainbowOutputFile)).resolves.toBeDefined();
@@ -163,7 +163,7 @@ describe("CLI", () => {
 
         expect(() =>
           cli.parse([
-            "convert",
+            "convert-sql",
             "--input-file",
             sqlInputFile,
             "--output-file",
@@ -174,7 +174,7 @@ describe("CLI", () => {
         // Successful convert with args
         const ingestCli = createCLI({ exitProcess: false });
         await ingestCli.parse([
-          "convert",
+          "convert-sql",
           "--input-file",
           sqlInputFile,
           "--output-file",
@@ -184,7 +184,7 @@ describe("CLI", () => {
           "--label-set-version",
           labelSetVersion.toString(),
         ]);
-        //command: pnpm convert --input-file test_ens_names.sql.gz --output-file test_ens_names_0.ensrainbow --label-set-id test-ens-names --label-set-version 0
+        //command: pnpm convert-sql --input-file test_ens_names.sql.gz --output-file test_ens_names_0.ensrainbow --label-set-id test-ens-names --label-set-version 0
         //verify that the file is created
 
         await expect(stat(ensrainbowOutputFile)).resolves.toBeDefined();
@@ -211,7 +211,7 @@ describe("CLI", () => {
 
         expect(() =>
           cli.parse([
-            "convert",
+            "convert-sql",
             "--input-file",
             sqlInputFile,
             "--output-file",
@@ -222,7 +222,7 @@ describe("CLI", () => {
         const ingestCli2 = createCLI({ exitProcess: false });
         // Successful convert with args
         await ingestCli2.parse([
-          "convert",
+          "convert-sql",
           "--input-file",
           sqlInputFile,
           "--output-file",
@@ -266,7 +266,7 @@ describe("CLI", () => {
         // Successful convert with label set version 2
         const convertCli = createCLI({ exitProcess: false });
         await convertCli.parse([
-          "convert",
+          "convert-sql",
           "--input-file",
           sqlInputFile,
           "--output-file",
@@ -318,7 +318,7 @@ describe("CLI", () => {
         // Create second file with different label set id and label set version 0
         const convertCli = createCLI({ exitProcess: false });
         await convertCli.parse([
-          "convert",
+          "convert-sql",
           "--input-file",
           sqlInputFile,
           "--output-file",
@@ -331,7 +331,7 @@ describe("CLI", () => {
 
         // Create third file with different label set id and label set version 1
         await convertCli.parse([
-          "convert",
+          "convert-sql",
           "--input-file",
           sqlInputFile,
           "--output-file",

apps/ensrainbow/src/cli.ts

@@ -13,6 +13,7 @@ import {
 } from "@ensnode/ensnode-sdk";
 
 import { convertCommand } from "@/commands/convert-command";
+import { convertCsvCommand } from "@/commands/convert-csv-command";
 // import { ingestCommand } from "@/commands/ingest-command";
 import { ingestProtobufCommand } from "@/commands/ingest-protobuf-command";
 import { purgeCommand } from "@/commands/purge-command";
@@ -56,11 +57,20 @@ interface PurgeArgs {
 
 interface ConvertArgs {
   "input-file": string;
-  "output-file": string;
+  "output-file"?: string;
   "label-set-id": LabelSetId;
   "label-set-version": LabelSetVersion;
 }
 
+interface ConvertCsvArgs {
+  "input-file": string;
+  "output-file"?: string;
+  "label-set-id": LabelSetId;
+  "progress-interval"?: number;
+  "existing-db-path"?: string;
+  silent?: boolean;
+}
+
 export interface CLIOptions {
   exitProcess?: boolean;
 }
@@ -184,36 +194,86 @@ export function createCLI(options: CLIOptions = {}) {
       )
       .command(
         "convert",
-        "Convert rainbow tables from SQL dump to protobuf format",
+        "Convert rainbow tables from CSV format to ensrainbow format",
         (yargs: Argv) => {
           return yargs
             .option("input-file", {
               type: "string",
-              description: "Path to the gzipped SQL dump file",
-              default: join(process.cwd(), "ens_names.sql.gz"),
+              description: "Path to the CSV input file",
+              demandOption: true,
+            })
+            .option("label-set-id", {
+              type: "string",
+              description: "Label set id for the generated ensrainbow file",
+              demandOption: true,
             })
+            .coerce("label-set-id", buildLabelSetId)
             .option("output-file", {
               type: "string",
-              description: "Path to the output protobuf file",
-              default: join(process.cwd(), "rainbow-records.ensrainbow"),
+              description:
+                "Path to where the resulting ensrainbow file will be output (if not provided, will be generated automatically)",
+            })
+            .option("progress-interval", {
+              type: "number",
+              description: "Number of records to process before logging progress",
+              default: 50000,
+            })
+            .option("existing-db-path", {
+              type: "string",
+              description:
+                "Path to existing ENSRainbow database to filter out existing labels and determine the next label set version (if not provided, version will be 0)",
+            })
+            .option("silent", {
+              type: "boolean",
+              description: "Disable progress bar (useful for scripts)",
+              default: false,
+            });
+        },
+        async (argv: ArgumentsCamelCase<ConvertCsvArgs>) => {
+          await convertCsvCommand({
+            inputFile: argv["input-file"],
+            outputFile: argv["output-file"],
+            labelSetId: argv["label-set-id"],
+            progressInterval: argv["progress-interval"],
+            existingDbPath: argv["existing-db-path"],
+            silent: argv["silent"],
+          });
+        },
+      )
+      .command(
+        "convert-sql",
+        "Convert rainbow tables from legacy SQL dump to ensrainbow format",
+        (yargs: Argv) => {
+          return yargs
+            .option("input-file", {
+              type: "string",
+              description: "Path to the gzipped SQL dump file",
+              default: join(process.cwd(), "ens_names.sql.gz"),
             })
             .option("label-set-id", {
               type: "string",
-              description: "Label set id for the rainbow record collection",
+              description: "Label set id for the generated ensrainbow file",
               demandOption: true,
             })
             .coerce("label-set-id", buildLabelSetId)
             .option("label-set-version", {
               type: "number",
-              description: "Label set version for the rainbow record collection",
+              description: "Label set version for the generated ensrainbow file",
               demandOption: true,
             })
-            .coerce("label-set-version", buildLabelSetVersion);
+            .coerce("label-set-version", buildLabelSetVersion)
+            .option("output-file", {
+              type: "string",
+              description: "Path to where the resulting ensrainbow file will be output",
+            });
         },
         async (argv: ArgumentsCamelCase<ConvertArgs>) => {
+          const outputFile =
+            argv["output-file"] ??
+            join(process.cwd(), `${argv["label-set-id"]}_${argv["label-set-version"]}.ensrainbow`);
           await convertCommand({
             inputFile: argv["input-file"],
-            outputFile: argv["output-file"],
+            outputFile,
             labelSetId: argv["label-set-id"],
             labelSetVersion: argv["label-set-version"],
           });