Add missing filter options (#331)

* Add empty string and NaN checks to null filter

* Add filters for defined and not defined

* Only check that the first value is a primitive type

* Add valid and not valid filters

* Add filters for dates

* Add isValid function and export it

* Pass the correct value to isValid

* Remove unused nd and d filters

* Remove prettierignore file

* Revert "Only check that the first value is a primitive type"

This reverts commit b701de1.

* Remove old comment

* Extract validators into separate functions to avoid redundant checks

* Rename getValidator getTypeValidator

* Add getValidator tests

* Create null/not null filters for valid/invalid column values

* Don't repeatedly re-define source

Co-authored-by: Annie Zhang <annie@observablehq.com>

Co-authored-by: Annie Zhang <annie@observablehq.com>

Author: mkfreeman

src/index.js

@@ -8,5 +8,6 @@ export {
   arrayIsPrimitive,
   isDataArray,
   isDatabaseClient,
-  __table as applyDataTableOperations
+  __table as applyDataTableOperations,
+  getTypeValidator
 } from "./table.js";

src/table.js

@@ -398,7 +398,7 @@ function appendWhereEntry({type, operands}, args, escaper) {
   if (operands.length === 2) {
     if (["in", "nin"].includes(type)) {
       // Fallthrough to next parent block.
-    } else if (["c", "nc"].includes(type)) {
+    } else if (["c", "nc", "v", "nv"].includes(type)) {
       // TODO: Case (in)sensitive?
       appendOperand(operands[0], args, escaper);
       switch (type) {
@@ -408,6 +408,14 @@ function appendWhereEntry({type, operands}, args, escaper) {
         case "nc":
           appendSql(` NOT LIKE `, args);
           break;
+        // JavaScript "not valid" filter translate to a SQL "IS NULL"
+        case "nv":
+          appendSql(` IS NULL`, args);
+          break;
+        // JavaScript "valid" filter translate to a SQL "IS NOT NULL"
+        case "v":
+          appendSql(` IS NOT NULL`, args);
+          break;
       }
       appendOperand(likeOperand(operands[1]), args, escaper);
       return;
@@ -480,6 +488,42 @@ function likeOperand(operand) {
   return {...operand, value: `%${operand.value}%`};
 }
 
+// Functions for checking type validity
+const isValidNumber = (value) => typeof value === "number" && !Number.isNaN(value);
+const isValidString = (value) => typeof value === "string";
+const isValidBoolean = (value) => typeof value === "boolean";
+const isValidBigint = (value) => typeof value === "bigint";
+const isValidDate = (value) => value instanceof Date && !isNaN(value);
+const isValidBuffer = (value) => value instanceof ArrayBuffer;
+const isValidArray = (value) => Array.isArray(value);
+const isValidObject = (value) => typeof value === "object" && value !== null;
+const isValidOther = (value) => value != null;
+
+// Function to get the correct validity checking function based on type
+export function getTypeValidator(colType) {
+  switch (colType) {
+    case "string":
+      return isValidString;
+    case "bigint":
+      return isValidBigint;
+    case "boolean":
+      return isValidBoolean;
+    case "number":
+      return isValidNumber;
+    case "date":
+      return isValidDate;
+    case "buffer":
+      return isValidBuffer;
+    case "array":
+      return isValidArray;
+    case "object":
+      return isValidObject;
+    case "other":
+    default:
+      return isValidOther;
+  }
+}
+
 // This function applies table cell operations to an in-memory table (array of
 // objects); it should be equivalent to the corresponding SQL query. TODO Use
 // DuckDBClient for data arrays, too, and then we wouldn’t need our own __table
@@ -493,6 +537,20 @@ export function __table(source, operations) {
     const [{value: column}] = operands;
     const values = operands.slice(1).map(({value}) => value);
     switch (type) {
+      // valid (matches the column type)
+      case "v": {
+        const [colType] = values;
+        const isValid = getTypeValidator(colType);
+        source = source.filter(d => isValid(d[column]));
+        break;
+      }
+      // not valid (doesn't match the column type)
+      case "nv": {
+        const [colType] = values;
+        const isValid = getTypeValidator(colType);
+        source = source.filter(d => !isValid(d[column]));
+        break;
+      }
       case "eq": {
         const [value] = values;
         if (value instanceof Date) {

test/table-test.js

@@ -1,4 +1,4 @@
-import {makeQueryTemplate, __table} from "../src/table.js";
+import {getTypeValidator, makeQueryTemplate, __table} from "../src/table.js";
 import assert from "assert";
 
 export const EMPTY_TABLE_DATA = {
@@ -542,3 +542,85 @@ describe("__table", () => {
     );
   });
 });
+
+describe("getTypeValidator filters accurately", () => {
+  let source = [
+    {label: "string", value: "string"},
+    {label: "object", value: {}},
+    {label: "buffer", value: new ArrayBuffer()},
+    {label: "boolean", value: true},
+    {label: "array", value: [1, 2, 3]},
+    {label: "number", value: 10},
+    {label: "date", value: new Date(1)},
+    // eslint-disable-next-line no-undef
+    {label: "bigint", value: BigInt(10)},
+    {label: "null", value: null},
+    {label: "NaN", value: NaN},
+    {label: "undefined"}
+   ];
+
+  it("filters strings", () => {
+    const isValid = getTypeValidator("string");
+    assert.deepStrictEqual(source.filter(d => isValid(d.value)), [{label: "string", value: "string"}]);
+  });
+
+  it("filters buffers", () => {
+    const isValid = getTypeValidator("buffer");
+    assert.deepStrictEqual(source.filter(d => isValid(d.value)), [{label: "buffer", value: new ArrayBuffer()}]);
+  });
+
+  it("filters numbers", () => {
+    const isValid = getTypeValidator("number");
+    assert.deepStrictEqual(source.filter(d => isValid(d.value)), [{label: "number", value: 10}]);
+  });
+
+  it("filters booleans", () => {
+    const isValid = getTypeValidator("boolean");
+    assert.deepStrictEqual(source.filter(d => isValid(d.value)), [{label: "boolean", value: true}]);
+  });
+
+  it("filters arrays", () => {
+    const isValid = getTypeValidator("array");
+    assert.deepStrictEqual(source.filter(d => isValid(d.value)), [{label: "array", value: [1, 2, 3]}]);
+  });
+
+  it("filters dates", () => {
+    const isValid = getTypeValidator("date");
+    assert.deepStrictEqual(source.filter(d => isValid(d.value)), [{label: "date", value: new Date(1)}]);
+  });
+
+  it("filters BigInts", () => {
+    const isValid = getTypeValidator("bigint");
+    // eslint-disable-next-line no-undef
+    assert.deepStrictEqual(source.filter(d => isValid(d.value)), [{label: "bigint", value: BigInt(10)}]);
+  });
+
+  it("filters objects", () => {
+    const isValid = getTypeValidator("object");
+    assert.deepStrictEqual(source.filter(d => isValid(d.value)),
+      [
+        {label: "object", value: {}},
+        {label: "buffer", value: new ArrayBuffer()},
+        {label: "array", value: [1, 2, 3]},
+        {label: "date", value: new Date(1)}]
+    );
+  });
+
+  it("filters other", () => {
+    const isValid = getTypeValidator("other");
+    assert.deepStrictEqual(source.filter(d => isValid(d.value)),
+    [
+      {label: "string", value: "string"},
+      {label: "object", value: {}},
+      {label: "buffer", value: new ArrayBuffer()},
+      {label: "boolean", value: true},
+      {label: "array", value: [1, 2, 3]},
+      {label: "number", value: 10},
+      {label: "date", value: new Date(1)},
+      // eslint-disable-next-line no-undef
+      {label: "bigint", value: BigInt(10)},
+      {label: "NaN", value: NaN}
+    ]
+    );
+  });
+});