Bolt 6.0 and Vector Type (#1293)

Introduces the Neo4j Vector type, a wrapper for JavaScript TypedArrays, allowing Vectors to be sent to and from the database.

To support these, the bolt protocol version 6.0 is introduced.

Author: MaxAake

README.md

@@ -503,3 +503,29 @@ var driver = neo4j.driver(
   { disableLosslessIntegers: true }
 )
 ```
+
+#### Writing and reading Vectors
+
+Neo4j supports storing vector embeddings in a dedicated vector type. Sending large lists with the driver will result in significant overhead as each value will be transmitted with type information, so the 6.0.0 release of the driver introduced the Neo4j Vector type.
+
+The Vector type supports signed integers of 8, 16, 32 and 64 bits, and floats of 32 and 64 bits. The Vector type is a wrapper for JavaScript TypedArrays of those types.
+
+To create a neo4j Vector in your code, do the following:
+
+```javascript
+var neo4j = require('neo4j-driver')
+
+var typedArray = Float32Array.from([1, 2, 3]) //this is how to convert a regular array of numbers into a TypedArray, useful if you handle vectors as regular arrays in your code
+
+var neo4jVector = neo4j.vector(typedArray) //this creates a neo4j Vector of type Float32, containing the values [1, 2, 3]
+
+driver.executeQuery('CREATE (n {embeddings: $myVectorParam})', { myVectorParam: neo4jVector })
+```
+
+To access the data in a retrieved Vector you can do the following:
+
+```javascript
+var retrievedTypedArray = neo4jVector.asTypedArray() //This will return a TypedArray of the same type as the Vector
+
+var retrievedArray = Array.from(retrievedTypedArray) //This will convert the TypedArray to a regular array of Numbers. (Not safe for Int64 arrays)
+```

packages/bolt-connection/src/bolt/bolt-protocol-v1.transformer.js

@@ -23,7 +23,8 @@ import {
   UnboundRelationship,
   Path,
   toNumber,
-  PathSegment
+  PathSegment,
+  Vector
 } from 'neo4j-driver-core'
 
 import { structure } from '../packstream'
@@ -43,6 +44,8 @@ const UNBOUND_RELATIONSHIP_STRUCT_SIZE = 3
 const PATH = 0x50
 const PATH_STRUCT_SIZE = 3
 
+const VECTOR = 0x56
+
 /**
  * Creates the Node Transformer
  * @returns {TypeTransformer}
@@ -177,9 +180,27 @@ function createPathTransformer () {
   })
 }
 
+/**
+ * Creates a typeTransformer that throws errors if vectors are transmitted.
+ * @returns {TypeTransformer}
+ */
+function createVectorTransformer () {
+  return new TypeTransformer({
+    signature: VECTOR,
+    isTypeInstance: object => object instanceof Vector,
+    toStructure: _ => {
+      throw newError('Sending vector types require server and driver to be communicating with Bolt protocol 6.0 or later. Please update your database version.')
+    },
+    fromStructure: _ => {
+      throw newError('Server tried to send Vector object, but server and driver are communicating on a version of the Bolt protocol that does not support vectors.')
+    }
+  })
+}
+
 export default {
   createNodeTransformer,
   createRelationshipTransformer,
   createUnboundRelationshipTransformer,
-  createPathTransformer
+  createPathTransformer,
+  createVectorTransformer
 }

packages/bolt-connection/src/bolt/bolt-protocol-v6x0.js

@@ -0,0 +1,39 @@
+/**
+ * Copyright (c) "Neo4j"
+ * Neo4j Sweden AB [https://neo4j.com]
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import BoltProtocolV5x8 from './bolt-protocol-v5x8'
+
+import transformersFactories from './bolt-protocol-v6x0.transformer'
+import Transformer from './transformer'
+
+import { internal } from 'neo4j-driver-core'
+
+const {
+  constants: { BOLT_PROTOCOL_V6_0 }
+} = internal
+
+export default class BoltProtocol extends BoltProtocolV5x8 {
+  get version () {
+    return BOLT_PROTOCOL_V6_0
+  }
+
+  get transformer () {
+    if (this._transformer === undefined) {
+      this._transformer = new Transformer(Object.values(transformersFactories).map(create => create(this._config, this._log)))
+    }
+    return this._transformer
+  }
+}

packages/bolt-connection/src/bolt/bolt-protocol-v6x0.transformer.js

@@ -0,0 +1,138 @@
+/**
+ * Copyright (c) "Neo4j"
+ * Neo4j Sweden AB [https://neo4j.com]
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import v5x8 from './bolt-protocol-v5x8.transformer'
+import { TypeTransformer } from './transformer'
+import { structure } from '../packstream'
+import { Vector, newError } from 'neo4j-driver-core'
+const VECTOR = 0x56
+const FLOAT_32 = 0xc6
+const FLOAT_64 = 0xc1
+const INT_8 = 0xc8
+const INT_16 = 0xc9
+const INT_32 = 0xca
+const INT_64 = 0xcb
+
+const typeToTypeMarker = {
+  INT8: INT_8,
+  INT16: INT_16,
+  INT32: INT_32,
+  INT64: INT_64,
+  FLOAT32: FLOAT_32,
+  FLOAT64: FLOAT_64
+}
+
+function createVectorTransformer () {
+  return new TypeTransformer({
+    signature: VECTOR,
+    isTypeInstance: object => object instanceof Vector,
+    toStructure: vector => {
+      const typeMarker = typeToTypeMarker[vector.getType()]
+      if (typeMarker === undefined) {
+        throw newError(`Vector object has unknown type: ${vector.getType()}`)
+      }
+      const buffer = fixBufferEndianness(typeMarker, vector.asTypedArray().buffer)
+      const struct = new structure.Structure(VECTOR, [Int8Array.from([typeMarker]), new Int8Array(buffer)])
+      return struct
+    },
+    fromStructure: structure => {
+      const typeMarker = Uint8Array.from(structure.fields[0])[0]
+      const byteArray = structure.fields[1]
+      const buffer = fixBufferEndianness(typeMarker, byteArray.buffer)
+      switch (typeMarker) {
+        case INT_8:
+          return new Vector(new Int8Array(buffer))
+        case INT_16:
+          return new Vector(new Int16Array(buffer))
+        case INT_32:
+          return new Vector(new Int32Array(buffer))
+        case INT_64:
+          return new Vector(new BigInt64Array(buffer))
+        case FLOAT_32:
+          return new Vector(new Float32Array(buffer))
+        case FLOAT_64:
+          return new Vector(new Float64Array(buffer))
+        default:
+          throw newError(`Received Vector structure with unsupported type marker: ${typeMarker}`)
+      }
+    }
+  })
+}
+
+function fixBufferEndianness (typeMarker, buffer) {
+  const isLittleEndian = checkLittleEndian()
+  if (isLittleEndian) {
+    const setview = new DataView(new ArrayBuffer(buffer.byteLength))
+    // we want exact byte accuracy, so we cannot simply get the value from the typed array
+    const getview = new DataView(buffer)
+    let set
+    let get
+    let elementSize
+    switch (typeMarker) {
+      case INT_8:
+        elementSize = 1
+        set = setview.setInt8.bind(setview)
+        get = getview.getInt8.bind(getview)
+        break
+      case INT_16:
+        elementSize = 2
+        set = setview.setInt16.bind(setview)
+        get = getview.getInt16.bind(getview)
+        break
+      case INT_32:
+        elementSize = 4
+        set = setview.setInt32.bind(setview)
+        get = getview.getInt32.bind(getview)
+        break
+      case INT_64:
+        elementSize = 8
+        set = setview.setBigInt64.bind(setview)
+        get = getview.getBigInt64.bind(getview)
+        break
+      case FLOAT_32:
+        elementSize = 4
+        set = setview.setInt32.bind(setview)
+        get = getview.getInt32.bind(getview)
+        break
+      case FLOAT_64:
+        elementSize = 8
+        set = setview.setBigInt64.bind(setview)
+        get = getview.getBigInt64.bind(getview)
+        break
+      default:
+        throw newError(`Vector is of unsupported type ${typeMarker}`)
+    }
+    for (let i = 0; i < buffer.byteLength; i += elementSize) {
+      set(i, get(i, isLittleEndian))
+    }
+    return setview.buffer
+  } else {
+    return buffer
+  }
+}
+
+function checkLittleEndian () {
+  const dataview = new DataView(new ArrayBuffer(2))
+  dataview.setInt16(0, 1000, true)
+  const typeArray = new Int16Array(dataview.buffer)
+  return typeArray[0] === 1000
+}
+
+export default {
+  ...v5x8,
+  createVectorTransformer
+}

packages/bolt-connection/src/bolt/create.js

@@ -33,6 +33,7 @@ import BoltProtocolV5x5 from './bolt-protocol-v5x5'
 import BoltProtocolV5x6 from './bolt-protocol-v5x6'
 import BoltProtocolV5x7 from './bolt-protocol-v5x7'
 import BoltProtocolV5x8 from './bolt-protocol-v5x8'
+import BoltProtocolV6x0 from './bolt-protocol-v6x0'
 // eslint-disable-next-line no-unused-vars
 import { Chunker, Dechunker } from '../channel'
 import ResponseHandler from './response-handler'
@@ -266,6 +267,14 @@ function createProtocol (
         log,
         onProtocolError,
         serversideRouting)
+    case 6.0:
+      return new BoltProtocolV6x0(server,
+        chunker,
+        packingConfig,
+        createResponseHandler,
+        log,
+        onProtocolError,
+        serversideRouting)
     default:
       throw newError('Unknown Bolt protocol version: ' + version)
   }

packages/bolt-connection/src/bolt/handshake.js

@@ -19,7 +19,7 @@ import { alloc } from '../channel'
 import { newError } from 'neo4j-driver-core'
 
 const BOLT_MAGIC_PREAMBLE = 0x6060b017
-const AVAILABLE_BOLT_PROTOCOLS = ['5.8', '5.7', '5.6', '5.4', '5.3', '5.2', '5.1', '5.0', '4.4', '4.3', '4.2', '3.0'] // bolt protocols the client will accept, ordered by preference
+const AVAILABLE_BOLT_PROTOCOLS = ['6.0', '5.8', '5.7', '5.6', '5.4', '5.3', '5.2', '5.1', '5.0', '4.4', '4.3', '4.2', '3.0'] // bolt protocols the client will accept, ordered by preference
 const DESIRED_CAPABILITES = 0
 
 function version (major, minor) {

packages/bolt-connection/test/bolt/__snapshots__/bolt-protocol-v1.test.js.snap

@@ -1,13 +1,5 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`#unit BoltProtocolV1 .packable() should resultant function not pack graph types (Node) 1`] = `"It is not allowed to pass nodes in query parameters, given: (c:a {a:"b"})"`;
-
-exports[`#unit BoltProtocolV1 .packable() should resultant function not pack graph types (Path) 1`] = `"It is not allowed to pass paths in query parameters, given: [object Object]"`;
-
-exports[`#unit BoltProtocolV1 .packable() should resultant function not pack graph types (Relationship) 1`] = `"It is not allowed to pass relationships in query parameters, given: (e)-[:a {b:"c"}]->(f)"`;
-
-exports[`#unit BoltProtocolV1 .packable() should resultant function not pack graph types (UnboundRelationship) 1`] = `"It is not allowed to pass unbound relationships in query parameters, given: -[:a {b:"c"}]->"`;
-
 exports[`#unit BoltProtocolV1 .packable() should pack types introduced afterwards as Map (Date) 1`] = `
 {
   "day": 1,
@@ -86,6 +78,16 @@ exports[`#unit BoltProtocolV1 .packable() should pack types introduced afterward
 }
 `;
 
+exports[`#unit BoltProtocolV1 .packable() should resultant function not pack graph types (Node) 1`] = `"It is not allowed to pass nodes in query parameters, given: (c:a {a:"b"})"`;
+
+exports[`#unit BoltProtocolV1 .packable() should resultant function not pack graph types (Path) 1`] = `"It is not allowed to pass paths in query parameters, given: [object Object]"`;
+
+exports[`#unit BoltProtocolV1 .packable() should resultant function not pack graph types (Relationship) 1`] = `"It is not allowed to pass relationships in query parameters, given: (e)-[:a {b:"c"}]->(f)"`;
+
+exports[`#unit BoltProtocolV1 .packable() should resultant function not pack graph types (UnboundRelationship) 1`] = `"It is not allowed to pass unbound relationships in query parameters, given: -[:a {b:"c"}]->"`;
+
+exports[`#unit BoltProtocolV1 .unpack() should error out of unpacking Vectors 1`] = `"Server tried to send Vector object, but server and driver are communicating on a version of the Bolt protocol that does not support vectors."`;
+
 exports[`#unit BoltProtocolV1 .unpack() should not unpack graph types with wrong size(Node with less fields) 1`] = `"Wrong struct size for Node, expected 3 but was 2"`;
 
 exports[`#unit BoltProtocolV1 .unpack() should not unpack graph types with wrong size(Node with more fields) 1`] = `"Wrong struct size for Node, expected 3 but was 4"`;

packages/bolt-connection/test/bolt/__snapshots__/bolt-protocol-v6x0.test.js.snap

@@ -0,0 +1,61 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`#unit BoltProtocolV6x0 .packable() should resultant function not pack graph types (Node) 1`] = `"It is not allowed to pass nodes in query parameters, given: (c:a {a:"b"})"`;
+
+exports[`#unit BoltProtocolV6x0 .packable() should resultant function not pack graph types (Path) 1`] = `"It is not allowed to pass paths in query parameters, given: [object Object]"`;
+
+exports[`#unit BoltProtocolV6x0 .packable() should resultant function not pack graph types (Relationship) 1`] = `"It is not allowed to pass relationships in query parameters, given: (e)-[:a {b:"c"}]->(f)"`;
+
+exports[`#unit BoltProtocolV6x0 .packable() should resultant function not pack graph types (UnboundRelationship) 1`] = `"It is not allowed to pass unbound relationships in query parameters, given: -[:a {b:"c"}]->"`;
+
+exports[`#unit BoltProtocolV6x0 .unpack() should not unpack with wrong size (Date with less fields) 1`] = `"Wrong struct size for Date, expected 1 but was 0"`;
+
+exports[`#unit BoltProtocolV6x0 .unpack() should not unpack with wrong size (Date with more fields) 1`] = `"Wrong struct size for Date, expected 1 but was 2"`;
+
+exports[`#unit BoltProtocolV6x0 .unpack() should not unpack with wrong size (DateTimeWithZoneId with less fields) 1`] = `"Wrong struct size for DateTimeWithZoneId, expected 3 but was 2"`;
+
+exports[`#unit BoltProtocolV6x0 .unpack() should not unpack with wrong size (DateTimeWithZoneId with more fields) 1`] = `"Wrong struct size for DateTimeWithZoneId, expected 3 but was 4"`;
+
+exports[`#unit BoltProtocolV6x0 .unpack() should not unpack with wrong size (DateTimeWithZoneOffset with less fields) 1`] = `"Wrong struct size for DateTimeWithZoneOffset, expected 3 but was 2"`;
+
+exports[`#unit BoltProtocolV6x0 .unpack() should not unpack with wrong size (DateTimeWithZoneOffset with more fields) 1`] = `"Wrong struct size for DateTimeWithZoneOffset, expected 3 but was 4"`;
+
+exports[`#unit BoltProtocolV6x0 .unpack() should not unpack with wrong size (Duration with less fields) 1`] = `"Wrong struct size for Duration, expected 4 but was 3"`;
+
+exports[`#unit BoltProtocolV6x0 .unpack() should not unpack with wrong size (Duration with more fields) 1`] = `"Wrong struct size for Duration, expected 4 but was 5"`;
+
+exports[`#unit BoltProtocolV6x0 .unpack() should not unpack with wrong size (LocalDateTime with less fields) 1`] = `"Wrong struct size for LocalDateTime, expected 2 but was 1"`;
+
+exports[`#unit BoltProtocolV6x0 .unpack() should not unpack with wrong size (LocalDateTime with more fields) 1`] = `"Wrong struct size for LocalDateTime, expected 2 but was 3"`;
+
+exports[`#unit BoltProtocolV6x0 .unpack() should not unpack with wrong size (LocalTime with less fields) 1`] = `"Wrong struct size for LocalTime, expected 1 but was 0"`;
+
+exports[`#unit BoltProtocolV6x0 .unpack() should not unpack with wrong size (LocalTime with more fields) 1`] = `"Wrong struct size for LocalTime, expected 1 but was 2"`;
+
+exports[`#unit BoltProtocolV6x0 .unpack() should not unpack with wrong size (Node with less fields) 1`] = `"Wrong struct size for Node, expected 4 but was 3"`;
+
+exports[`#unit BoltProtocolV6x0 .unpack() should not unpack with wrong size (Node with more fields) 1`] = `"Wrong struct size for Node, expected 4 but was 5"`;
+
+exports[`#unit BoltProtocolV6x0 .unpack() should not unpack with wrong size (Path with less fields) 1`] = `"Wrong struct size for Path, expected 3 but was 2"`;
+
+exports[`#unit BoltProtocolV6x0 .unpack() should not unpack with wrong size (Path with more fields) 1`] = `"Wrong struct size for Path, expected 3 but was 4"`;
+
+exports[`#unit BoltProtocolV6x0 .unpack() should not unpack with wrong size (Point with less fields) 1`] = `"Wrong struct size for Point2D, expected 3 but was 2"`;
+
+exports[`#unit BoltProtocolV6x0 .unpack() should not unpack with wrong size (Point with more fields) 1`] = `"Wrong struct size for Point2D, expected 3 but was 4"`;
+
+exports[`#unit BoltProtocolV6x0 .unpack() should not unpack with wrong size (Point3D with less fields) 1`] = `"Wrong struct size for Point3D, expected 4 but was 3"`;
+
+exports[`#unit BoltProtocolV6x0 .unpack() should not unpack with wrong size (Point3D with more fields) 1`] = `"Wrong struct size for Point3D, expected 4 but was 5"`;
+
+exports[`#unit BoltProtocolV6x0 .unpack() should not unpack with wrong size (Relationship with less fields) 1`] = `"Wrong struct size for Relationship, expected 8 but was 5"`;
+
+exports[`#unit BoltProtocolV6x0 .unpack() should not unpack with wrong size (Relationship with more fields) 1`] = `"Wrong struct size for Relationship, expected 8 but was 9"`;
+
+exports[`#unit BoltProtocolV6x0 .unpack() should not unpack with wrong size (Time with less fields) 1`] = `"Wrong struct size for Time, expected 2 but was 1"`;
+
+exports[`#unit BoltProtocolV6x0 .unpack() should not unpack with wrong size (Time with more fileds) 1`] = `"Wrong struct size for Time, expected 2 but was 3"`;
+
+exports[`#unit BoltProtocolV6x0 .unpack() should not unpack with wrong size (UnboundRelationship with less fields) 1`] = `"Wrong struct size for UnboundRelationship, expected 4 but was 3"`;
+
+exports[`#unit BoltProtocolV6x0 .unpack() should not unpack with wrong size (UnboundRelationship with more fields) 1`] = `"Wrong struct size for UnboundRelationship, expected 4 but was 5"`;