[8.19] Add custom vector data converters for high performance ingest scenarios (#8794) (#8795)

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

Author: flobernd

src/Elastic.Clients.Elasticsearch/_Shared/Core/Configuration/ElasticsearchClientSettings.cs

@@ -115,6 +115,8 @@ public abstract class ElasticsearchClientSettingsBase<TConnectionSettings> :
 	private readonly Serializer _sourceSerializer;
 	private BeforeRequestEvent? _onBeforeRequest;
 	private bool _experimentalEnableSerializeNullInferredValues;
+	private FloatVectorDataEncoding _floatVectorDataEncoding = Serialization.FloatVectorDataEncoding.Base64;
+	private ByteVectorDataEncoding _byteVectorDataEncoding = Serialization.ByteVectorDataEncoding.Base64;
 	private ExperimentalSettings _experimentalSettings = new();
 
 	private bool _defaultDisableAllInference;
@@ -165,6 +167,8 @@ protected ElasticsearchClientSettingsBase(
 	FluentDictionary<Type, string> IElasticsearchClientSettings.RouteProperties => _routeProperties;
 	Serializer IElasticsearchClientSettings.SourceSerializer => _sourceSerializer;
 	BeforeRequestEvent? IElasticsearchClientSettings.OnBeforeRequest => _onBeforeRequest;
+	FloatVectorDataEncoding IElasticsearchClientSettings.FloatVectorDataEncoding => _floatVectorDataEncoding;
+	ByteVectorDataEncoding IElasticsearchClientSettings.ByteVectorDataEncoding => _byteVectorDataEncoding;
 	ExperimentalSettings IElasticsearchClientSettings.Experimental => _experimentalSettings;
 
 	bool IElasticsearchClientSettings.ExperimentalEnableSerializeNullInferredValues => _experimentalEnableSerializeNullInferredValues;
@@ -198,6 +202,18 @@ public TConnectionSettings DefaultFieldNameInferrer(Func<string, string> fieldNa
 	public TConnectionSettings ExperimentalEnableSerializeNullInferredValues(bool enabled = true) =>
 		Assign(enabled, (a, v) => a._experimentalEnableSerializeNullInferredValues = v);
 
+	/// <inheritdoc cref="IElasticsearchClientSettings.FloatVectorDataEncoding"/>
+	/// <param name="encoding">The default vector data encoding to use.</param>
+	/// <returns>This settings instance for chaining.</returns>
+	public TConnectionSettings FloatVectorDataEncoding(FloatVectorDataEncoding encoding) =>
+		Assign(encoding, (a, v) => a._floatVectorDataEncoding = v);
+
+	/// <inheritdoc cref="IElasticsearchClientSettings.ByteVectorDataEncoding"/>
+	/// <param name="encoding">The default vector data encoding to use.</param>
+	/// <returns>This settings instance for chaining.</returns>
+	public TConnectionSettings ByteVectorDataEncoding(ByteVectorDataEncoding encoding) =>
+		Assign(encoding, (a, v) => a._byteVectorDataEncoding = v);
+
 	public TConnectionSettings Experimental(ExperimentalSettings settings) =>
 		Assign(settings, (a, v) => a._experimentalSettings = v);
 

src/Elastic.Clients.Elasticsearch/_Shared/Core/Configuration/IElasticsearchClientSettings.cs

@@ -5,7 +5,10 @@
 using System;
 using System.Collections.Generic;
 using System.Reflection;
+
 using Elastic.Clients.Elasticsearch.Requests;
+using Elastic.Clients.Elasticsearch.Serialization;
+
 using Elastic.Transport;
 
 namespace Elastic.Clients.Elasticsearch;
@@ -116,14 +119,37 @@ public interface IElasticsearchClientSettings : ITransportConfiguration
 	BeforeRequestEvent? OnBeforeRequest { get; }
 
 	/// <summary>
-	/// This is an advanced setting which controls serialization behaviour for inferred properies such as ID, routing and index name.
-	/// <para>When enabled, it may reduce allocations on serialisation paths where the cost can be more significant, such as in bulk operations.</para>
+	/// This is an advanced setting which controls serialization behaviour for inferred properties such as ID, routing and index name.
+	/// <para>When enabled, it may reduce allocations on serialization paths where the cost can be more significant, such as in bulk operations.</para>
 	/// <para>As a by-product it may cause null values to be included in the serialized data and impact payload size. This will only be a concern should some
-	/// typed not have inferrence mappings defined for the required properties.</para>
+	/// typed not have inference mappings defined for the required properties.</para>
 	/// </summary>
-	/// <remarks>This is marked as experiemental and may be removed or renamed in the future once its impact is evaluated.</remarks>
+	/// <remarks>This is marked as experimental and may be removed or renamed in the future once its impact is evaluated.</remarks>
 	bool ExperimentalEnableSerializeNullInferredValues { get; }
 
+	/// <summary>
+	/// Controls the vector data encoding to use for <see cref="ReadOnlyMemory{T}"/> properties
+	/// in documents during ingestion when the <see cref="FloatVectorDataConverter"/> is used.
+	/// </summary>
+	/// <remarks>
+	///	Setting this value to <see cref="FloatVectorDataEncoding.Legacy"/> provides backwards
+	/// compatibility when talking to Elasticsearch servers with a version older than 9.3.0
+	/// (required for <see cref="ByteVectorDataEncoding.Base64"/>).
+	/// </remarks>
+	FloatVectorDataEncoding FloatVectorDataEncoding { get; }
+
+	/// <summary>
+	/// Controls the vector data encoding to use for <see cref="ReadOnlyMemory{T}"/> properties
+	/// in documents during ingestion when the <see cref="ByteVectorDataConverter"/> is used.
+	/// </summary>
+	/// <remarks>
+	///	Setting this value to <see cref="ByteVectorDataEncoding.Legacy"/> provides backwards
+	/// compatibility when talking to Elasticsearch servers with a version older than 8.14.0
+	/// (required for <see cref="ByteVectorDataEncoding.Hex"/>) or older than 9.3.0 (required
+	/// for <see cref="ByteVectorDataEncoding.Base64"/>).
+	/// </remarks>
+	ByteVectorDataEncoding ByteVectorDataEncoding { get; }
+
 	/// <summary>
 	/// Experimental settings.
 	/// </summary>

src/Elastic.Clients.Elasticsearch/_Shared/Core/LazyJsonConverter.cs

@@ -3,6 +3,7 @@
 // See the LICENSE file in the project root for more information.
 
 using System;
+using System.Diagnostics.CodeAnalysis;
 using System.Text.Json;
 using System.Text.Json.Serialization;
 
@@ -14,13 +15,13 @@ public sealed class LazyJsonConverter : JsonConverter<LazyJson>
 {
 	private IElasticsearchClientSettings? _settings;
 
+	[UnconditionalSuppressMessage("AOT", "IL3050:Calling members annotated with 'RequiresDynamicCodeAttribute'", Justification = "Always using explicit TypeInfoResolver")]
+	[UnconditionalSuppressMessage("Trimming", "IL2026:Members annotated with 'RequiresUnreferencedCodeAttribute'", Justification = "Always using explicit TypeInfoResolver")]
 	public override LazyJson Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
 	{
 		InitializeSettings(options);
 
-#pragma warning disable IL2026, IL3050 // The `TypeInfoResolver` for `RequestResponseConverter` knows how to handle `JsonElement`.
 		return new LazyJson(JsonSerializer.Deserialize<JsonElement>(ref reader, options), _settings!);
-#pragma warning restore IL2026, IL3050
 	}
 
 	private void InitializeSettings(JsonSerializerOptions options)

src/Elastic.Clients.Elasticsearch/_Shared/Next/JsonWriterExtensions.cs

@@ -249,6 +249,36 @@ public static void WriteUnionValue<T1, T2>(this Utf8JsonWriter writer, JsonSeria
 		);
 	}
 
+	public static void WriteMemoryValue<T>(this Utf8JsonWriter writer, JsonSerializerOptions options, ReadOnlyMemory<T> memory,
+		JsonWriteFunc<T>? writeElement)
+	{
+		if (writeElement is null)
+		{
+			var converter = options.GetConverter<T>(null);
+
+			writeElement = (w, o, v) =>
+			{
+				if ((v is null) && !converter.HandleNull)
+				{
+					w.WriteNullValue();
+					return;
+				}
+
+				converter.Write(w, v, o);
+			};
+		}
+
+		writer.WriteStartArray();
+
+		var span = memory.Span;
+		foreach (var element in span)
+		{
+			writeElement(writer, options, element);
+		}
+
+		writer.WriteEndArray();
+	}
+
 	#endregion Delegate Based Write Methods
 
 	#region Specialized Write Methods