From c03471d89b979dd497506fac2a8937719ea09491 Mon Sep 17 00:00:00 2001 From: Arkadiusz Biel Date: Sun, 16 Oct 2022 15:56:01 +0100 Subject: [PATCH] Feat: base implementation for some services in algolia --- .../CommandHandlers/IndexCommandHandler.cs | 32 +++ .../CommandHandlers/ReindexCommandHandler.cs | 62 +++++ .../RemoveCommandCommandHandler.cs | 75 ++++++ .../Application/ElasticSearchModelMapper.cs | 48 ++++ .../ConfigurationEnabledDirectoryFactory.cs | 44 +++ ...uceneSearchExecutorWithFacetsAndFilters.cs | 251 ++++++++++++++++++ .../LuceneSearchQueryWithFiltersAndFacets.cs | 87 ++++++ .../SearcherWithFacetsAndFilters.cs | 9 + .../LuceneEngine/SimplExamineLuceneIndex.cs | 42 +++ .../SimplExamineLuceneSearcher.cs | 18 ++ .../ElasticSearchQueryHandler.cs | 96 +++++++ .../BaseExamineQueryTranslatorService.cs | 60 +++++ .../IElasticSearchQueryTranslatorService.cs | 10 + .../BooleanSubQueryExamineTranslator.cs | 66 +++++ .../DateRangeQueryExamineTranslator.cs | 43 +++ .../FuzzySubQueryExamineTranslator.cs | 36 +++ .../SubQueries/ISubQueryExamineTranslator.cs | 16 ++ .../LongRangeQueryExamineTranslator.cs | 43 +++ .../PrefixSubQueryExamineTranslator.cs | 36 +++ .../SubQueries/RangeQueryElasticTranslator.cs | 43 +++ .../TermSubQueryExamineTranslator.cs | 36 +++ .../TermsSubQueryExamineTranslator.cs | 41 +++ .../Configuration/AuthenticationModes.cs | 11 + .../ExamineSearchConfiguration.cs | 13 + .../HeadlessSearchServerConfigExtensions.cs | 26 ++ .../ExamineSearchModule.cs | 67 +++++ .../Models/AnalyzerType.cs | 11 + .../Models/ElasticProperty.cs | 20 ++ .../Models/ElasticSearchModel.cs | 26 ++ .../SImpl.SearchModule.Algolia.csproj | 19 ++ src/SImpl.SearchModule.sln | 6 + 31 files changed, 1393 insertions(+) create mode 100644 src/SImpl.SearchModule.Algolia/Application/CommandHandlers/IndexCommandHandler.cs create mode 100644 src/SImpl.SearchModule.Algolia/Application/CommandHandlers/ReindexCommandHandler.cs create mode 100644 src/SImpl.SearchModule.Algolia/Application/CommandHandlers/RemoveCommandCommandHandler.cs create mode 100644 src/SImpl.SearchModule.Algolia/Application/ElasticSearchModelMapper.cs create mode 100644 src/SImpl.SearchModule.Algolia/Application/Factories/ConfigurationEnabledDirectoryFactory.cs create mode 100644 src/SImpl.SearchModule.Algolia/Application/LuceneEngine/LuceneSearchExecutorWithFacetsAndFilters.cs create mode 100644 src/SImpl.SearchModule.Algolia/Application/LuceneEngine/LuceneSearchQueryWithFiltersAndFacets.cs create mode 100644 src/SImpl.SearchModule.Algolia/Application/LuceneEngine/SearcherWithFacetsAndFilters.cs create mode 100644 src/SImpl.SearchModule.Algolia/Application/LuceneEngine/SimplExamineLuceneIndex.cs create mode 100644 src/SImpl.SearchModule.Algolia/Application/LuceneEngine/SimplExamineLuceneSearcher.cs create mode 100644 src/SImpl.SearchModule.Algolia/Application/QueryHandlers/ElasticSearchQueryHandler.cs create mode 100644 src/SImpl.SearchModule.Algolia/Application/Services/BaseExamineQueryTranslatorService.cs create mode 100644 src/SImpl.SearchModule.Algolia/Application/Services/IElasticSearchQueryTranslatorService.cs create mode 100644 src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/BooleanSubQueryExamineTranslator.cs create mode 100644 src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/DateRangeQueryExamineTranslator.cs create mode 100644 src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/FuzzySubQueryExamineTranslator.cs create mode 100644 src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/ISubQueryExamineTranslator.cs create mode 100644 src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/LongRangeQueryExamineTranslator.cs create mode 100644 src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/PrefixSubQueryExamineTranslator.cs create mode 100644 src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/RangeQueryElasticTranslator.cs create mode 100644 src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/TermSubQueryExamineTranslator.cs create mode 100644 src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/TermsSubQueryExamineTranslator.cs create mode 100644 src/SImpl.SearchModule.Algolia/Configuration/AuthenticationModes.cs create mode 100644 src/SImpl.SearchModule.Algolia/Configuration/ExamineSearchConfiguration.cs create mode 100644 src/SImpl.SearchModule.Algolia/Configuration/HeadlessSearchServerConfigExtensions.cs create mode 100644 src/SImpl.SearchModule.Algolia/ExamineSearchModule.cs create mode 100644 src/SImpl.SearchModule.Algolia/Models/AnalyzerType.cs create mode 100644 src/SImpl.SearchModule.Algolia/Models/ElasticProperty.cs create mode 100644 src/SImpl.SearchModule.Algolia/Models/ElasticSearchModel.cs create mode 100644 src/SImpl.SearchModule.Algolia/SImpl.SearchModule.Algolia.csproj diff --git a/src/SImpl.SearchModule.Algolia/Application/CommandHandlers/IndexCommandHandler.cs b/src/SImpl.SearchModule.Algolia/Application/CommandHandlers/IndexCommandHandler.cs new file mode 100644 index 0000000..d87dd9b --- /dev/null +++ b/src/SImpl.SearchModule.Algolia/Application/CommandHandlers/IndexCommandHandler.cs @@ -0,0 +1,32 @@ +using Algolia.Search.Clients; +using SImpl.CQRS.Commands; +using SImpl.SearchModule.Abstraction.Commands; +using SImpl.SearchModule.Abstraction.Models; +using SImpl.SearchModule.Algolia.Configuration; + +namespace SImpl.SearchModule.Algolia.Application.CommandHandlers +{ + public class IndexCommandHandler : ICommandHandler + { + private readonly ISearchClient _algoliaSearchClient; + private readonly AlgoliaSearchConfiguration _configuration; + private readonly ILogger _logger; + + + public IndexCommandHandler(ISearchClient algoliaSearchClient, AlgoliaSearchConfiguration configuration, + ILogger logger) + { + _algoliaSearchClient = algoliaSearchClient; + _configuration = configuration; + _logger = logger; + } + + public async Task HandleAsync(IndexCommand command) + { + var indexName = _configuration.IndexPrefixName + command.Index.ToLowerInvariant(); + var index= _algoliaSearchClient.InitIndex(indexName); + var result = await index.SaveObjectsAsync(command.Models); + } + + } +} \ No newline at end of file diff --git a/src/SImpl.SearchModule.Algolia/Application/CommandHandlers/ReindexCommandHandler.cs b/src/SImpl.SearchModule.Algolia/Application/CommandHandlers/ReindexCommandHandler.cs new file mode 100644 index 0000000..aa568dc --- /dev/null +++ b/src/SImpl.SearchModule.Algolia/Application/CommandHandlers/ReindexCommandHandler.cs @@ -0,0 +1,62 @@ +using SImpl.CQRS.Commands; +using SImpl.SearchModule.Abstraction.Commands; +using SImpl.SearchModule.Abstraction.Models; +using SImpl.SearchModule.Algolia.Configuration; + +namespace SImpl.SearchModule.Algolia.Application.CommandHandlers +{ + public class ReIndexCommandHandler : ICommandHandler + { + private readonly IExamineManager _examineManager; + private readonly ExamineSearchConfiguration _configuration; + private readonly ILogger _logger; + + + public ReIndexCommandHandler(IExamineManager examineManager, ExamineSearchConfiguration configuration, + ILogger logger) + { + _examineManager = examineManager; + _configuration = configuration; + _logger = logger; + } + + public async Task HandleAsync(ReIndexCommand command) + { + var indexName = _configuration.IndexPrefixName + command.Index.ToLowerInvariant(); + _examineManager.TryGetIndex(indexName, + out IIndex examineIndex); + if (examineIndex == null) + { + _logger.LogError($"Examine index not found {indexName}"); + return; + } + + try + { + + examineIndex.CreateIndex(); + + + examineIndex.IndexItems(TranslateModel(command.Models)); + } + catch (Exception e) + { + _logger.LogError($"Indexing for {indexName} failed", e); + } + } + + private IEnumerable TranslateModel(List commandModels) + { + var modelList = new List(); + foreach (var model in commandModels) + { + var translatedModel = ValueSet.FromObject(model.Id, "search", model.ContentType, model); + + modelList.Add(translatedModel); + } + + return modelList; + } + } +} + diff --git a/src/SImpl.SearchModule.Algolia/Application/CommandHandlers/RemoveCommandCommandHandler.cs b/src/SImpl.SearchModule.Algolia/Application/CommandHandlers/RemoveCommandCommandHandler.cs new file mode 100644 index 0000000..4050a77 --- /dev/null +++ b/src/SImpl.SearchModule.Algolia/Application/CommandHandlers/RemoveCommandCommandHandler.cs @@ -0,0 +1,75 @@ +using SImpl.CQRS.Commands; +using SImpl.SearchModule.Abstraction.Commands; +using SImpl.SearchModule.Algolia.Configuration; + +namespace SImpl.SearchModule.Algolia.Application.CommandHandlers +{ + public class RemoveCommandCommandHandler : ICommandHandler + { + private readonly IExamineManager _examineManager; + private readonly ExamineSearchConfiguration _configuration; + private readonly ILogger _logger; + + public RemoveCommandCommandHandler(IExamineManager examineManager, ExamineSearchConfiguration configuration, + ILogger logger) + { + _examineManager = examineManager; + _configuration = configuration; + _logger = logger; + } + + public async Task HandleAsync(RemoveCommand command) + { + var indexName = _configuration.IndexPrefixName + command.Index.ToLowerInvariant(); + _examineManager.TryGetIndex(indexName, + out IIndex examineIndex); + if (examineIndex == null) + { + _logger.LogError($"Examine index not found {indexName}"); + return; + } + + if (command.Models.Any()) + { + try + { + _logger.LogInformation($"Deleted items {string.Join(", ",command.Models.Select(x=>x.Id.ToString()).ToList())}"); + examineIndex.DeleteFromIndex(command.Models.Select(x=>x.Id)); + + } + catch (Exception e) + { + _logger.LogError($"remove from index {indexName} failed"); + } + } + else if (command.ModelsIds.Any()) + { + try + { _logger.LogInformation($"Deleted items {string.Join(", ",command.ModelsIds)}"); + + examineIndex.DeleteFromIndex(command.ModelsIds.Select(x=>x.ToString())); + + } + catch (Exception e) + { + _logger.LogError($"remove from index {indexName} failed"); + } + + }else if (command.ModelsKeys.Any()) + { + try + { _logger.LogInformation($"Deleted items {string.Join(", ",command.ModelsKeys)}"); + + examineIndex.DeleteFromIndex(command.ModelsKeys); + + } + catch (Exception e) + { + _logger.LogError($"remove from index {indexName} failed"); + } + + } + + } + } +} \ No newline at end of file diff --git a/src/SImpl.SearchModule.Algolia/Application/ElasticSearchModelMapper.cs b/src/SImpl.SearchModule.Algolia/Application/ElasticSearchModelMapper.cs new file mode 100644 index 0000000..f362de8 --- /dev/null +++ b/src/SImpl.SearchModule.Algolia/Application/ElasticSearchModelMapper.cs @@ -0,0 +1,48 @@ +using System.Globalization; +using SImpl.SearchModule.Abstraction.Models; +using SImpl.SearchModule.Algolia.Models; + +namespace SImpl.SearchModule.Algolia.Application +{ + public class ElasticSearchModelMapper + { + public static string MapToSimpleTypeName(Type type) + { + return $"{type.FullName}, {type.Assembly.GetName().Name}"; + } + + public static ElasticSearchModel Map(ISearchModel model) + { + return new ElasticSearchModel + { + Id = model.ContentKey, + AdditionalKeys = model.AdditionalKeys, + Culture = model.Culture.IetfLanguageTag.ToLower(), + Content = model.Content, + Facet = model.Facet, + ContentType = model.ContentType, + Tags = model.Tags, + IndexedAt = model.IndexedAt, + ViewModelType = MapToSimpleTypeName(model.ViewModelType), + CustomProperties = model.CustomProperties, + }; + } + + public static ISearchModel Map(ElasticSearchModel model) + { + return new BaseSearchModel + { + ContentKey = model.Id, + Content = model.Content, + AdditionalKeys = model.AdditionalKeys, + ContentType = model.ContentType, + Facet = model.Facet, + Culture = new CultureInfo(model.Culture), + IndexedAt = model.IndexedAt, + Tags = model.Tags?.ToList(), + ViewModelType = Type.GetType(model.ViewModelType), + CustomProperties = model.CustomProperties, + }; + } + } +} \ No newline at end of file diff --git a/src/SImpl.SearchModule.Algolia/Application/Factories/ConfigurationEnabledDirectoryFactory.cs b/src/SImpl.SearchModule.Algolia/Application/Factories/ConfigurationEnabledDirectoryFactory.cs new file mode 100644 index 0000000..58b0312 --- /dev/null +++ b/src/SImpl.SearchModule.Algolia/Application/Factories/ConfigurationEnabledDirectoryFactory.cs @@ -0,0 +1,44 @@ +using Microsoft.Extensions.DependencyInjection; +using SImpl.SearchModule.Algolia.Configuration; + +namespace SImpl.SearchModule.Algolia.Application.Factories +{ + public class ConfigurationEnabledDirectoryFactory : DirectoryFactoryBase + { + private readonly IServiceProvider _services; + private readonly ExamineSearchConfiguration _examineSearchConfiguration; + private readonly IApplicationRoot _applicationRoot; + private IDirectoryFactory _directoryFactory; + + public ConfigurationEnabledDirectoryFactory( + IServiceProvider services, + ExamineSearchConfiguration examineSearchConfiguration, + IApplicationRoot applicationRoot) + { + _services = services; + _examineSearchConfiguration = examineSearchConfiguration; + _applicationRoot = applicationRoot; + } + + protected override Lucene.Net.Store.Directory CreateDirectory( + LuceneIndex luceneIndex, + bool forceUnlock) + { + _directoryFactory = CreateFactory(); + return _directoryFactory.CreateDirectory(luceneIndex, forceUnlock); + } + + /// + /// Creates a directory factory based on the configured value and ensures that + /// + private IDirectoryFactory CreateFactory() + { + DirectoryInfo applicationRoot = _applicationRoot.ApplicationRoot; + if (!applicationRoot.Exists) + System.IO.Directory.CreateDirectory(applicationRoot.FullName); + + return (IDirectoryFactory) this._services.GetRequiredService(_examineSearchConfiguration + .LuceneDirectoryFactory); + } + } +} \ No newline at end of file diff --git a/src/SImpl.SearchModule.Algolia/Application/LuceneEngine/LuceneSearchExecutorWithFacetsAndFilters.cs b/src/SImpl.SearchModule.Algolia/Application/LuceneEngine/LuceneSearchExecutorWithFacetsAndFilters.cs new file mode 100644 index 0000000..e3b3c7c --- /dev/null +++ b/src/SImpl.SearchModule.Algolia/Application/LuceneEngine/LuceneSearchExecutorWithFacetsAndFilters.cs @@ -0,0 +1,251 @@ +namespace SImpl.SearchModule.Algolia.Application.LuceneEngine +{ + + /// + /// An implementation of the search results returned from Lucene.Net + /// + public class LuceneSearchExecutorWithFacetsAndFilters + { + private readonly QueryOptions _options; + private readonly IEnumerable _sortField; + private readonly ISearchContext _searchContext; + private readonly Query _luceneQuery; + private readonly Filter _filter; + private readonly Dictionary _highlitherQueries; + private readonly ISet _fieldsToLoad; + private int? _maxDoc; + + internal LuceneSearchExecutorWithFacetsAndFilters(QueryOptions options, Query query, Filter filter, Dictionary highlitherQueries, + IEnumerable sortField, ISearchContext searchContext, ISet fieldsToLoad) + { + _options = options ?? QueryOptions.Default; + _luceneQuery = query ?? throw new ArgumentNullException(nameof(query)); + _filter = filter; + _highlitherQueries = highlitherQueries; + _fieldsToLoad = fieldsToLoad; + _sortField = sortField ?? throw new ArgumentNullException(nameof(sortField)); + _searchContext = searchContext ?? throw new ArgumentNullException(nameof(searchContext)); + } + + private int MaxDoc + { + get + { + if (_maxDoc == null) + { + using (ISearcherReference searcher = _searchContext.GetSearcher()) + { + _maxDoc = searcher.IndexSearcher.IndexReader.MaxDoc; + } + } + return _maxDoc.Value; + } + } + + public ISearchResults Execute() + { + var extractTermsSupported = CheckQueryForExtractTerms(_luceneQuery); + + if (extractTermsSupported) + { + //This try catch is because analyzers strip out stop words and sometimes leave the query + //with null values. This simply tries to extract terms, if it fails with a null + //reference then its an invalid null query, NotSupporteException occurs when the query is + //valid but the type of query can't extract terms. + //This IS a work-around, theoretically Lucene itself should check for null query parameters + //before throwing exceptions. + try + { + var set = new HashSet(); + _luceneQuery.ExtractTerms(set); + } + catch (NullReferenceException) + { + //this means that an analyzer has stipped out stop words and now there are + //no words left to search on + + //it could also mean that potentially a IIndexFieldValueType is throwing a null ref + return LuceneSearchResults.Empty; + } + catch (NotSupportedException) + { + //swallow this exception, we should continue if this occurs. + } + } + + var maxResults = Math.Min((_options.Skip + 1) * _options.Take, MaxDoc); + maxResults = maxResults >= 1 ? maxResults : QueryOptions.DefaultMaxResults; + + ICollector topDocsCollector; + SortField[] sortFields = _sortField as SortField[] ?? _sortField.ToArray(); + if (sortFields.Length > 0) + { + topDocsCollector = TopFieldCollector.Create( + new Sort(sortFields), maxResults, false, false, false, false); + } + else + { + topDocsCollector = TopScoreDocCollector.Create(maxResults, true); + } + + FacetsCollector facetsCollector = new FacetsCollector(true); + using (ISearcherReference searcher = _searchContext.GetSearcher()) + { + //todo: figure out facet fields + + if (_filter == null) + { + searcher.IndexSearcher.Search(_luceneQuery, topDocsCollector); + + } + else + { + searcher.IndexSearcher.Search(_luceneQuery,_filter, topDocsCollector); + } + TopDocs topDocs; + if (sortFields.Length > 0) + { + topDocs = ((TopFieldCollector)topDocsCollector).GetTopDocs(_options.Skip, _options.Take); + } + else + { + topDocs = ((TopScoreDocCollector)topDocsCollector).GetTopDocs(_options.Skip, _options.Take); + } + + var totalItemCount = topDocs.TotalHits; + + var results = new List(); + + + for (int i = 0; i < topDocs.ScoreDocs.Length; i++) + { + var result = GetSearchResult(i, topDocs, searcher.IndexSearcher); + foreach (var query in _highlitherQueries) + { + QueryScorer scorer = new QueryScorer(query.Value); + SimpleHTMLFormatter formatter = new SimpleHTMLFormatter("em", "em"); + Highlighter highlighter = new Highlighter(formatter, scorer); + highlighter.TextFragmenter = new SimpleFragmenter(); + TokenStream stream = new StandardAnalyzer(LuceneVersion.LUCENE_48).GetTokenStream(query.Key,result[query.Key] ); + highlighter.GetBestFragments(stream,result[query.Key],int.MaxValue); + results.Add(result); + } + + } + + + return new LuceneSearchResults(results, totalItemCount); + } + } + + private ISearchResult GetSearchResult(int index, TopDocs topDocs, IndexSearcher luceneSearcher) + { + // I have seen IndexOutOfRangeException here which is strange as this is only called in one place + // and from that one place "i" is always less than the size of this collection. + // but we'll error check here anyways + if (topDocs?.ScoreDocs.Length < index) + { + return null; + } + + var scoreDoc = topDocs.ScoreDocs[index]; + + var docId = scoreDoc.Doc; + Document doc; + if (_fieldsToLoad != null) + { + doc = luceneSearcher.Doc(docId, _fieldsToLoad); + } + else + { + doc = luceneSearcher.Doc(docId); + } + var score = scoreDoc.Score; + var result = CreateSearchResult(doc, score); + + return result; + } + + /// + /// Creates the search result from a + /// + /// The doc to convert. + /// The score. + /// A populated search result object + private ISearchResult CreateSearchResult(Document doc, float score) + { + var id = doc.Get("id"); + + if (string.IsNullOrEmpty(id) == true) + { + id = doc.Get(ExamineFieldNames.ItemIdFieldName); + } + + var searchResult = new SearchResult(id, score, () => + { + //we can use lucene to find out the fields which have been stored for this particular document + var fields = doc.Fields; + + var resultVals = new Dictionary>(); + + foreach (var field in fields.Cast()) + { + var fieldName = field.Name; + var values = doc.GetValues(fieldName); + + if (resultVals.TryGetValue(fieldName, out var resultFieldVals)) + { + foreach (var value in values) + { + if (!resultFieldVals.Contains(value)) + { + resultFieldVals.Add(value); + } + } + } + else + { + resultVals[fieldName] = values.ToList(); + } + } + + return resultVals; + }); + + return searchResult; + } + + private bool CheckQueryForExtractTerms(Query query) + { + if (query is BooleanQuery bq) + { + foreach (BooleanClause clause in bq.Clauses) + { + //recurse + var check = CheckQueryForExtractTerms(clause.Query); + if (!check) + { + return false; + } + } + } + + if (query is LateBoundQuery lbq) + { + return CheckQueryForExtractTerms(lbq.Wrapped); + } + + Type queryType = query.GetType(); + + if (typeof(TermRangeQuery).IsAssignableFrom(queryType) + || typeof(WildcardQuery).IsAssignableFrom(queryType) + || typeof(FuzzyQuery).IsAssignableFrom(queryType) + || (queryType.IsGenericType && queryType.GetGenericTypeDefinition().IsAssignableFrom(typeof(NumericRangeQuery<>)))) + { + return false; //ExtractTerms() not supported by TermRangeQuery, WildcardQuery,FuzzyQuery and will throw NotSupportedException + } + + return true; + } + } +} diff --git a/src/SImpl.SearchModule.Algolia/Application/LuceneEngine/LuceneSearchQueryWithFiltersAndFacets.cs b/src/SImpl.SearchModule.Algolia/Application/LuceneEngine/LuceneSearchQueryWithFiltersAndFacets.cs new file mode 100644 index 0000000..9f902f9 --- /dev/null +++ b/src/SImpl.SearchModule.Algolia/Application/LuceneEngine/LuceneSearchQueryWithFiltersAndFacets.cs @@ -0,0 +1,87 @@ +namespace SImpl.SearchModule.Algolia.Application.LuceneEngine +{ + public class LuceneSearchQueryWithFiltersAndFacets : LuceneSearchQuery + { + private readonly ISearchContext _searchContext; + private ISet _fieldsToLoad = null; + private BooleanQuery FilterQuery; + private Dictionary highlighterQueries; + + public LuceneSearchQueryWithFiltersAndFacets(ISearchContext searchContext, string category, Analyzer analyzer, + LuceneSearchOptions searchOptions, BooleanOperation occurance) : base(searchContext, category, analyzer, + searchOptions, occurance) + { + _searchContext = searchContext; + } + + public ISearchResults Execute(QueryOptions options = null) => Search(options); + + private ISearchResults Search(QueryOptions options) + { + // capture local + var query = Query; + + if (!string.IsNullOrEmpty(Category)) + { + // rebuild the query + IList existingClauses = query.Clauses; + + if (existingClauses.Count == 0) + { + // Nothing to search. This can occur in cases where an analyzer for a field doesn't return + // anything since it strips all values. + return EmptySearchResults.Instance; + } + + query = new BooleanQuery + { + // prefix the category field query as a must + { + GetFieldInternalQuery(ExamineFieldNames.CategoryFieldName, + new ExamineValue(Examineness.Explicit, Category), false), + Occur.MUST + } + }; + + // add the ones that we're already existing + foreach (var c in existingClauses) + { + query.Add(c); + } + } + + Filter filter = FilterQuery != null ? new QueryWrapperFilter(FilterQuery) : null; + + var executor = new LuceneSearchExecutorWithFacetsAndFilters(options, query, filter, highlighterQueries, + SortFields, _searchContext, _fieldsToLoad); + + var pagesResults = executor.Execute(); + + return pagesResults; + } + + public void Not(Query translate) + { + if (translate != null) + { + Query.Add(translate, Occur.MUST_NOT); + } + } + + public void And(Query translate) + { + if (translate != null) + { + Query.Add(translate, Occur.MUST); + } + } + + public void Or(Query translate) + { + if (translate != null) + { + Query.Add(translate, Occur.SHOULD); + } + } + } +} \ No newline at end of file diff --git a/src/SImpl.SearchModule.Algolia/Application/LuceneEngine/SearcherWithFacetsAndFilters.cs b/src/SImpl.SearchModule.Algolia/Application/LuceneEngine/SearcherWithFacetsAndFilters.cs new file mode 100644 index 0000000..54ea49f --- /dev/null +++ b/src/SImpl.SearchModule.Algolia/Application/LuceneEngine/SearcherWithFacetsAndFilters.cs @@ -0,0 +1,9 @@ +namespace SImpl.SearchModule.Algolia.Application.LuceneEngine +{ + public class LuceneSearchResultsWithFacetsAndFilters : LuceneSearchResults + { + public LuceneSearchResultsWithFacetsAndFilters(IReadOnlyCollection results, int totalItemCount) : base(results, totalItemCount) + { + } + } +} \ No newline at end of file diff --git a/src/SImpl.SearchModule.Algolia/Application/LuceneEngine/SimplExamineLuceneIndex.cs b/src/SImpl.SearchModule.Algolia/Application/LuceneEngine/SimplExamineLuceneIndex.cs new file mode 100644 index 0000000..bef0d35 --- /dev/null +++ b/src/SImpl.SearchModule.Algolia/Application/LuceneEngine/SimplExamineLuceneIndex.cs @@ -0,0 +1,42 @@ +namespace SImpl.SearchModule.Algolia.Application.LuceneEngine +{ + public class SimplExamineLuceneIndex : LuceneIndex + { + private readonly Lazy _searcher; + private ControlledRealTimeReopenThread _nrtReopenThread; + public SimplExamineLuceneIndex(ILoggerFactory loggerFactory, string name, IOptionsMonitor indexOptions) : base(loggerFactory, name, indexOptions) + { + _searcher = new Lazy(CreateSearcher); + } + public override ISearcher Searcher => _searcher.Value; + private LuceneSearcher CreateSearcher() + { + var possibleSuffixes = new[] { "Index", "Indexer" }; + var name = Name; + foreach (var suffix in possibleSuffixes) + { + //trim the "Indexer" / "Index" suffix if it exists + if (!name.EndsWith(suffix)) + continue; + name = name.Substring(0, name.LastIndexOf(suffix, StringComparison.Ordinal)); + } + + TrackingIndexWriter writer = IndexWriter; + var searcherManager = new SearcherManager(writer.IndexWriter, true, new SearcherFactory()); + searcherManager.AddListener(this); + + _nrtReopenThread = new ControlledRealTimeReopenThread(writer, searcherManager, 5.0, 1.0) + { + Name = $"{Name} NRT Reopen Thread", + IsBackground = true + }; + + _nrtReopenThread.Start(); + + // wait for most recent changes when first creating the searcher + WaitForChanges(); + + return new SimplExamineLuceneSearcher(name + "Searcher", searcherManager, FieldAnalyzer, FieldValueTypeCollection); + } + } +} \ No newline at end of file diff --git a/src/SImpl.SearchModule.Algolia/Application/LuceneEngine/SimplExamineLuceneSearcher.cs b/src/SImpl.SearchModule.Algolia/Application/LuceneEngine/SimplExamineLuceneSearcher.cs new file mode 100644 index 0000000..c447ca1 --- /dev/null +++ b/src/SImpl.SearchModule.Algolia/Application/LuceneEngine/SimplExamineLuceneSearcher.cs @@ -0,0 +1,18 @@ +namespace SImpl.SearchModule.Algolia.Application.LuceneEngine +{ + public class SimplExamineLuceneSearcher : LuceneSearcher + { + public SimplExamineLuceneSearcher(string name, SearcherManager searcherManager, Analyzer analyzer, FieldValueTypeCollection fieldValueTypeCollection) : base(name, searcherManager, analyzer, fieldValueTypeCollection) + { + } + public override IQuery CreateQuery(string category = null, BooleanOperation defaultOperation = BooleanOperation.And) + => CreateQuery(category, defaultOperation, LuceneAnalyzer, new LuceneSearchOptions()); + public IQuery CreateQuery(string category, BooleanOperation defaultOperation, Analyzer luceneAnalyzer, LuceneSearchOptions searchOptions) + { + if (luceneAnalyzer == null) + throw new ArgumentNullException(nameof(luceneAnalyzer)); + + return new LuceneSearchQueryWithFiltersAndFacets(GetSearchContext(), category, luceneAnalyzer, searchOptions, defaultOperation); + } + } +} \ No newline at end of file diff --git a/src/SImpl.SearchModule.Algolia/Application/QueryHandlers/ElasticSearchQueryHandler.cs b/src/SImpl.SearchModule.Algolia/Application/QueryHandlers/ElasticSearchQueryHandler.cs new file mode 100644 index 0000000..cbb46c3 --- /dev/null +++ b/src/SImpl.SearchModule.Algolia/Application/QueryHandlers/ElasticSearchQueryHandler.cs @@ -0,0 +1,96 @@ +using System.Globalization; +using SImpl.CQRS.Queries; +using SImpl.SearchModule.Abstraction.Models; +using SImpl.SearchModule.Abstraction.Queries; +using SImpl.SearchModule.Abstraction.Results; +using SImpl.SearchModule.Algolia.Application.LuceneEngine; +using SImpl.SearchModule.Algolia.Application.Services; +using SImpl.SearchModule.Algolia.Configuration; + +namespace SImpl.SearchModule.Algolia.Application.QueryHandlers +{ + public class ExamineSearchQueryHandler : IQueryHandler + { + private readonly IExamineManager _examineManager; + private readonly ExamineSearchConfiguration _configuration; + private readonly ILogger _logger; + private readonly IAlgoliaQueryTranslatorService _algoliaQueryTranslatorService; + + + public ExamineSearchQueryHandler(IExamineManager examineManager, ExamineSearchConfiguration configuration, + ILogger logger, IAlgoliaQueryTranslatorService algoliaQueryTranslatorService) + { + _examineManager = examineManager; + _configuration = configuration; + _logger = logger; + _algoliaQueryTranslatorService = algoliaQueryTranslatorService; + } + + public async Task HandleAsync(SearchQuery query) + { + var indexName = _configuration.IndexPrefixName + query.Index.ToLowerInvariant(); + _examineManager.TryGetIndex(indexName, + out IIndex examineIndex); + if (examineIndex == null) + { + _logger.LogError($"Examine index not found {indexName}"); + return new SimplQueryResult() + { + SearchModels = new List(), + Pagination = new Pagination() + { + Total = 0, + TotalNumberOfPages = 0 + }, + HighLighter = new HighLighter() + }; + } + LuceneSearchQueryWithFiltersAndFacets searchDescriptor = _algoliaQueryTranslatorService.Translate(examineIndex,query) as LuceneSearchQueryWithFiltersAndFacets; + var searcher = searchDescriptor.Execute(); + if (_configuration.EnableDebugInformation) + { + _logger.LogInformation(searchDescriptor.ToString()); + } + var resultModel = new SimplQueryResult() + { + SearchModels = searcher.Select(x=>TranslateModel(x)).ToList(), + // Aggregations = TranslateAggregations(result.Aggregations), + Pagination = new Pagination() + { + CurrentPage = query.Page, + PageSize = query.PageSize, + Total = searcher.TotalItemCount, + TotalNumberOfPages = (int)Math.Ceiling((searcher.TotalItemCount / (double)query.PageSize)) + }, + // HighLighter = TranslateHighLighter(result.Hits) + }; + return resultModel; + + //todo: implement when translation done + return new SimplQueryResult() + { + SearchModels = new List(), + Pagination = new Pagination() + { + Total = 0, + TotalNumberOfPages = 0 + }, + HighLighter = new HighLighter() + }; + } + + + + private ISearchModel TranslateModel(ISearchResult commandModels) + { + + var translatedModel = new BaseSearchModel(); + translatedModel.Id = commandModels.Id; + translatedModel.ContentType = commandModels.Values["contentType"]; + translatedModel.Culture = new CultureInfo( commandModels.Values["culture"].ToString()); + translatedModel.Facet = commandModels.Values["facet"].ToString(); +//todo: figure out mapping back rest of properties + return translatedModel; + } + } +} \ No newline at end of file diff --git a/src/SImpl.SearchModule.Algolia/Application/Services/BaseExamineQueryTranslatorService.cs b/src/SImpl.SearchModule.Algolia/Application/Services/BaseExamineQueryTranslatorService.cs new file mode 100644 index 0000000..8f569a9 --- /dev/null +++ b/src/SImpl.SearchModule.Algolia/Application/Services/BaseExamineQueryTranslatorService.cs @@ -0,0 +1,60 @@ +using Algolia.Search.Models.Search; +using SImpl.CQRS.Queries; +using SImpl.SearchModule.Abstraction.Queries; +using SImpl.SearchModule.Algolia.Application.LuceneEngine; +using SImpl.SearchModule.Algolia.Application.Services.SubQueries; + +namespace SImpl.SearchModule.Algolia.Application.Services +{ + public class BaseAlgoliaQueryTranslatorService : IAlgoliaQueryTranslatorService + { + private IEnumerable _collection; + private readonly IExamineManager _examineManager; + + public BaseAlgoliaQueryTranslatorService(IEnumerable collection, IExamineManager examineManager) + { + _collection = collection; + _examineManager = examineManager; + } + + public Query Translate(ISearchQuery query) + where T : class + { + + Query translatedQuery = new Query(); + + foreach (var booleanQuery in query) + { + var type = booleanQuery.Value.GetType(); + var handlerType = + typeof(ISubQueryExamineTranslator<>).MakeGenericType(type); + + var translator = + _collection.FirstOrDefault(x => x.GetType().GetInterfaces().Any(x => x == handlerType)); + if (translator == null) + { + continue; + } + switch (booleanQuery.Key) + { + case Occurance.MustNot: + translatedQuery.Filters+= $"NOT ({ translator.Translate(_collection, booleanQuery.Value)})"; + + break; + case Occurance.Must: + translatedQuery.Filters+= $"And ({ translator.Translate(_collection, booleanQuery.Value)})"; + + break; + case Occurance.Should: + translatedQuery.Filters+= $"OR ({ translator.Translate(_collection, booleanQuery.Value)})"; + break; + } + } + + return translatedQuery; + + } + + + } +} \ No newline at end of file diff --git a/src/SImpl.SearchModule.Algolia/Application/Services/IElasticSearchQueryTranslatorService.cs b/src/SImpl.SearchModule.Algolia/Application/Services/IElasticSearchQueryTranslatorService.cs new file mode 100644 index 0000000..fe85064 --- /dev/null +++ b/src/SImpl.SearchModule.Algolia/Application/Services/IElasticSearchQueryTranslatorService.cs @@ -0,0 +1,10 @@ +using Algolia.Search.Models.Search; +using SImpl.SearchModule.Abstraction.Queries; + +namespace SImpl.SearchModule.Algolia.Application.Services +{ + public interface IAlgoliaQueryTranslatorService + { + public Query Translate(IIndex index,ISearchQuery query) where T : class; + } +} \ No newline at end of file diff --git a/src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/BooleanSubQueryExamineTranslator.cs b/src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/BooleanSubQueryExamineTranslator.cs new file mode 100644 index 0000000..8b6e65b --- /dev/null +++ b/src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/BooleanSubQueryExamineTranslator.cs @@ -0,0 +1,66 @@ +using SImpl.SearchModule.Abstraction.Queries; +using SImpl.SearchModule.Abstraction.Queries.subqueries; +using SImpl.SearchModule.Algolia.Application.LuceneEngine; + +namespace SImpl.SearchModule.Algolia.Application.Services.SubQueries +{ + public class BooleanSubQueryExamineTranslator : ISubQueryExamineTranslator + { + + public string Translate(IEnumerable collection, ISearchSubQuery query) where TViewModel : class + { + var boolQuery = (BoolSearchSubQuery)query; + if (!boolQuery.NestedQueries.Any()) + { + return null; + } + var nestedExamineQuery=nestedQuery as LuceneSearchQueryWithFiltersAndFacets; + foreach (var booleanQuery in boolQuery.NestedQueries) + { + var type = booleanQuery.GetType(); + var handlerType = + typeof(ISubQueryExamineTranslator<>).MakeGenericType(type); + + var translator = + collection.FirstOrDefault(x => x.GetType().GetInterfaces().Any(x => x == handlerType)); + if (translator == null) + { + continue; + } + switch (booleanQuery.Occurance) + { + case Occurance.MustNot: + nestedExamineQuery.Not(translator.Translate(searcher,collection, booleanQuery)); + break; + case Occurance.Must: + nestedExamineQuery.And(translator.Translate(searcher,collection, booleanQuery)); + break; + case Occurance.Should: + nestedExamineQuery.Or(translator.Translate(searcher,collection, booleanQuery)); + break; + } + } + + if (!nestedQuery.Query.Clauses.Any()) + { + return null; + } + return nestedQuery.Query; + + } + + private BooleanOperation MapOccuranceToExamine(Occurance queryOccurance) + { + switch (queryOccurance) + { + case Occurance.Should: + return BooleanOperation.Or; + case Occurance.Must: + return BooleanOperation.And; + case Occurance.MustNot: + return BooleanOperation.Not; + } + return BooleanOperation.Or; + } + } +} \ No newline at end of file diff --git a/src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/DateRangeQueryExamineTranslator.cs b/src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/DateRangeQueryExamineTranslator.cs new file mode 100644 index 0000000..b9b2452 --- /dev/null +++ b/src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/DateRangeQueryExamineTranslator.cs @@ -0,0 +1,43 @@ +using SImpl.SearchModule.Abstraction.Queries; +using SImpl.SearchModule.Abstraction.Queries.subqueries; +using SImpl.SearchModule.Algolia.Application.LuceneEngine; + +namespace SImpl.SearchModule.Algolia.Application.Services.SubQueries +{ + public class DateRangeQueryExamineTranslator : ISubQueryExamineTranslator + { + public Query Translate(ISearcher searcher, IEnumerable collection, + ISearchSubQuery query) where TViewModel : class + { + var searcherBase = searcher as BaseLuceneSearcher; + var nestedQuery = new LuceneSearchQueryWithFiltersAndFacets(searcherBase.GetSearchContext(), "baseSearch", + searcherBase.LuceneAnalyzer, new LuceneSearchOptions(), MapOccuranceToExamine(query.Occurance)); + var termSubQuery = (DateRangeQuery)query; + if (termSubQuery.MinValue == null && termSubQuery.MaxValue == null || termSubQuery.Field == null) + { + return null; + } + nestedQuery.RangeQuery( + new[] { termSubQuery.Field }, + termSubQuery.MinValue, + termSubQuery.MaxValue, + maxInclusive: termSubQuery.IncludeMaxEdge, minInclusive: termSubQuery.IncludeMinEdge); + return nestedQuery.Query; + } + + private BooleanOperation MapOccuranceToExamine(Occurance queryOccurance) + { + switch (queryOccurance) + { + case Occurance.Should: + return BooleanOperation.Or; + case Occurance.Must: + return BooleanOperation.And; + case Occurance.MustNot: + return BooleanOperation.Not; + } + + return BooleanOperation.Or; + } + } +} \ No newline at end of file diff --git a/src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/FuzzySubQueryExamineTranslator.cs b/src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/FuzzySubQueryExamineTranslator.cs new file mode 100644 index 0000000..e83dc3f --- /dev/null +++ b/src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/FuzzySubQueryExamineTranslator.cs @@ -0,0 +1,36 @@ +using SImpl.SearchModule.Abstraction.Queries; +using SImpl.SearchModule.Algolia.Application.LuceneEngine; +using FuzzyQuery = SImpl.SearchModule.Abstraction.Queries.FuzzyQuery; + +namespace SImpl.SearchModule.Algolia.Application.Services.SubQueries +{ + public class FuzzySubQueryExamineTranslator : ISubQueryExamineTranslator + { + public Query Translate(ISearcher searcher,IEnumerable collection, ISearchSubQuery query) where TViewModel : class + { + var searcherBase = searcher as BaseLuceneSearcher; + var nestedQuery = new LuceneSearchQueryWithFiltersAndFacets(searcherBase.GetSearchContext(),"baseSearch" ,searcherBase.LuceneAnalyzer, new LuceneSearchOptions(), MapOccuranceToExamine(query.Occurance)); + var termSubQuery = (FuzzyQuery)query; + if (termSubQuery.Value == null || termSubQuery.Field == null) + { + return null; + } + nestedQuery.Field(termSubQuery.Field,termSubQuery.Value.ToString().Fuzzy()); + return nestedQuery.Query; + } + + private BooleanOperation MapOccuranceToExamine(Occurance queryOccurance) + { + switch (queryOccurance) + { + case Occurance.Should: + return BooleanOperation.Or; + case Occurance.Must: + return BooleanOperation.And; + case Occurance.MustNot: + return BooleanOperation.Not; + } + return BooleanOperation.Or; + } + } +} \ No newline at end of file diff --git a/src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/ISubQueryExamineTranslator.cs b/src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/ISubQueryExamineTranslator.cs new file mode 100644 index 0000000..2266429 --- /dev/null +++ b/src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/ISubQueryExamineTranslator.cs @@ -0,0 +1,16 @@ +using Algolia.Search.Models.Search; +using SImpl.SearchModule.Abstraction.Queries; + +namespace SImpl.SearchModule.Algolia.Application.Services.SubQueries +{ + public interface ISubQueryExamineTranslator : ISubQueryElasticTranslator where T : ISearchSubQuery + { + + } + + public interface ISubQueryElasticTranslator + { + public string Translate(IEnumerable collection, + ISearchSubQuery query) where TViewModel : class; + } +} \ No newline at end of file diff --git a/src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/LongRangeQueryExamineTranslator.cs b/src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/LongRangeQueryExamineTranslator.cs new file mode 100644 index 0000000..2b6cf75 --- /dev/null +++ b/src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/LongRangeQueryExamineTranslator.cs @@ -0,0 +1,43 @@ +using SImpl.SearchModule.Abstraction.Queries; +using SImpl.SearchModule.Algolia.Application.LuceneEngine; +using LongRange = SImpl.SearchModule.Abstraction.Queries.subqueries.LongRange; + +namespace SImpl.SearchModule.Algolia.Application.Services.SubQueries +{ + public class LongRangeQueryExamineTranslator : ISubQueryExamineTranslator + { + public Query Translate(ISearcher searcher, IEnumerable collection, + ISearchSubQuery query) where TViewModel : class + { + var searcherBase = searcher as BaseLuceneSearcher; + var nestedQuery = new LuceneSearchQueryWithFiltersAndFacets(searcherBase.GetSearchContext(), "baseSearch", + searcherBase.LuceneAnalyzer, new LuceneSearchOptions(), MapOccuranceToExamine(query.Occurance)); + var termSubQuery = (LongRange)query; + if (termSubQuery.MinValue == null && termSubQuery.MaxValue == null || termSubQuery.Field == null) + { + return null; + } + nestedQuery.RangeQuery( + new[] { termSubQuery.Field }, + termSubQuery.MinValue, + termSubQuery.MaxValue, + maxInclusive: termSubQuery.IncludeMaxEdge, minInclusive: termSubQuery.IncludeMinEdge); + return nestedQuery.Query; + } + + private BooleanOperation MapOccuranceToExamine(Occurance queryOccurance) + { + switch (queryOccurance) + { + case Occurance.Should: + return BooleanOperation.Or; + case Occurance.Must: + return BooleanOperation.And; + case Occurance.MustNot: + return BooleanOperation.Not; + } + + return BooleanOperation.Or; + } + } +} \ No newline at end of file diff --git a/src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/PrefixSubQueryExamineTranslator.cs b/src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/PrefixSubQueryExamineTranslator.cs new file mode 100644 index 0000000..c0a145b --- /dev/null +++ b/src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/PrefixSubQueryExamineTranslator.cs @@ -0,0 +1,36 @@ +using SImpl.SearchModule.Abstraction.Queries; +using SImpl.SearchModule.Abstraction.Queries.subqueries; +using SImpl.SearchModule.Algolia.Application.LuceneEngine; + +namespace SImpl.SearchModule.Algolia.Application.Services.SubQueries +{ + public class PrefixSubQueryExamineTranslator: ISubQueryExamineTranslator + { + public Query Translate(ISearcher searcher,IEnumerable collection, ISearchSubQuery query) where TViewModel : class + { + var searcherBase = searcher as BaseLuceneSearcher; + var nestedQuery = new LuceneSearchQueryWithFiltersAndFacets(searcherBase.GetSearchContext(),"baseSearch" ,searcherBase.LuceneAnalyzer, new LuceneSearchOptions(), MapOccuranceToExamine(query.Occurance)); + var termSubQuery = (PrefixSubQuery)query; + if (termSubQuery.Value == null || termSubQuery.Field == null) + { + return null; + } + nestedQuery.Field(termSubQuery.Field,termSubQuery.Value.ToString().MultipleCharacterWildcard()); + return nestedQuery.Query; + } + + private BooleanOperation MapOccuranceToExamine(Occurance queryOccurance) + { + switch (queryOccurance) + { + case Occurance.Should: + return BooleanOperation.Or; + case Occurance.Must: + return BooleanOperation.And; + case Occurance.MustNot: + return BooleanOperation.Not; + } + return BooleanOperation.Or; + } + } +} \ No newline at end of file diff --git a/src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/RangeQueryElasticTranslator.cs b/src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/RangeQueryElasticTranslator.cs new file mode 100644 index 0000000..d622753 --- /dev/null +++ b/src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/RangeQueryElasticTranslator.cs @@ -0,0 +1,43 @@ +using SImpl.SearchModule.Abstraction.Queries; +using SImpl.SearchModule.Abstraction.Queries.subqueries; +using SImpl.SearchModule.Algolia.Application.LuceneEngine; + +namespace SImpl.SearchModule.Algolia.Application.Services.SubQueries +{ + public class NumericRangeQueryExamineTranslator : ISubQueryExamineTranslator + { + public Query Translate(ISearcher searcher, IEnumerable collection, + ISearchSubQuery query) where TViewModel : class + { + var searcherBase = searcher as BaseLuceneSearcher; + var nestedQuery = new LuceneSearchQueryWithFiltersAndFacets(searcherBase.GetSearchContext(), "baseSearch", + searcherBase.LuceneAnalyzer, new LuceneSearchOptions(), MapOccuranceToExamine(query.Occurance)); + var termSubQuery = (NumericRange)query; + if (termSubQuery.MinValue == null && termSubQuery.MaxValue == null || termSubQuery.Field == null) + { + return null; + } + nestedQuery.RangeQuery( + new[] { termSubQuery.Field }, + termSubQuery.MinValue, + termSubQuery.MaxValue, + maxInclusive: termSubQuery.IncludeMaxEdge, minInclusive: termSubQuery.IncludeMinEdge); + return nestedQuery.Query; + } + + private BooleanOperation MapOccuranceToExamine(Occurance queryOccurance) + { + switch (queryOccurance) + { + case Occurance.Should: + return BooleanOperation.Or; + case Occurance.Must: + return BooleanOperation.And; + case Occurance.MustNot: + return BooleanOperation.Not; + } + + return BooleanOperation.Or; + } + } +} \ No newline at end of file diff --git a/src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/TermSubQueryExamineTranslator.cs b/src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/TermSubQueryExamineTranslator.cs new file mode 100644 index 0000000..c326e59 --- /dev/null +++ b/src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/TermSubQueryExamineTranslator.cs @@ -0,0 +1,36 @@ +using SImpl.SearchModule.Abstraction.Queries; +using SImpl.SearchModule.Abstraction.Queries.subqueries; +using SImpl.SearchModule.Algolia.Application.LuceneEngine; + +namespace SImpl.SearchModule.Algolia.Application.Services.SubQueries +{ + public class TermSubQueryExamineTranslator : ISubQueryExamineTranslator + { + public Query Translate(ISearcher searcher,IEnumerable collection, ISearchSubQuery query) where TViewModel : class + { + var searcherBase = searcher as BaseLuceneSearcher; + var nestedQuery = new LuceneSearchQueryWithFiltersAndFacets(searcherBase.GetSearchContext(),"baseSearch" ,searcherBase.LuceneAnalyzer, new LuceneSearchOptions(), MapOccuranceToExamine(query.Occurance)); + var termSubQuery = (TermSubQuery)query; + if (termSubQuery.Value == null || termSubQuery.Field == null) + { + return null; + } + nestedQuery.Field(termSubQuery.Field,termSubQuery.Value.ToString()); + return nestedQuery.Query; + } + + private BooleanOperation MapOccuranceToExamine(Occurance queryOccurance) + { + switch (queryOccurance) + { + case Occurance.Should: + return BooleanOperation.Or; + case Occurance.Must: + return BooleanOperation.And; + case Occurance.MustNot: + return BooleanOperation.Not; + } + return BooleanOperation.Or; + } + } +} \ No newline at end of file diff --git a/src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/TermsSubQueryExamineTranslator.cs b/src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/TermsSubQueryExamineTranslator.cs new file mode 100644 index 0000000..92737d5 --- /dev/null +++ b/src/SImpl.SearchModule.Algolia/Application/Services/SubQueries/TermsSubQueryExamineTranslator.cs @@ -0,0 +1,41 @@ +using SImpl.SearchModule.Abstraction.Queries; +using SImpl.SearchModule.Abstraction.Queries.subqueries; +using SImpl.SearchModule.Algolia.Application.LuceneEngine; + +namespace SImpl.SearchModule.Algolia.Application.Services.SubQueries +{ + public class TermsSubQueryExamineTranslator : ISubQueryExamineTranslator + { + public Query Translate(ISearcher searcher, IEnumerable collection, + ISearchSubQuery query) where TViewModel : class + { + var searcherBase = searcher as BaseLuceneSearcher; + + var nestedQuery = new LuceneSearchQueryWithFiltersAndFacets(searcherBase.GetSearchContext(), "baseSearch", + searcherBase.LuceneAnalyzer, new LuceneSearchOptions(), MapOccuranceToExamine(query.Occurance)); + var termSubQuery = (TermsSubQuery)query; + if (termSubQuery.Value == null || termSubQuery.Field == null) + { + return null; + } + nestedQuery.GroupedAnd(new List() { termSubQuery.Field }.ToArray(), + termSubQuery.Value.Select(x => x.ToString()).ToArray()); + return nestedQuery.Query; + } + + private BooleanOperation MapOccuranceToExamine(Occurance queryOccurance) + { + switch (queryOccurance) + { + case Occurance.Should: + return BooleanOperation.Or; + case Occurance.Must: + return BooleanOperation.And; + case Occurance.MustNot: + return BooleanOperation.Not; + } + + return BooleanOperation.Or; + } + } +} \ No newline at end of file diff --git a/src/SImpl.SearchModule.Algolia/Configuration/AuthenticationModes.cs b/src/SImpl.SearchModule.Algolia/Configuration/AuthenticationModes.cs new file mode 100644 index 0000000..bba5a74 --- /dev/null +++ b/src/SImpl.SearchModule.Algolia/Configuration/AuthenticationModes.cs @@ -0,0 +1,11 @@ +namespace SImpl.SearchModule.Algolia.Configuration +{ + public enum AuthenticationModes + { + Default, + Uri, + CloudAuthentication, + CloudApiAuthentication, + ConnectionSettingsValues + } +} \ No newline at end of file diff --git a/src/SImpl.SearchModule.Algolia/Configuration/ExamineSearchConfiguration.cs b/src/SImpl.SearchModule.Algolia/Configuration/ExamineSearchConfiguration.cs new file mode 100644 index 0000000..ea63c86 --- /dev/null +++ b/src/SImpl.SearchModule.Algolia/Configuration/ExamineSearchConfiguration.cs @@ -0,0 +1,13 @@ +namespace SImpl.SearchModule.Algolia.Configuration +{ + public class AlgoliaSearchConfiguration + { + public string IndexPrefixName { get; set; } = ""; + public List IndexName { get; set; } = new List(); + + public Type LuceneDirectoryFactory { get; set; } = typeof(SyncedFileSystemDirectoryFactory); + public FieldDefinitionCollection FieldsDefinition { get; set; } + public bool EnableDebugInformation { get; set; } + public Type SearchService { get; set; } + } +} \ No newline at end of file diff --git a/src/SImpl.SearchModule.Algolia/Configuration/HeadlessSearchServerConfigExtensions.cs b/src/SImpl.SearchModule.Algolia/Configuration/HeadlessSearchServerConfigExtensions.cs new file mode 100644 index 0000000..ca08cf1 --- /dev/null +++ b/src/SImpl.SearchModule.Algolia/Configuration/HeadlessSearchServerConfigExtensions.cs @@ -0,0 +1,26 @@ +using SImpl.Host.Builders; + +namespace SImpl.SearchModule.Algolia.Configuration +{ + public static class ElasticSearchModuleConfigExtensions + { + public static void UseExamine(this ISImplHostBuilder hostBuilder, Action configure) + { + var existingModule = hostBuilder.GetConfiguredModule(); + var config = existingModule != null? existingModule.Config : new ExamineSearchConfiguration(); + configure.Invoke(config); + if (existingModule == null) + { + AttachModule(hostBuilder, config); + } + } + + private static void AttachModule(ISImplHostBuilder novicellAppBuilder, + ExamineSearchConfiguration examineSearchConfiguration) + { + var module = new ExamineSearchModule(examineSearchConfiguration); + + novicellAppBuilder.AttachNewOrGetConfiguredModule(() => module); + } + } +} \ No newline at end of file diff --git a/src/SImpl.SearchModule.Algolia/ExamineSearchModule.cs b/src/SImpl.SearchModule.Algolia/ExamineSearchModule.cs new file mode 100644 index 0000000..508c0d0 --- /dev/null +++ b/src/SImpl.SearchModule.Algolia/ExamineSearchModule.cs @@ -0,0 +1,67 @@ +using System.Reflection; +using Algolia.Search.Clients; +using Microsoft.Extensions.DependencyInjection; +using SImpl.CQRS.Commands; +using SImpl.CQRS.Queries; +using SImpl.Host.Builders; +using SImpl.Modules; +using SImpl.SearchModule.Algolia.Application.Factories; +using SImpl.SearchModule.Algolia.Application.LuceneEngine; +using SImpl.SearchModule.Algolia.Application.Services; +using SImpl.SearchModule.Algolia.Application.Services.SubQueries; +using SImpl.SearchModule.Algolia.Configuration; + +namespace SImpl.SearchModule.Algolia +{ + + public class ExamineSearchModule : IAspNetPreModule,IHostBuilderConfigureModule, ISImplModule + { + public ExamineSearchModule(AlgoliaSearchConfiguration config) + { + Config = config; + } + + public AlgoliaSearchConfiguration Config { get; set; } + + public string Name { get; } = nameof(ExamineSearchModule); + public void Configure(IApplicationBuilder app, IWebHostEnvironment env) + { + + } + + public void ConfigureHostBuilder(ISImplHostBuilder hostBuilder) + { + hostBuilder.UseCqrsCommands(x => x.AddCommandHandlersFromAssembly(typeof(ExamineSearchModule).Assembly)); + hostBuilder.UseCqrsQueries(x => x.AddQueryHandlersFromAssembly(typeof(ExamineSearchModule).Assembly)); + + + } + + public void ConfigureServices(IServiceCollection services) + { + services.AddSingleton(typeof(AlgoliaSearchConfiguration), (services) => Config); + services.AddSingleton(new SearchClient(Config.AppId, Config.ApiKey)); + services.AddTransient(); + services.AddTransient(typeof(IIndexingService), + Config.SearchService); + + services.AddSingleton(); + services.Scan(s => + s.FromAssemblies(new List(){typeof(ExamineSearchModule).Assembly}) + .AddClasses(c => c.AssignableTo(typeof(IAlgoliaQueryTranslatorService))) + .AsImplementedInterfaces() + .WithTransientLifetime()); + services.Scan(s => + s.FromAssemblies(new List(){typeof(ExamineSearchModule).Assembly}) + .AddClasses(c => c.AssignableTo(typeof(ISubQueryExamineTranslator<>))) + .AsImplementedInterfaces() + .WithTransientLifetime()); + foreach (var indexName in Config.IndexName) + { + services.AddExamineLuceneIndex(Config.IndexPrefixName+indexName, Config.FieldsDefinition); + + } + } + + } +} \ No newline at end of file diff --git a/src/SImpl.SearchModule.Algolia/Models/AnalyzerType.cs b/src/SImpl.SearchModule.Algolia/Models/AnalyzerType.cs new file mode 100644 index 0000000..bb0b18a --- /dev/null +++ b/src/SImpl.SearchModule.Algolia/Models/AnalyzerType.cs @@ -0,0 +1,11 @@ +namespace SImpl.SearchModule.Algolia.Models +{ + public enum AnalyzerType + { + Keyword, + Binary, + Boolean, + Text, + Date + } +} \ No newline at end of file diff --git a/src/SImpl.SearchModule.Algolia/Models/ElasticProperty.cs b/src/SImpl.SearchModule.Algolia/Models/ElasticProperty.cs new file mode 100644 index 0000000..0274d1b --- /dev/null +++ b/src/SImpl.SearchModule.Algolia/Models/ElasticProperty.cs @@ -0,0 +1,20 @@ +namespace SImpl.SearchModule.Algolia.Models +{ + public class TextElasticProperty : IElasticProperty + { + public string Name { get; set; } + public double? Boost { get; set; } + public bool? SplitQueriesOnWhiteSpace { get; set; } + } + public class DateElasticProperty : IElasticProperty + { + public string Name { get; set; } + public double? Boost { get; set; } + public bool? SplitQueriesOnWhiteSpace { get; set; } + public string Format { get; set; } + } + public interface IElasticProperty + { + string Name { get; set; } + } +} \ No newline at end of file diff --git a/src/SImpl.SearchModule.Algolia/Models/ElasticSearchModel.cs b/src/SImpl.SearchModule.Algolia/Models/ElasticSearchModel.cs new file mode 100644 index 0000000..3ffb5e2 --- /dev/null +++ b/src/SImpl.SearchModule.Algolia/Models/ElasticSearchModel.cs @@ -0,0 +1,26 @@ +using SImpl.Common; + +namespace SImpl.SearchModule.Algolia.Models +{ + public class ElasticSearchModel : IEntity + { + public string Id { get; set; } + public List AdditionalKeys { get; set; } = new List(); + public DateTime? IndexedAt { get; set; } + public string Culture { get; set; } + public string SearchCulture + { + get + { + return Culture.ToLower().Replace("-",""); + } + set { } + } + public string Content { get; set; } + public IEnumerable Tags { get; set; } = new List(); + public string ContentType { get; set; } + public string Facet { get; set; } + public string ViewModelType { get; set; } + public IDictionary> CustomProperties { get; set; } = new Dictionary>(); + } +} \ No newline at end of file diff --git a/src/SImpl.SearchModule.Algolia/SImpl.SearchModule.Algolia.csproj b/src/SImpl.SearchModule.Algolia/SImpl.SearchModule.Algolia.csproj new file mode 100644 index 0000000..8d8b7d6 --- /dev/null +++ b/src/SImpl.SearchModule.Algolia/SImpl.SearchModule.Algolia.csproj @@ -0,0 +1,19 @@ + + + + net6.0 + enable + enable + False + + + + + + + + + + + + diff --git a/src/SImpl.SearchModule.sln b/src/SImpl.SearchModule.sln index b86b167..edb2179 100644 --- a/src/SImpl.SearchModule.sln +++ b/src/SImpl.SearchModule.sln @@ -14,6 +14,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SImpl.SearchModule.Core", " EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SImpl.SearchModule.Client", "SImpl.SearchModule.Client\SImpl.SearchModule.Client.csproj", "{31AA6274-2653-4F02-8F4E-3BFA3B7CB532}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SImpl.SearchModule.Algolia", "SImpl.SearchModule.Algolia\SImpl.SearchModule.Algolia.csproj", "{C8906583-F199-4B87-A4AF-589CF9BF60B7}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -48,5 +50,9 @@ Global {31AA6274-2653-4F02-8F4E-3BFA3B7CB532}.Debug|Any CPU.Build.0 = Debug|Any CPU {31AA6274-2653-4F02-8F4E-3BFA3B7CB532}.Release|Any CPU.ActiveCfg = Release|Any CPU {31AA6274-2653-4F02-8F4E-3BFA3B7CB532}.Release|Any CPU.Build.0 = Release|Any CPU + {C8906583-F199-4B87-A4AF-589CF9BF60B7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C8906583-F199-4B87-A4AF-589CF9BF60B7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C8906583-F199-4B87-A4AF-589CF9BF60B7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C8906583-F199-4B87-A4AF-589CF9BF60B7}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal