From 4b0609dd64dad2dcc260f30c4e3ee9464d81bbcf Mon Sep 17 00:00:00 2001 From: Niall Gallagher Date: Mon, 29 Jan 2018 23:33:03 +0000 Subject: [PATCH 1/2] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3e60f127a..13fdd4e94 100644 --- a/README.md +++ b/README.md @@ -511,7 +511,7 @@ ResultSet results = cars.retrieve(query, queryOptions(orderBy(descending(Ca ResultSet results = cars.retrieve(query, queryOptions(orderBy(descending(Car.PRICE), ascending(Car.DOORS)))); ``` -Note that ordering results as above uses the default _materialize_ ordering strategy. This is relatively expensive, dependent on the number of objects matching the query, and can cause latency in accessing the first object. It requires all results to be materialized into a sorted set up-front _before iteration can begin_. However ordering results in this way also implicitly eliminates duplicates. +Note that ordering results as above uses the default _materialize_ ordering strategy. This is relatively expensive, dependent on the number of objects matching the query, and can cause latency in accessing the first object. It requires all results to be materialized into a sorted set up-front _before iteration can begin_. ### Index-accelerated ordering ### From 27d4a1390399e5d66d61de8dba51c503d5146989 Mon Sep 17 00:00:00 2001 From: Eduardo Barrios <3553088+ejbarrios@users.noreply.github.com> Date: Sat, 11 Aug 2018 22:47:13 -0700 Subject: [PATCH 2/2] Enable compound queries to use compound indices even if the attributes are specified in different order --- .../compound/support/CompoundAttribute.java | 25 ++++++- .../index/compound/support/CompoundQuery.java | 37 ++++++++-- .../compound/support/CompoundValueTuple.java | 37 ++++++++-- .../IndexedCollectionFunctionalTest.java | 3 +- .../support/ScrambledQueriesTest.java | 71 +++++++++++++++++++ 5 files changed, 160 insertions(+), 13 deletions(-) create mode 100644 code/src/test/java/com/googlecode/cqengine/index/compound/support/ScrambledQueriesTest.java diff --git a/code/src/main/java/com/googlecode/cqengine/index/compound/support/CompoundAttribute.java b/code/src/main/java/com/googlecode/cqengine/index/compound/support/CompoundAttribute.java index 96ecbdfa0..73a5909ef 100644 --- a/code/src/main/java/com/googlecode/cqengine/index/compound/support/CompoundAttribute.java +++ b/code/src/main/java/com/googlecode/cqengine/index/compound/support/CompoundAttribute.java @@ -78,7 +78,7 @@ public CompoundAttribute(List> attributes) { if (attributes.size() < 2) { throw new IllegalStateException("Cannot create a compound index on fewer than two attributes: " + attributes.size()); } - this.attributes = attributes; + this.attributes = deduplicateAndSort(attributes); } public int size() { @@ -131,15 +131,25 @@ public Iterable> getValues(O object, QueryOptions queryOpt // Values for first attribute: 1 // Values for second attribute: "bar", "baz" // Values for third attribute: 2.0, 3.0, 4.0 + // + // or in list form : + // [[1], [bar, baz], [2.0, 3.0, 4.0]] + // // ...then we should generate and index the object against the following tuples: // [[1, bar, 2.0], [1, bar, 3.0], [1, bar, 4.0], [1, baz, 2.0], [1, baz, 3.0], [1, baz, 4.0]] + // note that ordering of attributes is preserved, we take advantage of this below List> listsOfValueCombinations = TupleCombinationGenerator.generateCombinations(attributeValueLists); // STEP 3. // Wrap each of the unique combinations in a CompoundValueTuple object... List> tuples = new ArrayList>(listsOfValueCombinations.size()); for (List valueCombination : listsOfValueCombinations) { - tuples.add(new CompoundValueTuple(valueCombination)); + Map, Object> mappedTuples = new HashMap, Object>(); + // here we take advantage of the fact that ordering of attributes is preserved when the tuples are generated + for (int i = 0; i < attributes.size(); i++) { + mappedTuples.put(attributes.get(i), valueCombination.get(i)); + } + tuples.add(new CompoundValueTuple(mappedTuples)); } // Return the list of CompoundValueTuple objects... return tuples; @@ -169,4 +179,15 @@ public String toString() { '}'; } + private static List> deduplicateAndSort(List> attributes) { + final List> deduplicatedAttributes = new ArrayList>(new HashSet>(attributes)); + + Collections.sort(deduplicatedAttributes, new Comparator>() { + @Override + public int compare(Attribute o1, Attribute o2) { + return o1.getAttributeName().compareTo(o2.getAttributeName()); + } + }); + return deduplicatedAttributes; + } } diff --git a/code/src/main/java/com/googlecode/cqengine/index/compound/support/CompoundQuery.java b/code/src/main/java/com/googlecode/cqengine/index/compound/support/CompoundQuery.java index 4fb5a4275..c1cf5a1f2 100644 --- a/code/src/main/java/com/googlecode/cqengine/index/compound/support/CompoundQuery.java +++ b/code/src/main/java/com/googlecode/cqengine/index/compound/support/CompoundQuery.java @@ -17,13 +17,18 @@ import com.googlecode.cqengine.attribute.Attribute; import com.googlecode.cqengine.query.Query; +import com.googlecode.cqengine.query.logical.And; +import com.googlecode.cqengine.query.logical.LogicalQuery; import com.googlecode.cqengine.query.option.QueryOptions; import com.googlecode.cqengine.query.simple.Equal; import com.googlecode.cqengine.query.simple.SimpleQuery; -import com.googlecode.cqengine.query.logical.And; import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; /** * A query which wraps a {@link CompoundAttribute}, used only in the query engine's internal communication @@ -71,16 +76,18 @@ public CompoundAttribute getCompoundAttribute() { } public CompoundValueTuple getCompoundValueTuple() { - List attributeValues = new ArrayList(andQuery.getSimpleQueries().size()); + Map, Object> attributeValues = new HashMap, Object>(); for (SimpleQuery simpleQuery : andQuery.getSimpleQueries()) { Equal equal = (Equal) simpleQuery; - attributeValues.add(equal.getValue()); + attributeValues.put(equal.getAttribute(), equal.getValue()); } return new CompoundValueTuple(attributeValues); } public static CompoundQuery fromAndQueryIfSuitable(And andQuery) { - if (andQuery.hasLogicalQueries()) { + andQuery = flatten(andQuery); + + if (andQuery == null) { return null; } List> attributeList = new ArrayList>(andQuery.getSimpleQueries().size()); @@ -95,4 +102,26 @@ public static CompoundQuery fromAndQueryIfSuitable(And andQuery) { return new CompoundQuery(andQuery, compoundAttribute); } + /** + * Flatten an And query, bringing any nested And queries up the top level query if possible + * + * @param andQuery the And query to flatten + * @return the flattened And query, or null if it cannot be converted into a flat And query + */ + private static And flatten(And andQuery) { + final Set> flatQuerySet = new HashSet>(); + for (LogicalQuery childQuery : andQuery.getLogicalQueries()) { + if (childQuery instanceof And) { + And flatQuery = flatten((And) childQuery); + if (flatQuery == null) { + return null; + } + flatQuerySet.addAll(flatQuery.getSimpleQueries()); + } else { + return null; + } + } + flatQuerySet.addAll(andQuery.getSimpleQueries()); + return new And(flatQuerySet); + } } diff --git a/code/src/main/java/com/googlecode/cqengine/index/compound/support/CompoundValueTuple.java b/code/src/main/java/com/googlecode/cqengine/index/compound/support/CompoundValueTuple.java index 778db1d0d..862d70789 100644 --- a/code/src/main/java/com/googlecode/cqengine/index/compound/support/CompoundValueTuple.java +++ b/code/src/main/java/com/googlecode/cqengine/index/compound/support/CompoundValueTuple.java @@ -1,12 +1,12 @@ /** * Copyright 2012-2015 Niall Gallagher - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,8 +17,12 @@ import com.googlecode.cqengine.attribute.Attribute; +import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.List; +import java.util.Map; +import java.util.TreeMap; /** * A tuple (ordered list) of values, extracted from the fields of an object, according to, and in the same order as, the @@ -35,8 +39,8 @@ public class CompoundValueTuple { private final List attributeValues; private final int hashCode; - public CompoundValueTuple(List attributeValues) { - this.attributeValues = attributeValues; + public CompoundValueTuple(Map, ?> attributeToValues) { + this.attributeValues = getOrderedAttributeValues(attributeToValues); this.hashCode = attributeValues.hashCode(); } @@ -69,4 +73,25 @@ public String toString() { ", hashCode=" + hashCode + '}'; } + + /** + * Given attribute values, sort them to ensure that compound index entries in the index engine + * work regardless of how the given CompoundQuery is ordered + *

+ * Note: to be more specific, without this, the {@code equals} method will return false when + * two CompoundValueTuples contain the same values but are ordered differently + * + * @param attributeToValues a map of attributes and their corresponding values that were extracted from an object or query + * @return a list of ordered attribute values that will be used in this CompoundValueTuple + */ + private static List getOrderedAttributeValues(Map, ?> attributeToValues) { + final TreeMap, Object> attributeValuesSortedByAttributeName = new TreeMap, Object>(new Comparator>() { + @Override + public int compare(Attribute o1, Attribute o2) { + return o1.getAttributeName().compareTo(o2.getAttributeName()); + } + }); + attributeValuesSortedByAttributeName.putAll(attributeToValues); + return new ArrayList(attributeValuesSortedByAttributeName.values()); + } } diff --git a/code/src/test/java/com/googlecode/cqengine/IndexedCollectionFunctionalTest.java b/code/src/test/java/com/googlecode/cqengine/IndexedCollectionFunctionalTest.java index eced4ab16..a50c524d4 100644 --- a/code/src/test/java/com/googlecode/cqengine/IndexedCollectionFunctionalTest.java +++ b/code/src/test/java/com/googlecode/cqengine/IndexedCollectionFunctionalTest.java @@ -15,6 +15,7 @@ */ package com.googlecode.cqengine; +import com.google.common.collect.ImmutableMap; import com.googlecode.cqengine.attribute.Attribute; import com.googlecode.cqengine.attribute.SimpleAttribute; import com.googlecode.cqengine.attribute.StandingQueryAttribute; @@ -524,7 +525,7 @@ public CompoundValueTuple getQuantizedValue(CompoundValueTuple tuple) String manufacturer = (String) tupleValues.next(); String model = (String) tupleValues.next(); String quantizedModel = "Focus".equals(model) ? "Focus" : "Other"; - return new CompoundValueTuple(Arrays.asList(manufacturer, quantizedModel)); + return new CompoundValueTuple(ImmutableMap., Object>of(Car.MANUFACTURER, manufacturer, Car.MODEL, quantizedModel)); } }, Car.MANUFACTURER, Car.MODEL) ), diff --git a/code/src/test/java/com/googlecode/cqengine/index/compound/support/ScrambledQueriesTest.java b/code/src/test/java/com/googlecode/cqengine/index/compound/support/ScrambledQueriesTest.java new file mode 100644 index 000000000..b030a5c96 --- /dev/null +++ b/code/src/test/java/com/googlecode/cqengine/index/compound/support/ScrambledQueriesTest.java @@ -0,0 +1,71 @@ +package com.googlecode.cqengine.index.compound.support; + +import com.google.common.collect.ImmutableList; +import com.googlecode.cqengine.ConcurrentIndexedCollection; +import com.googlecode.cqengine.IndexedCollection; +import com.googlecode.cqengine.examples.introduction.Car; +import com.googlecode.cqengine.index.compound.CompoundIndex; +import com.googlecode.cqengine.query.Query; +import com.googlecode.cqengine.query.logical.And; +import com.googlecode.cqengine.resultset.ResultSet; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import static com.googlecode.cqengine.query.QueryFactory.and; +import static com.googlecode.cqengine.query.QueryFactory.equal; + +/** + * @author Eduardo Barrios + */ +public class ScrambledQueriesTest { + + private final IndexedCollection cars = new ConcurrentIndexedCollection(); + + @Before + public void setUp() { + // add the index we want to test + cars.addIndex(CompoundIndex.onAttributes(Car.CAR_ID, Car.NAME, Car.DESCRIPTION, Car.FEATURES)); + + // Add some objects to the collection... + cars.add(new Car(1, "ford", "focus", ImmutableList.of("fwd", "hatchback", "black"))); + cars.add(new Car(2, "honda", "civic", ImmutableList.of("fwd", "sedan", "silver"))); + cars.add(new Car(3, "toyota", "prius", ImmutableList.of("hybrid", "fwd", "blue"))); + cars.add(new Car(4, "ford", "explorer", ImmutableList.of("awd", "green"))); + cars.add(new Car(5, "honda", "crv", ImmutableList.of("awd", "black"))); + cars.add(new Car(6, "toyota", "tundra", ImmutableList.of("4x4", "red"))); + } + + @Test + public void testUnscrambleQuery() { + + Query scrambledQuery = and( + equal(Car.NAME, "ford"), + equal(Car.FEATURES, "black"), + equal(Car.CAR_ID, 1), + equal(Car.DESCRIPTION, "focus") + ); + ResultSet flatQueryResult = cars.retrieve(scrambledQuery); + Assert.assertEquals(1, flatQueryResult.size()); + Assert.assertEquals(20, flatQueryResult.getRetrievalCost()); + } + + @Test + public void testFlattenQuery() { + And nestedQuery = and( + equal(Car.CAR_ID, 1), + and( + equal(Car.NAME, "ford"), + and( + equal(Car.DESCRIPTION, "focus"), + equal(Car.FEATURES, "black") + ) + ) + ); + + ResultSet nestedQueryResult = cars.retrieve(nestedQuery); + Assert.assertEquals(1, nestedQueryResult.size()); + Assert.assertEquals(20, nestedQueryResult.getRetrievalCost()); + + } +} \ No newline at end of file