Cleaned code (shingle/profile based algorithms)

Author: tdebatty

src/main/java/info/debatty/java/stringsimilarity/Cosine.java

@@ -25,6 +25,7 @@
 
 import info.debatty.java.stringsimilarity.interfaces.NormalizedStringSimilarity;
 import info.debatty.java.stringsimilarity.interfaces.NormalizedStringDistance;
+import java.util.Map;
 import net.jcip.annotations.Immutable;
 
 /**
@@ -72,39 +73,52 @@ public final double similarity(final String s1, final String s2) {
         if (s1.length() < getK() || s2.length() < getK()) {
             return 0;
         }
-        KShingling ks = new KShingling(getK());
-        int[] profile1 = ks.getArrayProfile(s1);
-        int[] profile2 = ks.getArrayProfile(s2);
+
+        Map<String, Integer> profile1 = getProfile(s1);
+        Map<String, Integer> profile2 = getProfile(s2);
 
         return dotProduct(profile1, profile2)
                 / (norm(profile1) * norm(profile2));
     }
 
+
+
     /**
      * Compute the norm L2 : sqrt(Sum_i( v_i²)).
      *
      * @param profile
      * @return L2 norm
      */
-    private static double norm(final int[] profile) {
+    private static double norm(final Map<String, Integer> profile) {
         double agg = 0;
 
-        for (int v : profile) {
-            agg += 1.0 * v * v;
+        for (Map.Entry<String, Integer> entry : profile.entrySet()) {
+            agg += 1.0 * entry.getValue() * entry.getValue();
         }
 
         return Math.sqrt(agg);
     }
 
     private static double dotProduct(
-            final int[] profile1, final int[] profile2) {
+            final Map<String, Integer> profile1,
+            final Map<String, Integer> profile2) {
+
+        // Loop over the smallest map
+        Map<String, Integer> small_profile = profile2;
+        Map<String, Integer> large_profile = profile1;
+        if (profile1.size() < profile2.size()) {
+            small_profile = profile1;
+            large_profile = profile2;
+        }
 
-        // profiles may not have the same length
-        int length = Math.min(profile1.length, profile2.length);
         double agg = 0;
-        for (int i = 0; i < length; i++) {
-            agg += 1.0 * profile1[i] * profile2[i];
+        for (Map.Entry<String, Integer> entry : small_profile.entrySet()) {
+            if (!large_profile.containsKey(entry.getKey())) {
+                continue;
+            }
+            agg += 1.0 * entry.getValue() * large_profile.get(entry.getKey());
         }
+
         return agg;
     }
 
@@ -118,4 +132,12 @@ public final double distance(final String s1, final String s2) {
         return 1.0 - similarity(s1, s2);
     }
 
+    public double similarity(
+            final Map<String, Integer> profile1,
+            final Map<String, Integer> profile2) {
+
+        return dotProduct(profile1, profile2)
+                / (norm(profile1) * norm(profile2));
+    }
+
 }

src/main/java/info/debatty/java/stringsimilarity/Jaccard.java

@@ -27,6 +27,9 @@
 import info.debatty.java.stringsimilarity.interfaces.MetricStringDistance;
 import info.debatty.java.stringsimilarity.interfaces.NormalizedStringSimilarity;
 import info.debatty.java.stringsimilarity.interfaces.NormalizedStringDistance;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
 import net.jcip.annotations.Immutable;
 
 /**
@@ -71,28 +74,22 @@ public Jaccard() {
      * @return
      */
     public final double similarity(final String s1, final String s2) {
-        KShingling ks = new KShingling(getK());
-        int[] profile1 = ks.getArrayProfile(s1);
-        int[] profile2 = ks.getArrayProfile(s2);
+        Map<String, Integer> profile1 = getProfile(s1);
+        Map<String, Integer> profile2 = getProfile(s2);
 
-        int length = Math.max(profile1.length, profile2.length);
-        profile1 = java.util.Arrays.copyOf(profile1, length);
-        profile2 = java.util.Arrays.copyOf(profile2, length);
+        Set<String> union = new HashSet<String>();
+        union.addAll(profile1.keySet());
+        union.addAll(profile2.keySet());
 
         int inter = 0;
-        int union = 0;
 
-        for (int i = 0; i < length; i++) {
-            if (profile1[i] > 0 || profile2[i] > 0) {
-                union++;
-
-                if (profile1[i] > 0 && profile2[i] > 0) {
-                    inter++;
-                }
+        for (String key : union) {
+            if (profile1.containsKey(key) && profile2.containsKey(key)) {
+                inter++;
             }
         }
 
-        return 1.0 * inter / union;
+        return 1.0 * inter / union.size();
     }
 
 

src/main/java/info/debatty/java/stringsimilarity/KShingling.java

@@ -1,6 +1,9 @@
 package info.debatty.java.stringsimilarity;
 
 import info.debatty.java.stringsimilarity.interfaces.StringDistance;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
 import net.jcip.annotations.Immutable;
 
 /**
@@ -21,7 +24,7 @@ public class QGram extends ShingleBased implements StringDistance {
      * string-matching with q-grams and maximal matches",
      * http://www.sciencedirect.com/science/article/pii/0304397592901434 The
      * distance between two strings is defined as the L1 norm of the difference
-     * of their profiles (the number of occurence of each k-shingle). Q-gram
+     * of their profiles (the number of occurences of each k-shingle). Q-gram
      * distance is a lower bound on Levenshtein distance, but can be computed in
      * O(|A| + |B|), where Levenshtein requires O(|A|.|B|)
      *
@@ -53,19 +56,27 @@ public QGram() {
      * @return
      */
     public final double distance(final String s1, final String s2) {
-        KShingling ks = new KShingling(getK());
-        int[] profile1 = ks.getArrayProfile(s1);
-        int[] profile2 = ks.getArrayProfile(s2);
-        int length = Math.max(profile1.length, profile2.length);
+        Map<String, Integer> profile1 = getProfile(s1);
+        Map<String, Integer> profile2 = getProfile(s2);
 
-        profile1 = java.util.Arrays.copyOf(profile1, length);
-        profile2 = java.util.Arrays.copyOf(profile2, length);
+        Set<String> union = new HashSet<String>();
+        union.addAll(profile1.keySet());
+        union.addAll(profile2.keySet());
 
-        int d = 0;
-        for (int i = 0; i < length; i++) {
-            d += Math.abs(profile1[i] - profile2[i]);
-        }
+        int agg = 0;
+        for (String key : union) {
+            int v1 = 0;
+            int v2 = 0;
+            if (profile1.containsKey(key)) {
+                v1 = profile1.get(key);
+            }
 
-        return d;
+
+            if (profile2.containsKey(key)) {
+                v2 = profile2.get(key);
+            }
+            agg += Math.abs(v1 - v2);
+        }
+        return agg;
     }
 }