Ex2

src/webdata/Analysis.java

@@ -0,0 +1,61 @@
+package webdata;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.Random;
+
+public class Analysis {
+    private IndexReader indexReader;
+    private TokensIndex tokensIndex;
+    private ArrayList<String> randomTokens;
+    private long getReviewsWithTokenTime;
+    private long getTokenFrequencytTime;
+
+    public Analysis(IndexReader indexReader){
+        this.indexReader = indexReader;
+        this.tokensIndex = indexReader.tokenIndex;
+        this.randomTokens = new ArrayList<>();
+        this.getReviewsWithTokenTime = 0;
+        this.getTokenFrequencytTime = 0;
+
+        getRandomTokens(100);
+        measureGetReviewsWithToken();
+        measureTokenFrequencyTime();
+    }
+
+    private void measureGetReviewsWithToken() {
+        long start = new Date().getTime();
+        for (String token: this.randomTokens){
+            indexReader.getReviewsWithToken(token);
+        }
+        long end = new Date().getTime();
+        this.getReviewsWithTokenTime = (end - start);
+    }
+
+    private void measureTokenFrequencyTime() {
+        long start = new Date().getTime();
+        for (String token: this.randomTokens){
+            indexReader.getTokenFrequency(token);
+        }
+        long end = new Date().getTime();
+        this.getTokenFrequencytTime = (end - start);
+    }
+
+    /**
+     * Get n random tokens from the index.
+     */
+    public void getRandomTokens(int n){
+        Random random = new Random();
+        for (int i=0; i < n; i++){
+            int randIndex = random.nextInt(this.tokensIndex.data.size()); // get random index
+            this.randomTokens.add(tokensIndex.getWordAt(randIndex));
+        }
+    }
+
+    public static void main(String[] args) {
+        IndexReader indexReader = new IndexReader("./Data_index");
+        Analysis analysis = new Analysis(indexReader);
+        System.out.println("getReviewsWithToken runtime: " + analysis.getReviewsWithTokenTime + "(ms)");
+        System.out.println("getTokenFrequency runtime: " + analysis.getTokenFrequencytTime + "(ms)");
+    }
+}

src/webdata/DataLoader.java

@@ -0,0 +1,63 @@
+package webdata;
+
+import java.io.BufferedReader;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+public class DataLoader implements Iterable<ArrayList<String>> {
+    private BufferedReader br;
+    private ArrayList<String> reviewStrings;
+
+    public DataLoader(String inputFile) throws FileNotFoundException {
+        br = new BufferedReader(new FileReader(inputFile));
+        reviewStrings = new ArrayList<>();
+    }
+
+    public ArrayList<String> readSingleReview() {
+        String line;
+        try {
+            while((line = br.readLine()) != null) {
+                if (line.contains("product/productId") && reviewStrings.size() != 0) {
+                    ArrayList<String> ret = reviewStrings;
+                    reviewStrings = new ArrayList<String>();
+                    reviewStrings.add(line);
+                    return ret;
+                }
+                reviewStrings.add(line);
+            }
+        } catch (IOException e) {
+            e.printStackTrace();
+            System.exit(1);
+        }
+        return reviewStrings;
+    }
+
+    public Iterator<ArrayList<String>> iterator() {
+        return new Iterator<>() {
+            @Override
+            public boolean hasNext(){
+                try {
+                    br.mark(1);
+                    int i = br.read();
+                    br.reset();
+                    return (i != -1);
+                } catch (IOException e) {
+                    return false;
+                }
+            }
+
+            @Override
+            public ArrayList<String> next() {
+                return readSingleReview();
+            }
+
+            @Override
+            public void remove() {
+                throw new UnsupportedOperationException();
+            }
+        };
+    }
+}

src/webdata/DataParser.java

@@ -5,52 +5,110 @@
 
 
 public class DataParser {
-    ArrayList<HashMap<String, String>> allReviews = new ArrayList<>();
-    public static final List<String> INTEREST_FIELDS = Arrays.asList("productId", "score", "helpfulness", "text");
+    public class Review{
+        private String text;
+        private String productId;
+        private String score;
+        private String helpfulness;
 
+        public String getText() {
+            return text;
+        }
+
+        public String getProductId() {
+            return productId;
+        }
+
+        public String getHelpfulness() {
+            return helpfulness;
+        }
+
+        public String getScore() {
+            return score;
+        }
+
+        public void setHelpfulness(String helpfulness) {
+            this.helpfulness = helpfulness;
+        }
+
+        public void setProductId(String productId) {
+            this.productId = productId;
+        }
+
+        public void setScore(String score) {
+            this.score = score;
+        }
+
+        public void setText(String text) {
+            this.text = text;
+        }
+    }
 
     /**
      * Given product review data, parses the data and creates a new list where each entry i contains hashmap with the fields
      * of the review, i.e: productId->value, score->value, helpfulness->value, text->value.
      * inputFile is the path to the file containing the review data
      */
-    public DataParser(String inputFile) throws IOException {
-        BufferedReader br = new BufferedReader(new FileReader(inputFile));
-        String line;
-        StringBuilder review = new StringBuilder();
-        while((line = br.readLine()) != null) {
-            if (line.contains("product/productId")){
-                if (!review.toString().equals("")){
-                    allReviews.add(parse_review(review.toString()));
-                }
-                review = new StringBuilder(line);
-            }
-            else{
-                review.append(line);
-            }
+    public List<Review> parseData(List<String> rawReviews){
+        ArrayList<Review> allReviews = new ArrayList<>();
+        for (String review: rawReviews){
+            allReviews.add(parseReview(review));
         }
-        allReviews.add(parse_review(review.toString()));  // add the last review
+        return allReviews;
     }
 
     /**
-     * Given a single review, parse the review and return a hash table containing only the relevant fields of the
-     * review, i.e: productId, score, helpfulness, text.
-     * @param review: the review that should be parsed.
-     * @return a hash table where the keys are the relevant fields mentioned above and their corresponding values.
+     * Given a single review, parse the review and return a Review object, containing all relevant information from the
+     * given review, i.e. productId, score, helpfulness and text.
      */
-    private static HashMap<String, String> parse_review(String review){
-        List<String> fields = Arrays.asList(review.split("review/"));
-        HashMap<String, String> review_fields = new HashMap<String, String>();
+    public Review parseReview(String review){
+        ArrayList<String> fields = new ArrayList<>(Arrays.asList(review.split("review/")));
+        Review parsedReview = new Review();
 
-        review_fields.put("productId", fields.get(0).split(": ")[1].split("product/")[0]);
+        parsedReview.setProductId(fields.get(0).split(": ")[1].split("product/")[0]);
         for (int i=1; i<fields.size(); i++){
             String field = fields.get(i);
-            List<String> field_value = Arrays.asList(field.split(": "));
-            if (INTEREST_FIELDS.contains(field_value.get(0))) {
-                review_fields.put(field_value.get(0), String.join(":", field_value.subList(1, field_value.size())));
+            List<String> fieldValue = Arrays.asList(field.split(": "));
+            if (fieldValue.get(0).equals("text")) {
+                parsedReview.setText(String.join(": ", fieldValue.subList(1, fieldValue.size())));
+            } else if (fieldValue.get(0).equals("helpfulness")) {
+                parsedReview.setHelpfulness(fieldValue.get(1));
+            } else if (fieldValue.get(0).equals("score")) {
+                parsedReview.setScore(fieldValue.get(1));
+            }
+        }
+        return parsedReview;
+    }
+
+    public Review parseReview(ArrayList<String> review){
+        Review parsedReview = new Review();
+        StringBuilder text = new StringBuilder();
+        boolean readingText = false;
+        for (String line : review){
+            if (readingText && !line.equals("")) {
+                text.append(" ");
+                text.append(line);
+                continue;
+            }
+            int prefix = line.indexOf("/");
+            int delim = line.indexOf(":");
+            if (prefix == -1 || delim == -1 || delim < prefix) {
+                continue;
+            }
+            String field = line.substring(prefix + 1, delim);
+            if (field.equals("text")){
+                text.append(line.substring(delim + 2));
+                readingText = true;
+            } else if (field.equals("productId")) {
+                parsedReview.setProductId(line.substring(delim + 2));
+            } else if (field.equals("helpfulness")) {
+                parsedReview.setHelpfulness(line.substring(delim + 2));
+            } else if (field.equals("score")) {
+                parsedReview.setScore(line.substring(delim + 2));
             }
         }
-        return review_fields;
+        parsedReview.setText(text.toString());
+        return parsedReview;
     }
 }