From f1d25e2cf634c7d61749fd73a05b54ac56b85e6c Mon Sep 17 00:00:00 2001 From: kelvinqian00 Date: Tue, 31 Jan 2023 11:01:26 -0500 Subject: [PATCH 1/5] Add VSCode directories to gitignore --- .gitignore | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitignore b/.gitignore index ac1ec62d..3cc3a598 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,10 @@ failures/ /logs/ .DS_Store /dev-resources/bench/*.json + +# VSCode +.calva/ +.clj-kondo/ +.cljs_node_repl/ +.lsp/ +.vscode/ From 9a083e7edb2b7a2e45d63a203bdd572fd8a51296 Mon Sep 17 00:00:00 2001 From: kelvinqian00 Date: Tue, 31 Jan 2023 11:03:02 -0500 Subject: [PATCH 2/5] Add lint workflow to CI --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d4ae689..615cced2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,11 @@ jobs: - name: Test Xapipe run: make test + lint: + uses: yetanalytics/workflow-linter/.github/workflows/linter.yml@v2023.01.20 + with: + lint-directories: 'src/bench src/build src/cli src/lib src/test' + nvd_scan: uses: yetanalytics/actions/.github/workflows/nvd-scan.yml@v0.0.4 with: From f8e4256bd3ba3e7e0c2925682a7c3acc2a4d9ff3 Mon Sep 17 00:00:00 2001 From: kelvinqian00 Date: Tue, 31 Jan 2023 11:18:32 -0500 Subject: [PATCH 3/5] Remove warnings in bench namespaces --- src/bench/com/yetanalytics/xapipe/bench.clj | 20 +++++++++---------- .../com/yetanalytics/xapipe/bench/maths.clj | 16 ++++++++++++++- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/bench/com/yetanalytics/xapipe/bench.clj b/src/bench/com/yetanalytics/xapipe/bench.clj index f7ed4333..2de13271 100644 --- a/src/bench/com/yetanalytics/xapipe/bench.clj +++ b/src/bench/com/yetanalytics/xapipe/bench.clj @@ -8,8 +8,7 @@ [clojure.java.io :as io] [com.yetanalytics.xapipe.util.time :as tu] [java-time :as t] - [java-time.seqs :as tseq]) - (:import [java.time Instant])) + [java-time.seqs :as tseq])) (defn- stamp-seq "Create a lazy sequence of timestamps" @@ -238,15 +237,14 @@ (flush) r))) _ (do (print "\n\n") (flush)) - ] - (let [first-run (first runs)] - (merge (dissoc first-run - :s-per-sec - :total-ms) - {:label label - :runs num-runs - :total-ms (maths/mean (map :total-ms runs)) - :s-per-sec (maths/mean (map :s-per-sec runs))})))) + first-run (first runs)] + (merge (dissoc first-run + :s-per-sec + :total-ms) + {:label label + :runs num-runs + :total-ms (maths/mean (map :total-ms runs)) + :s-per-sec (maths/mean (map :s-per-sec runs))}))) [;; Default tuning args (including calculated) {:label "defaults" :num-runs 10 diff --git a/src/bench/com/yetanalytics/xapipe/bench/maths.clj b/src/bench/com/yetanalytics/xapipe/bench/maths.clj index a04c565b..75ab9ef3 100644 --- a/src/bench/com/yetanalytics/xapipe/bench/maths.clj +++ b/src/bench/com/yetanalytics/xapipe/bench/maths.clj @@ -1,11 +1,14 @@ (ns com.yetanalytics.xapipe.bench.maths (:require [clojure.spec.alpha :as s] - [clojure.spec.gen.alpha :as sgen] [clojure.set :as cset])) ;; Origninally Cribbed from Avery and Jean ;; https://github.com/clojure-cookbook/clojure-cookbook/blob/master/01_primitive-data/1-20_simple-statistics.asciidoc +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Mean +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + (defn mean ^Double [coll] (let [[sum count] (reduce (fn [[s c] x] [(+ s x) (inc c)]) @@ -19,6 +22,10 @@ :args (s/cat :coll (s/coll-of number?)) :ret double?) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Median +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + (defn median ^Double [coll] (let [sorted (sort coll) cnt (count sorted)] @@ -35,6 +42,10 @@ :args (s/cat :coll (s/coll-of number?)) :ret (s/nilable double?)) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Mode +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + (defn mode [coll] (if-let [coll (not-empty coll)] (let [freqs (frequencies coll) @@ -64,6 +75,9 @@ (empty? ret) :else (println coll ret)))) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Standard Deviation +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (defn stddev ^Double [coll & {:keys [complete-population?] :or {complete-population? false}}] From 24261eb9bcc1a3a579ce043e979789c9a4ec02aa Mon Sep 17 00:00:00 2001 From: kelvinqian00 Date: Tue, 31 Jan 2023 11:19:41 -0500 Subject: [PATCH 4/5] Fix basic warnings in cli namespaces --- src/cli/com/yetanalytics/xapipe/cli.clj | 7 +++---- src/cli/com/yetanalytics/xapipe/cli/options.clj | 3 +-- src/cli/com/yetanalytics/xapipe/main.clj | 3 ++- .../yetanalytics/xapipe/metrics/impl/prometheus.clj | 10 +++++----- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/cli/com/yetanalytics/xapipe/cli.clj b/src/cli/com/yetanalytics/xapipe/cli.clj index 6bbeff68..7675d32d 100644 --- a/src/cli/com/yetanalytics/xapipe/cli.clj +++ b/src/cli/com/yetanalytics/xapipe/cli.clj @@ -19,8 +19,7 @@ [com.yetanalytics.xapipe.store.impl.file :as file-store] [com.yetanalytics.xapipe.store.impl.memory :as mem-store] [com.yetanalytics.xapipe.store.impl.noop :as noop-store] - [com.yetanalytics.xapipe.store.impl.redis :as redis-store] - [xapi-schema.spec.resources :as xsr]) + [com.yetanalytics.xapipe.store.impl.redis :as redis-store]) (:import [java.net URL])) ;; xAPI partial GET params @@ -81,8 +80,8 @@ (.getProtocol parsed) (.getAuthority parsed)) :xapi-prefix (.getPath parsed)}) - (catch Exception ex - (throw (ex-info (format "Could not parse LRS URL %s" url)))))) + (catch Exception _ + (throw (ex-info (format "Could not parse LRS URL %s" url) {}))))) (s/fdef force-stop-job! :args (s/cat :stop-fn fn? diff --git a/src/cli/com/yetanalytics/xapipe/cli/options.clj b/src/cli/com/yetanalytics/xapipe/cli/options.clj index c8e44efe..94478fbb 100644 --- a/src/cli/com/yetanalytics/xapipe/cli/options.clj +++ b/src/cli/com/yetanalytics/xapipe/cli/options.clj @@ -2,7 +2,6 @@ "clojure.tools.cli options for xapipe CLI" (:require [cheshire.core :as json] [clojure.spec.alpha :as s] - [clojure.java.io :as io] [clojure.string :as cs] [clojure.tools.cli :as cli] [com.yetanalytics.xapipe :as xapipe] @@ -10,7 +9,7 @@ [com.yetanalytics.xapipe.filter.path :as fpath])) (defn option-spec->spec-def - [[short-command long-command desc + [[short-command long-command _description & {:keys [id default parse-fn diff --git a/src/cli/com/yetanalytics/xapipe/main.clj b/src/cli/com/yetanalytics/xapipe/main.clj index 9069838b..edae1764 100644 --- a/src/cli/com/yetanalytics/xapipe/main.clj +++ b/src/cli/com/yetanalytics/xapipe/main.clj @@ -71,7 +71,8 @@ Delete a Job: (not= ?job-id-arg (:id ?from-json))) (throw (ex-info (format "--job-id %s does not match JSON job id %s" - ?job-id-arg (:id ?from-json)))))) + ?job-id-arg (:id ?from-json)) + {})))) ?job-id (or ?job-id-arg (:id ?from-json)) ?from-storage (and ?job-id (store/read-job diff --git a/src/cli/com/yetanalytics/xapipe/metrics/impl/prometheus.clj b/src/cli/com/yetanalytics/xapipe/metrics/impl/prometheus.clj index f8daac8f..0ba0412f 100644 --- a/src/cli/com/yetanalytics/xapipe/metrics/impl/prometheus.clj +++ b/src/cli/com/yetanalytics/xapipe/metrics/impl/prometheus.clj @@ -44,13 +44,13 @@ collectors)] (reify metrics/Reporter - (-gauge [this k v] + (-gauge [_ k v] (pro/set registry k v)) - (-counter [this k delta] + (-counter [_ k delta] (pro/inc registry k delta)) - (-histogram [this k v] + (-histogram [_ k v] (pro/observe registry k v)) - (-summary [this k v] + (-summary [_ k v] (pro/observe registry k v)) - (-flush! [this] + (-flush! [_] (pro-exp/push! registry))))) From 4494565c8fbd80f15bd8bb762fae3f3201b4717a Mon Sep 17 00:00:00 2001 From: kelvinqian00 Date: Tue, 31 Jan 2023 11:27:54 -0500 Subject: [PATCH 5/5] Remove unused requires in main lib namespaces --- src/lib/com/yetanalytics/xapipe.clj | 1 - src/lib/com/yetanalytics/xapipe/client.clj | 1 - src/lib/com/yetanalytics/xapipe/client/multipart_mixed.clj | 3 +-- src/lib/com/yetanalytics/xapipe/filter.clj | 5 ----- src/lib/com/yetanalytics/xapipe/filter/concept.clj | 4 +--- src/lib/com/yetanalytics/xapipe/job.clj | 4 +--- src/lib/com/yetanalytics/xapipe/job/json.clj | 3 +-- src/lib/com/yetanalytics/xapipe/job/state.clj | 4 +--- src/lib/com/yetanalytics/xapipe/job/state/errors.clj | 3 +-- src/lib/com/yetanalytics/xapipe/xapi.clj | 1 - 10 files changed, 6 insertions(+), 23 deletions(-) diff --git a/src/lib/com/yetanalytics/xapipe.clj b/src/lib/com/yetanalytics/xapipe.clj index 4054ad77..846e5b56 100644 --- a/src/lib/com/yetanalytics/xapipe.clj +++ b/src/lib/com/yetanalytics/xapipe.clj @@ -1,6 +1,5 @@ (ns com.yetanalytics.xapipe (:require [clojure.core.async :as a] - [clojure.java.io :as io] [clojure.spec.alpha :as s] [clojure.tools.logging :as log] [com.yetanalytics.xapipe.client :as client] diff --git a/src/lib/com/yetanalytics/xapipe/client.clj b/src/lib/com/yetanalytics/xapipe/client.clj index 19c6eb37..7734388a 100644 --- a/src/lib/com/yetanalytics/xapipe/client.clj +++ b/src/lib/com/yetanalytics/xapipe/client.clj @@ -12,7 +12,6 @@ [com.yetanalytics.xapipe.metrics :as metrics] [com.yetanalytics.xapipe.util :as u] [xapi-schema.spec :as xs] - [xapi-schema.spec.resources :as xsr] [com.yetanalytics.xapipe.util.time :as t] [com.yetanalytics.xapipe.spec.common :as cspec]) (:import [org.apache.http.impl.client CloseableHttpClient] diff --git a/src/lib/com/yetanalytics/xapipe/client/multipart_mixed.clj b/src/lib/com/yetanalytics/xapipe/client/multipart_mixed.clj index 02169272..f92025c9 100644 --- a/src/lib/com/yetanalytics/xapipe/client/multipart_mixed.clj +++ b/src/lib/com/yetanalytics/xapipe/client/multipart_mixed.clj @@ -6,8 +6,7 @@ [clojure.spec.alpha :as s] [clojure.spec.gen.alpha :as sgen] [clojure.string :as cs] - [xapi-schema.spec :as xs] - [xapi-schema.spec.resources :as xsr]) + [xapi-schema.spec :as xs]) (:import [java.io Writer diff --git a/src/lib/com/yetanalytics/xapipe/filter.clj b/src/lib/com/yetanalytics/xapipe/filter.clj index edf1590b..a16c4773 100644 --- a/src/lib/com/yetanalytics/xapipe/filter.clj +++ b/src/lib/com/yetanalytics/xapipe/filter.clj @@ -1,14 +1,9 @@ (ns com.yetanalytics.xapipe.filter "Apply profile-based filtering to statement streams." (:require [clojure.spec.alpha :as s] - [clojure.spec.gen.alpha :as sgen] [xapi-schema.spec :as xs] [cheshire.core :as json] [com.yetanalytics.persephone :as per] - [com.yetanalytics.persephone.pattern.fsm :as fsm] - [com.yetanalytics.persephone.pattern.fsm-spec :as fsm-spec] - [com.yetanalytics.persephone.template :as per-template] - [com.yetanalytics.persephone.utils.json :as per-json] [com.yetanalytics.pan.objects.pattern :as pat] [com.yetanalytics.pan.objects.profile :as prof] [com.yetanalytics.pan.objects.template :as template] diff --git a/src/lib/com/yetanalytics/xapipe/filter/concept.clj b/src/lib/com/yetanalytics/xapipe/filter/concept.clj index 7af766a0..52942706 100644 --- a/src/lib/com/yetanalytics/xapipe/filter/concept.clj +++ b/src/lib/com/yetanalytics/xapipe/filter/concept.clj @@ -1,8 +1,6 @@ (ns com.yetanalytics.xapipe.filter.concept "Functions to apply concept-based filtering to statement streams." - (:require [com.yetanalytics.persephone.template :as pt] - [com.yetanalytics.persephone :as p] - [com.yetanalytics.pan.objects.concepts.verb :as v] + (:require [com.yetanalytics.pan.objects.concepts.verb :as v] [com.yetanalytics.pan.objects.concepts.activity-type :as at] [com.yetanalytics.pan.objects.template :as t] [com.yetanalytics.pan.objects.concepts.attachment-usage-type :as aut] diff --git a/src/lib/com/yetanalytics/xapipe/job.clj b/src/lib/com/yetanalytics/xapipe/job.clj index 392da2c5..90d12d8b 100644 --- a/src/lib/com/yetanalytics/xapipe/job.clj +++ b/src/lib/com/yetanalytics/xapipe/job.clj @@ -4,9 +4,7 @@ [clojure.spec.gen.alpha :as sgen] [com.yetanalytics.xapipe.job.config :as config] [com.yetanalytics.xapipe.job.state :as state] - [com.yetanalytics.xapipe.job.state.errors :as errors] - [com.yetanalytics.xapipe.util.time :as t] - [xapi-schema.spec :as xs])) + [com.yetanalytics.xapipe.util.time :as t])) (def current-version 0) diff --git a/src/lib/com/yetanalytics/xapipe/job/json.clj b/src/lib/com/yetanalytics/xapipe/job/json.clj index b7490254..c8f1022e 100644 --- a/src/lib/com/yetanalytics/xapipe/job/json.clj +++ b/src/lib/com/yetanalytics/xapipe/job/json.clj @@ -6,8 +6,7 @@ [clojure.spec.gen.alpha :as sgen] [cognitect.transit :as transit] [com.yetanalytics.xapipe.job :as job] - [com.yetanalytics.xapipe.job.config :as config] - [xapi-schema.spec :as xs]) + [com.yetanalytics.xapipe.job.config :as config]) (:import [java.io ByteArrayInputStream ByteArrayOutputStream File])) (s/fdef write-transit-str diff --git a/src/lib/com/yetanalytics/xapipe/job/state.clj b/src/lib/com/yetanalytics/xapipe/job/state.clj index 3039debd..767d705e 100644 --- a/src/lib/com/yetanalytics/xapipe/job/state.clj +++ b/src/lib/com/yetanalytics/xapipe/job/state.clj @@ -1,10 +1,8 @@ (ns com.yetanalytics.xapipe.job.state (:require [clojure.spec.alpha :as s] - [clojure.spec.gen.alpha :as sgen] [com.yetanalytics.xapipe.filter :as filt] [com.yetanalytics.xapipe.job.state.errors :as errors] - [com.yetanalytics.xapipe.util.time :as t] - [xapi-schema.spec :as xs])) + [com.yetanalytics.xapipe.util.time :as t])) ;; Error vectors are stored at 3 levels for the source, ;; target, and entire job. diff --git a/src/lib/com/yetanalytics/xapipe/job/state/errors.clj b/src/lib/com/yetanalytics/xapipe/job/state/errors.clj index e203db40..155cd416 100644 --- a/src/lib/com/yetanalytics/xapipe/job/state/errors.clj +++ b/src/lib/com/yetanalytics/xapipe/job/state/errors.clj @@ -1,6 +1,5 @@ (ns com.yetanalytics.xapipe.job.state.errors - (:require [clojure.spec.alpha :as s] - [xapi-schema.spec :as xs])) + (:require [clojure.spec.alpha :as s])) (s/def ::message string?) (s/def ::type #{:job :source :target}) diff --git a/src/lib/com/yetanalytics/xapipe/xapi.clj b/src/lib/com/yetanalytics/xapipe/xapi.clj index 6c1e61a8..3bc4258d 100644 --- a/src/lib/com/yetanalytics/xapipe/xapi.clj +++ b/src/lib/com/yetanalytics/xapipe/xapi.clj @@ -2,7 +2,6 @@ (:require [clojure.spec.alpha :as s] [clojure.spec.gen.alpha :as sgen] [clojure.tools.logging :as log] - [com.yetanalytics.xapipe.client :as client] [com.yetanalytics.xapipe.client.multipart-mixed :as multipart] [xapi-schema.spec :as xs]))