[Bounty] Headless browser module #40

.gitignore

@@ -12,3 +12,12 @@ config/*.toml
 
 .idea
 .vscode/
+
+temp/
+.holy-lambda/
+?/
+.cpcache/
+.clj-kondo/
+.lsp/
+.DS_Store
+native-configuration/

headless/Dockerfile.amd

@@ -0,0 +1,5 @@
+FROM scratch
+
+COPY dist/bin/amd64/output-serial-gc headless
+
+ENTRYPOINT ["./headless"]

headless/Dockerfile.arm

@@ -0,0 +1,10 @@
+from dlbears/scratch-arm-dlls:latest as dlls
+from scratch
+USER root
+RUN apk add gcompat
+
+COPY --from=dlls /lib/ /lib
+COPY --from=dlls /lib64/ /lib64
+COPY dist/bin/arm64/output-noop-gc headless
+
+ENTRYPOINT ["./headless"]

headless/Dockerfile.script.amd

@@ -0,0 +1,35 @@
+FROM debian:stretch AS runner
+USER root
+ARG TARGETARCH
+ENV GRAAL_VERSION=22.3.0
+ENV JAVA_VERSION=11
+# Install chromedriver
+RUN apt-get update && apt install chromium-driver curl -y
+
+# Install script interpreter
+RUN curl -sLO https://raw.githubusercontent.com/babashka/babashka/master/install
+RUN chmod a+x install
+RUN ["/bin/bash", "./install", "--static"]
+
+# Install JVM (for loading jsoup pod and other java deps)
+RUN cd / && mkdir graal
+WORKDIR /graal
+
+RUN curl -sLO https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-${GRAAL_VERSION}/graalvm-ce-java${JAVA_VERSION}-linux-amd64-${GRAAL_VERSION}.tar.gz
+RUN tar -xzf graalvm-ce-java${JAVA_VERSION}-linux-amd64-${GRAAL_VERSION}.tar.gz
+
+RUN cd / && mkdir babashka
+WORKDIR /babashka
+
+COPY dist/jar/pod-jaydeesimon-jsoup-0.1-standalone.jar dist/jar/pod-jaydeesimon-jsoup-0.1-standalone.jar
+COPY bb.edn bb.edn
+COPY deps.edn deps.edn
+
+RUN apt-get install -y git
+ENV PATH="/graal/graalvm-ce-java${JAVA_VERSION}-${GRAAL_VERSION}/bin:${PATH}"
+ENV JAVA_HOME="/graal/graalvm-ce-java${JAVA_VERSION}-${GRAAL_VERSION}"
+RUN bb prepare
+
+COPY src/headless/find.cljc src/headless/find.cljc
+
+ENTRYPOINT ["bb", "./src/headless/find.cljc"]

headless/Dockerfile.script.arm

@@ -0,0 +1,34 @@
+FROM debian:stretch AS runner
+USER root
+ARG TARGETARCH
+ENV GRAAL_VERSION=22.3.0
+ENV JAVA_VERSION=11
+# Install chromedriver
+RUN apt-get update && apt install chromium-driver curl -y
+
+# Install script interpreter
+RUN curl -sLO https://raw.githubusercontent.com/babashka/babashka/master/install
+RUN chmod a+x install
+RUN ["/bin/bash", "./install", "--static"]
+
+# Install JVM (for loading jsoup pod and other java deps)
+RUN cd / && mkdir graal
+WORKDIR /graal
+RUN curl -sLO https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-${GRAAL_VERSION}/graalvm-ce-java${JAVA_VERSION}-linux-aarch64-${GRAAL_VERSION}.tar.gz
+RUN tar -xzf graalvm-ce-java${JAVA_VERSION}-linux-aarch64-${GRAAL_VERSION}.tar.gz
+
+RUN cd / && mkdir babashka
+WORKDIR /babashka
+
+COPY dist/jar/pod-jaydeesimon-jsoup-0.1-standalone.jar dist/jar/pod-jaydeesimon-jsoup-0.1-standalone.jar
+COPY bb.edn bb.edn
+COPY deps.edn deps.edn
+
+RUN apt-get install -y git
+ENV PATH="/graal/graalvm-ce-java${JAVA_VERSION}-${GRAAL_VERSION}/bin:${PATH}"
+ENV JAVA_HOME="/graal/graalvm-ce-java${JAVA_VERSION}-${GRAAL_VERSION}"
+RUN bb prepare
+
+COPY src/headless/find.cljc src/headless/find.cljc
+
+ENTRYPOINT ["bb", "./src/headless/find.cljc"]

headless/bb.edn

@@ -0,0 +1,76 @@
+{:paths ["src" "jsoup"]
+ :deps {etaoin/etaoin {:mvn/version "1.0.38"}
+        org.clojars.askonomm/ruuter {:mvn/version "1.3.2"} 
+        io.github.FieryCod/holy-lambda-babashka-tasks
+        {:git/url   "https://github.com/FieryCod/holy-lambda"
+         :deps/root "./modules/holy-lambda-babashka-tasks"
+         :sha       "1469bb96b85c2c65a52df9e3a4914dde1b4c816f"}
+        io.github.FieryCod/holy-lambda {:mvn/version "0.6.6"}}
+
+ :min-bb-version                                    "0.3.7"
+ :holy-lambda/options {;; User should use docker for local development and use `HL_NO_DOCKER` environment variable
+                       ;; set to "true" for CI based deployments.
+                       ;; For CI based deployments user should base it's builder image on `fierycod/graalvm-native-image:ce`
+                       :docker {;; Check https://docs.docker.com/network/
+                                ;; Network setting for future versions of HL will propagate to AWS SAM as well
+                                ;; Options: "host"|"bridge"|"overlay"|"none"|nil|"macvlan"
+                                :network            "host"
+
+                                ;; HL runs bb tasks in docker context which means that local libraries will not work out-of-the-box.
+                                ;;
+                                ;; To make local libraries work with HL you have to:
+                                ;; 1. Mount your local library folder as a docker volume.
+                                ;; 2. Modify the `deps.edn` to use library path available in docker context.
+                                ;; Preferably use `deps.edn` alias and reference it's name in `:build:clj-name`
+                                ;; ----------------------------------------------------------------------------
+                                ;; Single volume definition:
+                                ;;
+                                ;; {:docker "/path-where-to-mount-local-library-on-docker"
+                                ;;  :host   "relative-local-library-path"}
+                              ;;:volumes [{:docker     "/holy-lambda" 
+                              ;;           :host       "./"}]
+
+                                ;; GraalVM Community holy-lambda compatible docker image
+                                ;; You can always build your own GraalVM image with enterprise edition
+                                :image              "dlbears/graalvm-native-image:ee"}
+                       :build {:compile-cmd "clj -T:build uberjar"
+                               :graalvm-home nil}
+                       :runtime
+                       {;; :pods                {org.babashka/aws "0.0.5"}
+
+                        ;; For `:native` runtime you can provide your own bootstrap file
+                     ;;    :bootstrap-file "bootstrap"
+
+                        ;; For `:native` runtime you can provide some native resources which will be available during lambda execution
+                        ;; Resources are packed as is.
+                        :native-deps "resources"
+
+                        ;; Specify custom arguments for native image generation
+                        ;; Works only on `:native` runtime.
+                        ;; Check https://www.graalvm.org/reference-manual/native-image/Options/
+                        :native-image-args
+                        ["--verbose"
+                         ;"-H:+StaticExecutableWithDynamicLibC" ; aarch64 and x86
+                         ;"--static" ; x86 only
+                         ;"--libc=musl" ; x86 only
+                         "--gc=epsilon" ; [ g1 (x86 only), serial, epsilon ]
+                         "--no-fallback"
+                         "--enable-url-protocols=http,https"
+                         "--report-unsupported-elements-at-runtime"
+                         "-H:+AllowIncompleteClasspath"
+                         "--no-server"
+                         "--initialize-at-build-time"
+                         "--initialize-at-run-time=org.httpkit.client.ClientSslEngineFactory\\$SSLHolder,org.apache.http.impl.auth.NTLMEngineImpl"
+                         "--trace-object-instantiation=java.security.SecureRandom"]}}
+ :tasks {:requires                                ([holy-lambda.tasks])
+
+         hl:docker:run             holy-lambda.tasks/hl:docker:run
+
+         hl:native:conf            holy-lambda.tasks/hl:native:conf
+         hl:native:executable      holy-lambda.tasks/hl:native:executable
+         hl:babashka:sync          holy-lambda.tasks/hl:babashka:sync
+         hl:sync                   holy-lambda.tasks/hl:sync
+         hl:compile                holy-lambda.tasks/hl:compile
+         hl:doctor                 holy-lambda.tasks/hl:doctor
+         hl:clean                  holy-lambda.tasks/hl:clean
+         hl:version                holy-lambda.tasks/hl:version}}

headless/build.clj

@@ -0,0 +1,24 @@
+(ns build
+  (:require
+   [clojure.tools.build.api :as b]))
+
+(def class-dir ".holy-lambda/target/classes")
+(def basis (b/create-basis {:project "deps.edn"}))
+
+(defn clean [_]
+  (b/delete {:path ".holy-lambda/target"})
+  (b/delete {:path ".holy-lambda/build"}))
+
+
+;; Compile clojure code into standalone jar 
+(defn uberjar [_]
+  (b/copy-dir {:src-dirs ["src" "resources"]
+               :target-dir class-dir})
+  (b/compile-clj {:basis basis
+                  :src-dirs ["src"]
+                  :ns-compile ['headless.find] ;; Only package this namespace
+                  :class-dir class-dir})
+  (b/uber {:class-dir class-dir
+           :main 'headless.find
+           :basis basis
+           :uber-file ".holy-lambda/build/output.jar"}))

headless/deps.edn

@@ -0,0 +1,19 @@
+
+{:deps {etaoin/etaoin {:mvn/version "1.0.38"}
+        nrepl/bencode {:mvn/version "1.1.0"}
+        org.jsoup/jsoup {:mvn/version "1.15.3"}
+        org.clojars.askonomm/ruuter {:mvn/version "1.3.2"}
+        http-kit/http-kit {:mvn/version "2.6.0"}
+        org.clojure/core.match {:mvn/version "1.0.0"} 
+        org.babashka/cli {:mvn/version "0.5.40"}
+        babashka/babashka.pods {:mvn/version "0.1.0"}
+        org.clojure/clojure                  {:mvn/version "1.10.3"}
+        io.github.FieryCod/holy-lambda       {:mvn/version "0.6.6"}
+        com.github.clj-easy/graal-build-time {:mvn/version "0.1.4"}}
+
+ :paths  ["src" "resources" "jsoup"]
+ :aliases
+ {:build {:deps {io.github.clojure/tools.build {:git/tag "v0.8.3" :git/sha "0d20256"}}
+          :ns-default build
+          :exec-fn build/uber
+          :jvm-opts ["-Dclojure.compiler.direct-linking=true" "-Dclojure.spec.skip-macros=true"]}}}