CLJS-1960: Require CommonJS modules directly from a ClojureScript namespace

This patch addresses the first part of the solution outlined in the following
design doc:
https://github.com/clojure/clojurescript/wiki/Enhanced-Node.js-Modules-Support

It makes possible to specify, install and require Node.js dependencies directly
from ClojureScript namespaces. Future work can make it possible to support
specifying these dependencies in `deps.cljs` files and handling conflict
resolution between upstream foreign dependencies and foreign dependencies
specified directly in the compiler options.

Author: anmonteiro

src/main/cljs/cljs/module_deps.js

@@ -1,26 +1,71 @@
 var path = require('path');
 var mdeps = require('module-deps');
+var nodeResolve = require('resolve');
+var browserResolve = require('browser-resolve');
 
-var md = mdeps({});
-var deps_files = [];
+var target = 'CLJS_TARGET';
+var filename = path.resolve(__dirname, 'JS_FILE');
+var resolver = target === 'nodejs' ? nodeResolve : browserResolve;
+
+var md = mdeps({
+  resolve: function(id, parent, cb) {
+    // set the basedir properly so we don't try to resolve requires in the Closure
+    // Compiler processed `node_modules` folder.
+    parent.basedir = parent.filename === filename ? __dirname: path.dirname(parent.filename);
+
+    resolver(id, parent, cb);
+  },
+  filter: function(id) {
+    return !nodeResolve.isCore(id);
+}});
+
+var pkgJsons = [];
+var deps_files = {};
 
 md.on('package', function (pkg) {
   // we don't want to include the package.json for users' projects
   if (/node_modules/.test(pkg.__dirname)) {
-    deps_files.push({file: path.join(pkg.__dirname, 'package.json')});
+    var pkgJson = {
+      file: path.join(pkg.__dirname, 'package.json'),
+    };
+
+    if (pkg.name != null) {
+      pkgJson.provided = [ pkg.name ];
+    }
+
+    if (pkg.main != null) {
+      pkgJson.main = path.join(pkg.__dirname, pkg.main);
+    }
+
+    pkgJsons.push(pkgJson);
   }
 });
 
 md.on('file', function(file) {
-  deps_files.push({file: file});
+  deps_files[file] = { file: file };
 });
 
 md.on('end', function() {
-  process.stdout.write(JSON.stringify(deps_files));
+  for (var i = 0; i < pkgJsons.length; i++) {
+    var pkgJson = pkgJsons[i];
+
+    if (deps_files[pkgJson.main] != null && pkgJson.provided != null) {
+      deps_files[pkgJson.main].provides = pkgJson.provided;
+    }
+
+    deps_files[pkgJson.file] = { file: pkgJson.file };
+  }
+
+  var values = [];
+  for (var key in deps_files) {
+    values.push(deps_files[key]);
+  }
+
+  process.stdout.write(JSON.stringify(values));
 });
 
 md.end({
-  file: path.resolve(path.join(__dirname, 'JS_FILE'))
+  file: filename,
 });
 
 md.resume();

src/main/clojure/cljs/build/api.clj

@@ -21,11 +21,7 @@
             [cljs.compiler :as comp]
             [cljs.closure :as closure]
             [cljs.js-deps :as js-deps])
-  (:import [java.io
-            File StringWriter
-            BufferedReader
-            Writer InputStreamReader IOException]
-           [java.lang ProcessBuilder]))
+  (:import [java.io File]))
 
 ;; =============================================================================
 ;; Useful Utilities
@@ -219,57 +215,12 @@
    (binding [ana/*cljs-warning-handlers* (:warning-handlers opts ana/*cljs-warning-handlers*)]
      (closure/watch source opts compiler-env stop))))
 
-(defn- alive? [proc]
-  (try (.exitValue proc) false (catch IllegalThreadStateException _ true)))
-
-(defn- pipe [^Process proc in ^Writer out]
-  ;; we really do want system-default encoding here
-  (with-open [^java.io.Reader in (-> in InputStreamReader. BufferedReader.)]
-    (loop [buf (char-array 1024)]
-      (when (alive? proc)
-        (try
-          (let [len (.read in buf)]
-            (when-not (neg? len)
-              (.write out buf 0 len)
-              (.flush out)))
-          (catch IOException e
-            (when (and (alive? proc) (not (.contains (.getMessage e) "Stream closed")))
-              (.printStackTrace e *err*))))
-        (recur buf)))))
-
 (defn node-module-deps
   "EXPERIMENTAL: return the foreign libs entries as computed by running
    the module-deps package on the supplied JavaScript entry point. Assumes
    that the module-deps NPM package is either locally or globally installed."
-  [{:keys [file]}]
-  (let [code (string/replace
-               (slurp (io/resource "cljs/module_deps.js"))
-               "JS_FILE"
-               (string/replace file
-                 (System/getProperty "user.dir") ""))
-        proc (-> (ProcessBuilder.
-                   ["node" "--eval" code])
-               .start)
-        is   (.getInputStream proc)
-        iw   (StringWriter. (* 16 1024 1024))
-        es   (.getErrorStream proc)
-        ew   (StringWriter. (* 1024 1024))
-        _    (do (.start
-                   (Thread.
-                     (bound-fn [] (pipe proc is iw))))
-                 (.start
-                   (Thread.
-                     (bound-fn [] (pipe proc es ew)))))
-        err  (.waitFor proc)]
-    (if (zero? err)
-      (into []
-        (map (fn [{:strs [file]}] file
-               {:file file :module-type :commonjs}))
-        (next (json/read-str (str iw))))
-      (do
-        (when-not (.isAlive proc)
-          (println (str ew)))
-        []))))
+  [entry]
+  (closure/node-module-deps entry))
 
 (comment
   (node-module-deps
@@ -284,7 +235,7 @@
    the module-deps package on the supplied JavaScript entry points. Assumes
    that the module-deps NPM packages is either locally or globally installed."
   [entries]
-  (into [] (distinct (mapcat node-module-deps entries))))
+  (closure/node-inputs entries))
 
 (comment
   (node-inputs

src/main/clojure/cljs/closure.clj

@@ -47,7 +47,9 @@
             [clojure.data.json :as json]
             [clojure.tools.reader :as reader]
             [clojure.tools.reader.reader-types :as readers])
-  (:import [java.io File BufferedInputStream StringWriter]
+  (:import [java.lang ProcessBuilder]
+           [java.io File BufferedInputStream BufferedReader
+            Writer InputStreamReader IOException StringWriter]
            [java.net URL]
            [java.util.logging Level]
            [java.util List Random]
@@ -342,7 +344,6 @@
     (doseq [next (seq warnings)]
       (println "WARNING:" (.toString ^JSError next)))))
 
-
 ;; Protocols for IJavaScript and Compilable
 ;; ========================================
 
@@ -353,7 +354,7 @@
   (-source-map [this] "Return the CLJS compiler generated JS source mapping"))
 
 (extend-protocol deps/IJavaScript
-  
+
   String
   (-foreign? [this] false)
   (-closure-lib? [this] false)
@@ -362,7 +363,7 @@
   (-provides [this] (:provides (deps/parse-js-ns (string/split-lines this))))
   (-requires [this] (:requires (deps/parse-js-ns (string/split-lines this))))
   (-source [this] this)
-  
+
   clojure.lang.IPersistentMap
   (-foreign? [this] (:foreign this))
   (-closure-lib? [this] (:closure-lib this))
@@ -481,7 +482,7 @@
   returns a JavaScriptFile. In either case the return value satisfies
   IJavaScript."
   [^File file {:keys [output-file] :as opts}]
-    (if output-file 
+    (if output-file
       (let [out-file (io/file (util/output-directory opts) output-file)]
         (compiled-file (comp/compile-file file out-file opts)))
       (let [path (.getPath ^File file)]
@@ -567,17 +568,17 @@
     (case (.getProtocol this)
       "file" (-find-sources (io/file this) opts)
       "jar" (find-jar-sources this opts)))
-  
+
   clojure.lang.PersistentList
   (-compile [this opts]
     (compile-form-seq [this]))
   (-find-sources [this opts]
     [(ana/parse-ns [this] opts)])
-  
+
   String
   (-compile [this opts] (-compile (io/file this) opts))
   (-find-sources [this opts] (-find-sources (io/file this) opts))
-  
+
   clojure.lang.PersistentVector
   (-compile [this opts] (compile-form-seq this))
   (-find-sources [this opts]
@@ -1339,7 +1340,7 @@
 
   ;; optimize a ClojureScript form
   (optimize {:optimizations :simple} (-compile '(def x 3) {}))
-  
+
   ;; optimize a project
   (println (->> (-compile "samples/hello/src" {})
                 (apply add-dependencies {})
@@ -1730,7 +1731,7 @@
       (output-deps-file opts disk-sources))))
 
 (comment
-  
+
   ;; output unoptimized alone
   (output-unoptimized {} "goog.provide('test');\ngoog.require('cljs.core');\nalert('hello');\n")
   ;; output unoptimized with all dependencies
@@ -1916,12 +1917,19 @@
                 [lib])))]
     (into [] (mapcat expand-lib* libs))))
 
+(declare index-node-modules)
+
 (defn add-implicit-options
   [{:keys [optimizations output-dir]
     :or {optimizations :none
          output-dir "out"}
     :as opts}]
-  (let [opts (cond-> (update opts :foreign-libs expand-libs)
+  (let [opts (cond-> (update opts :foreign-libs
+                       (fn [libs]
+                         (into []
+                           (util/distinct-merge-by :file
+                             (index-node-modules opts)
+                             (expand-libs libs)))))
                (:closure-defines opts)
                (assoc :closure-defines
                  (into {}
@@ -1966,6 +1974,118 @@
       (nil? (:closure-module-roots opts))
       (assoc :closure-module-roots []))))
 
+(defn- alive? [proc]
+  (try (.exitValue proc) false (catch IllegalThreadStateException _ true)))
+
+(defn- pipe [^Process proc in ^Writer out]
+  ;; we really do want system-default encoding here
+  (with-open [^java.io.Reader in (-> in InputStreamReader. BufferedReader.)]
+    (loop [buf (char-array 1024)]
+      (when (alive? proc)
+        (try
+          (let [len (.read in buf)]
+            (when-not (neg? len)
+              (.write out buf 0 len)
+              (.flush out)))
+          (catch IOException e
+            (when (and (alive? proc) (not (.contains (.getMessage e) "Stream closed")))
+              (.printStackTrace e *err*))))
+        (recur buf)))))
+
+(defn maybe-install-node-deps!
+  [{:keys [npm-deps verbose] :as opts}]
+  (if-not (empty? npm-deps)
+    (do
+      (when (or ana/*verbose* verbose)
+        (util/debug-prn "Installing Node.js dependencies"))
+      (let [proc (-> (ProcessBuilder.
+                       (into ["npm" "install" "module-deps"]
+                         (map (fn [[dep version]] (str (name dep) "@" version)))
+                         npm-deps))
+                   .start)
+            is   (.getInputStream proc)
+            iw   (StringWriter. (* 16 1024 1024))
+            es   (.getErrorStream proc)
+            ew   (StringWriter. (* 1024 1024))
+            _    (do (.start
+                       (Thread.
+                         (bound-fn [] (pipe proc is iw))))
+                     (.start
+                       (Thread.
+                         (bound-fn [] (pipe proc es ew)))))
+            err  (.waitFor proc)]
+        (when (and (not (zero? err)) (not (.isAlive proc)))
+          (println (str ew)))
+        opts))
+    opts))
+
+(defn node-module-deps
+  "EXPERIMENTAL: return the foreign libs entries as computed by running
+   the module-deps package on the supplied JavaScript entry point. Assumes
+   that the module-deps NPM package is either locally or globally installed."
+  ([entry]
+   (node-module-deps entry
+     (when env/*compiler*
+       (:options @env/*compiler*))))
+  ([{:keys [file]} {:keys [target] :as opts}]
+   (let [code (-> (slurp (io/resource "cljs/module_deps.js"))
+                (string/replace "JS_FILE" file)
+                (string/replace "CLJS_TARGET" (str "" (when target (name target)))))
+         proc (-> (ProcessBuilder.
+                    ["node" "--eval" code])
+                .start)
+         is   (.getInputStream proc)
+         iw   (StringWriter. (* 16 1024 1024))
+         es   (.getErrorStream proc)
+         ew   (StringWriter. (* 1024 1024))
+         _    (do (.start
+                    (Thread.
+                      (bound-fn [] (pipe proc is iw))))
+                  (.start
+                    (Thread.
+                      (bound-fn [] (pipe proc es ew)))))
+         err  (.waitFor proc)]
+     (if (zero? err)
+       (into []
+         (map (fn [{:strs [file provides]}] file
+                (merge
+                  {:file file
+                   :module-type :commonjs}
+                  (when provides
+                    {:provides provides}))))
+         (next (json/read-str (str iw))))
+       (do
+         (when-not (.isAlive proc)
+           (println (str ew)))
+         [])))))
+
+(defn node-inputs
+  "EXPERIMENTAL: return the foreign libs entries as computed by running
+   the module-deps package on the supplied JavaScript entry points. Assumes
+   that the module-deps NPM packages is either locally or globally installed."
+  ([entries]
+   (node-inputs entries
+     (when env/*compiler*
+       (:options @env/*compiler*))))
+  ([entries opts]
+   (into [] (distinct (mapcat #(node-module-deps % opts) entries)))))
+
+(defn index-node-modules
+  ([]
+   (index-node-modules
+     (when env/*compiler*
+       (:options @env/*compiler*))))
+  ([{:keys [npm-deps] :as opts}]
+   (let [node-modules (io/file "node_modules")]
+     (when (and (.exists node-modules) (.isDirectory node-modules))
+       (let [modules (map name (keys npm-deps))
+             deps-file (io/file (str (util/output-directory opts) File/separator
+                                  "cljs$node_modules.js"))]
+         (util/mkdirs deps-file)
+         (with-open [w (io/writer deps-file)]
+           (run! #(.write w (str "require('" % "');\n")) modules))
+         (node-inputs [{:file (.getAbsolutePath deps-file)}]))))))
+
 (defn process-js-modules
   "Given the current compiler options, converts JavaScript modules to Google
   Closure modules and writes them to disk. Adds mapping from original module
@@ -2067,6 +2187,7 @@
      (env/with-compiler-env compiler-env
        (let [compiler-stats (:compiler-stats opts)
              all-opts (-> opts
+                          maybe-install-node-deps!
                           add-implicit-options
                           process-js-modules)]
          (check-output-to opts)