Merge branch 'parsers'

Author: extrabacon

README.md

@@ -5,7 +5,8 @@ A simple way to run Python scripts from Node.js with basic but efficient inter-p
 ## Features
 
 + Reliably spawn Python scripts in a child process
-+ Text, JSON and binary modes
++ Built-in text, JSON and binary modes
++ Custom parsers and formatters
 + Simple and efficient data transfers through stdin and stdout streams
 + Extended stack traces when an error is thrown
 
@@ -78,6 +79,12 @@ pyshell.end(function (err) {
 
 Use `.send(message)` to send a message to the Python script. Attach the `message` event to listen to messages emitted from the Python script.
 
+Use `options.mode` to quickly setup how data is sent and received between your Node and Python applications.
+
+  * use `text` mode for exchanging lines of text
+  * use `json` mode for exchanging JSON fragments
+  * use `binary` mode for anything else (data is sent and received as-is)
+
 For more details and examples including Python source code, take a look at the tests.
 
 ### Error Handling and extended stack traces
@@ -129,6 +136,9 @@ Creates an instance of `PythonShell` and starts the Python process
     * `text`: each line of data (ending with "\n") is emitted as a message (default)
     * `json`: each line of data (ending with "\n") is parsed as JSON and emitted as a message
     * `binary`: data is streamed as-is through `stdout` and `stdin`
+  * `formatter`: each message to send is transformed using this method, then appended with "\n"
+  * `parser`: each line of data (ending with "\n") is parsed with this function and its result is emitted as a message
+  * `encoding`: the text encoding to apply on the child process streams (default: "utf8")
   * `pythonPath`: The path where to locate the "python" executable. Default: "python"
   * `pythonOptions`: Array of option switches to pass to "python"
   * `scriptPath`: The default path where to look for scripts. Default: "./python"
@@ -178,9 +188,7 @@ PythonShell.run('script.py', function (err, results) {
 
 #### `.send(message)`
 
-Sends a message to the Python script via stdin. The data is formatted according to the selected mode (text or JSON). This method can be overridden in order to format the data in some other way.
-
-This method should not be used in binary mode.
+Sends a message to the Python script via stdin. The data is formatted according to the selected mode (text or JSON), or through a custom function when `formatter` is specified.
 
 Example:
 ```js
@@ -195,15 +203,15 @@ shell.send({ command: "do_stuff", args: [1, 2, 3] });
 
 #### `.receive(data)`
 
-Parses incoming data from the Python script written via stdout and emits `message` events. The data is parsed as JSON if mode has been set to "json". This method is called automatically as data is being received from stdout and can be overridden to parse the data differently.
+Parses incoming data from the Python script written via stdout and emits `message` events. This method is called automatically as data is being received from stdout.
 
 #### `.end(callback)`
 
 Closes the stdin stream, allowing the Python script to finish and exit. The optional callback is invoked when the process is terminated.
 
 #### event: `message`
 
-Fires when a chunk of data is parsed from the stdout stream via the `receive` method. This event is not emitted in binary mode.
+Fires when a chunk of data is parsed from the stdout stream via the `receive` method. If a `parser` method is specified, the result of this function will be the message value. This event is not emitted in binary mode.
 
 Example:
 ```js

index.js

@@ -30,6 +30,17 @@ function extend(obj) {
  * @constructor
  */
 var PythonShell = function (script, options) {
+
+    function resolve(type, val) {
+        if (typeof val === 'string') {
+            // use a built-in function using its name
+            return PythonShell[type][val];
+        } else if (typeof val === 'function') {
+            // use a custom function
+            return val;
+        }
+    }
+
     var self = this;
     var errorData = '';
     EventEmitter.call(this);
@@ -42,18 +53,20 @@ var PythonShell = function (script, options) {
     this.script = path.join(options.scriptPath || './python', script);
     this.command = pythonOptions.concat(this.script, scriptArgs);
     this.mode = options.mode || 'text';
+    this.formatter = resolve('format', options.formatter || this.mode);
+    this.parser = resolve('parse', options.parser || this.mode);
     this.terminated = false;
     this.childProcess = spawn(pythonPath, this.command, options);
 
     ['stdout', 'stdin', 'stderr'].forEach(function (name) {
         self[name] = self.childProcess[name];
-        self.mode !== 'binary' && self[name].setEncoding('utf8');
+        self.parser && self[name].setEncoding(options.encoding || 'utf8');
     });
 
-    // listen for incoming data on stdout
-    this.stdout.on('data', function (data) {
-        self.mode !== 'binary' && self.receive(data);
-    });
+    // parse incoming data on stdout
+    if (this.parser) {
+        this.stdout.on('data', PythonShell.prototype.receive.bind(this));
+    }
 
     // listen to stderr and emit errors for incoming data
     this.stderr.on('data', function (data) {
@@ -91,6 +104,28 @@ util.inherits(PythonShell, EventEmitter);
 // allow global overrides for options
 PythonShell.defaultOptions = {};
 
+// built-in formatters
+PythonShell.format = {
+    text: function toText(data) {
+        if (!data) return '';
+        else if (typeof data !== 'string') return data.toString();
+        return data;
+    },
+    json: function toJson(data) {
+        return JSON.stringify(data);
+    }
+};
+
+// built-in parsers
+PythonShell.parse = {
+    text: function asText(data) {
+        return data;
+    },
+    json: function asJson(data) {
+        return JSON.parse(data);
+    }
+};
+
 /**
  * Runs a Python script and returns collected messages
  * @param  {string}   script   The script to execute
@@ -143,22 +178,14 @@ PythonShell.prototype.parseError = function (data) {
 
 /**
  * Sends a message to the Python shell through stdin
- * This method
  * Override this method to format data to be sent to the Python process
  * @param {string|Object} data The message to send
  * @returns {PythonShell} The same instance for chaining calls
  */
 PythonShell.prototype.send = function (message) {
-    if (this.mode === 'binary') {
-        throw new Error('cannot send a message in binary mode, use stdin directly instead');
-    } else if (this.mode === 'json') {
-        // write a JSON formatted message
-        this.stdin.write(JSON.stringify(message) + '\n');
-    } else {
-        // write text-based message (default)
-        if (typeof message !== 'string') message = message.toString();
-        this.stdin.write(message + '\n');
-    }
+    var data = this.formatter ? this.formatter(message) : message;
+    if (this.mode !== 'binary') data += '\n';
+    this.stdin.write(data);
     return this;
 };
 
@@ -170,32 +197,28 @@ PythonShell.prototype.send = function (message) {
  */
 PythonShell.prototype.receive = function (data) {
     var self = this;
-    var lines = (''+data).split(/\n/g);
+    var parts = (''+data).split(/\n/g);
 
-    if (lines.length === 1) {
+    if (parts.length === 1) {
         // an incomplete record, keep buffering
-        this._remaining = (this._remaining || '') + lines[0];
+        this._remaining = (this._remaining || '') + parts[0];
         return this;
     }
 
-    var lastLine = lines.pop();
+    var lastLine = parts.pop();
     // fix the first line with the remaining from the previous iteration of 'receive'
-    lines[0] = (this._remaining || '') + lines[0];
+    parts[0] = (this._remaining || '') + parts[0];
     // keep the remaining for the next iteration of 'receive'
     this._remaining = lastLine;
 
-    lines.forEach(function (line) {
-        if (self.mode === 'json') {
-            try {
-                self.emit('message', JSON.parse(line));
-            } catch (err) {
-                self.emit('error', extend(
-                    new Error('invalid JSON message: ' + data + ' >> ' + err),
-                    { inner: err, data: line}
-                ));
-            }
-        } else {
-            self.emit('message', line);
+    parts.forEach(function (part) {
+        try {
+            self.emit('message', self.parser(part));
+        } catch(err) {
+            self.emit('error', extend(
+                new Error('invalid message: ' + data + ' >> ' + err),
+                { inner: err, data: part}
+            ));
         }
     });
 

package.json

@@ -10,8 +10,8 @@
   },
   "dependencies": {},
   "devDependencies": {
-    "should": "^3.1.3",
-    "mocha": "^1.17.1"
+    "should": "^6.0.0",
+    "mocha": "^2.2.5"
   },
   "repository": {
     "type": "git",

test/python/echo_binary.py

@@ -0,0 +1,4 @@
+import sys
+
+# simple binary echo script
+sys.stdout.write(sys.stdin.read())

test/test-python-shell.js

@@ -91,14 +91,35 @@ describe('PythonShell', function () {
                 done();
             });
         });
-        it('should throw when mode is "binary"', function (done) {
+        it('should use a custom formatter', function (done) {
             var pyshell = new PythonShell('echo_text.py', {
+                formatter: function (message) {
+                    return message.toUpperCase();
+                }
+            });
+            var output = '';
+            pyshell.stdout.on('data', function (data) {
+                output += ''+data;
+            });
+            pyshell.send('hello').send('world').end(function (err) {
+                if (err) return done(err);
+                output.should.be.exactly('HELLO\nWORLD\n');
+                done();
+            });
+        });
+        it('should write as-is when mode is "binary"', function (done) {
+            var pyshell = new PythonShell('echo_binary.py', {
                 mode: 'binary'
             });
-            (function () {
-                pyshell.send('hello world!');
-            }).should.throw();
-            pyshell.end(done);
+            var output = '';
+            pyshell.stdout.on('data', function (data) {
+                output += ''+data;
+            });
+            pyshell.send(new Buffer('i am not a string')).end(function (err) {
+                if (err) return done(err);
+                output.should.be.exactly('i am not a string');
+                done();
+            });
         });
     });
 
@@ -150,6 +171,22 @@ describe('PythonShell', function () {
             };
             pyshell.end(done);
         });
+        it('should use a custom parser function', function (done) {
+            var pyshell = new PythonShell('echo_text.py', {
+                mode: 'text',
+                parser: function (message) {
+                    return message.toUpperCase();
+                }
+            });
+            var count = 0;
+            pyshell.on('message', function (message) {
+                count === 0 && message.should.be.exactly('HELLO');
+                count === 1 && message.should.be.exactly('WORLD!');
+                count++;
+            }).on('close', function () {
+                count.should.be.exactly(2);
+            }).send('hello').send('world!').end(done);
+        });
     });
 
     describe('.end(callback)', function () {