From 8d1304629350b917c3e6a324b594402c91e423c1 Mon Sep 17 00:00:00 2001 From: tamtakoe Date: Fri, 17 Apr 2015 02:55:54 +0300 Subject: [PATCH 1/2] first commit --- .gitignore | 1 + Cakefile | 16 ------ README.md | 7 ++- index.js | 128 ++++++++++++++++++++++++++++++++++++++++++++++ lib/header.js | 70 ------------------------- lib/parser.js | 118 ------------------------------------------ package.json | 19 ++----- src/header.coffee | 42 --------------- src/parser.coffee | 90 -------------------------------- 9 files changed, 139 insertions(+), 352 deletions(-) delete mode 100644 Cakefile create mode 100644 index.js delete mode 100644 lib/header.js delete mode 100644 lib/parser.js delete mode 100644 src/header.coffee delete mode 100644 src/parser.coffee diff --git a/.gitignore b/.gitignore index d251b0e..dec0144 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.sublime-project *.sublime-workspace +/.idea/ node_modules/ \ No newline at end of file diff --git a/Cakefile b/Cakefile deleted file mode 100644 index 31e804a..0000000 --- a/Cakefile +++ /dev/null @@ -1,16 +0,0 @@ -fs = require 'fs' - -{print} = require 'sys' -{exec} = require 'child_process' - -build = (callback) -> - coffee = exec 'coffee -c -o lib src' - coffee.stderr.on 'data', (data) -> - process.stderr.write data.toString() - coffee.stdout.on 'data', (data) -> - print data.toString() - coffee.on 'exit', (code) -> - callback?() if code is 0 - -task 'build', 'Build lib/ from src/', -> - build() \ No newline at end of file diff --git a/README.md b/README.md index 26e1769..08c0da9 100644 --- a/README.md +++ b/README.md @@ -3,13 +3,16 @@ node-dbf This is an event-based dBase file parser for very efficiently reading data from *.dbf files. +Based on https://github.com/abstractvector/node-dbf but works without coffee and provide float type of numbers. +F.e. correct work with VMap0 geonames base (http://gis-lab.info/data/vmap0-settl-rus/population-gislab.7z) + To get started, simply install the module using npm: - npm install node-dbf + npm install dbf-parser and then `require` it: - var Parser = require('node-dbf'); + var Parser = require('dbf-parser'); #Classes diff --git a/index.js b/index.js new file mode 100644 index 0000000..95ee32a --- /dev/null +++ b/index.js @@ -0,0 +1,128 @@ +var fs = require('fs'); +var util = require('util'); +var EventEmitter = require('events').EventEmitter; + + +function Header(filename) { + this.filename = filename; + this.parseFieldSubRecord = this.parseFieldSubRecord.bind(this); + this.parseDate = this.parseDate.bind(this); + + return this; +} + +Header.prototype.parse = function(callback) { + var self = this; + return fs.readFile(this.filename, function(err, buffer) { + var i; + if (err) { + throw err; + } + self.type = (buffer.slice(0, 1)).toString('utf-8'); + self.dateUpdated = self.parseDate(buffer.slice(1, 4)); + self.numberOfRecords = self.convertBinaryToInteger(buffer.slice(4, 8)); + self.start = self.convertBinaryToInteger(buffer.slice(8, 10)); + self.recordLength = self.convertBinaryToInteger(buffer.slice(10, 12)); + self.fields = ((function() { + var _i, _ref, _results; + _results = []; + for (i = _i = 32, _ref = this.start - 32; _i <= _ref; i = _i += 32) { + _results.push(buffer.slice(i, i + 32)); + } + return _results; + }).call(self)).map(self.parseFieldSubRecord); + return callback(self); + }); +}; + +Header.prototype.parseDate = function(buffer) { + var day, month, year; + console.log(this.convertBinaryToInteger(buffer.slice(0, 1))); + year = 1900 + this.convertBinaryToInteger(buffer.slice(0, 1)); + month = (this.convertBinaryToInteger(buffer.slice(1, 2))) - 1; + day = this.convertBinaryToInteger(buffer.slice(2, 3)); + return new Date(year, month, day); +}; + +Header.prototype.parseFieldSubRecord = function(buffer) { + var header = { + name: ((buffer.slice(0, 11)).toString('utf-8')).replace(/[\u0000]+$/, ''), + type: (buffer.slice(11, 12)).toString('utf-8'), + displacement: this.convertBinaryToInteger(buffer.slice(12, 16)), + length: this.convertBinaryToInteger(buffer.slice(16, 17)), + decimalPlaces: this.convertBinaryToInteger(buffer.slice(17, 18)) + }; + + return header +}; + +Header.prototype.convertBinaryToInteger = function(buffer) { + return buffer.readInt32LE(0, true); +}; + + + + +function Parser(filename) { + this.filename = filename; + this.parseField = this.parseField.bind(this); + this.parseRecord = this.parseRecord.bind(this); + this.parse = this.parse.bind(this); +} + +util.inherits(Parser, EventEmitter); + +Parser.prototype.parse = function() { + var self = this; + this.emit('start', this); + this.header = new Header(this.filename); + this.header.parse(function(err) { + var sequenceNumber; + self.emit('header', self.header); + sequenceNumber = 0; + return fs.readFile(self.filename, function(err, buffer) { + var loc; + if (err) { + throw err; + } + loc = self.header.start; + while (loc < (self.header.start + self.header.numberOfRecords * self.header.recordLength) && loc < buffer.length) { + self.emit('record', self.parseRecord(++sequenceNumber, buffer.slice(loc, loc += self.header.recordLength))); + } + return self.emit('end', self); + }); + }); + return this; +}; + +Parser.prototype.parseRecord = function(sequenceNumber, buffer) { + var field, loc, record, _fn, _i, _len, _ref, + self = this; + record = { + '@sequenceNumber': sequenceNumber, + '@deleted': (buffer.slice(0, 1))[0] !== 32 + }; + loc = 1; + _ref = this.header.fields; + _fn = function(field) { + var value = record[field.name] = self.parseField(field, buffer.slice(loc, loc += field.length)); + + return value; + }; + for (_i = 0, _len = _ref.length; _i < _len; _i++) { + field = _ref[_i]; + _fn(field); + } + return record; +}; + +Parser.prototype.parseField = function(field, buffer) { + var value = buffer.toString('utf-8').replace(/^\x20+|\x20+$/g, ''); + + if (field.type === 'N') { + value = Number(value); + } + return value; +}; + +module.exports = Parser; \ No newline at end of file diff --git a/lib/header.js b/lib/header.js deleted file mode 100644 index 23637a5..0000000 --- a/lib/header.js +++ /dev/null @@ -1,70 +0,0 @@ -// Generated by CoffeeScript 1.7.1 -(function() { - var Header, fs, - __bind = function(fn, me){ return function(){ return fn.apply(me, arguments); }; }; - - fs = require('fs'); - - Header = (function() { - function Header(filename) { - this.filename = filename; - this.parseFieldSubRecord = __bind(this.parseFieldSubRecord, this); - this.parseDate = __bind(this.parseDate, this); - return this; - } - - Header.prototype.parse = function(callback) { - return fs.readFile(this.filename, (function(_this) { - return function(err, buffer) { - var i; - if (err) { - throw err; - } - _this.type = (buffer.slice(0, 1)).toString('utf-8'); - _this.dateUpdated = _this.parseDate(buffer.slice(1, 4)); - _this.numberOfRecords = _this.convertBinaryToInteger(buffer.slice(4, 8)); - _this.start = _this.convertBinaryToInteger(buffer.slice(8, 10)); - _this.recordLength = _this.convertBinaryToInteger(buffer.slice(10, 12)); - _this.fields = ((function() { - var _i, _ref, _results; - _results = []; - for (i = _i = 32, _ref = this.start - 32; _i <= _ref; i = _i += 32) { - _results.push(buffer.slice(i, i + 32)); - } - return _results; - }).call(_this)).map(_this.parseFieldSubRecord); - return callback(_this); - }; - })(this)); - }; - - Header.prototype.parseDate = function(buffer) { - var day, month, year; - year = 1900 + this.convertBinaryToInteger(buffer.slice(0, 1)); - month = (this.convertBinaryToInteger(buffer.slice(1, 2))) - 1; - day = this.convertBinaryToInteger(buffer.slice(2, 3)); - return new Date(year, month, day); - }; - - Header.prototype.parseFieldSubRecord = function(buffer) { - var header; - return header = { - name: ((buffer.slice(0, 11)).toString('utf-8')).replace(/[\u0000]+$/, ''), - type: (buffer.slice(11, 12)).toString('utf-8'), - displacement: this.convertBinaryToInteger(buffer.slice(12, 16)), - length: this.convertBinaryToInteger(buffer.slice(16, 17)), - decimalPlaces: this.convertBinaryToInteger(buffer.slice(17, 18)) - }; - }; - - Header.prototype.convertBinaryToInteger = function(buffer) { - return buffer.readInt32LE(0, true); - }; - - return Header; - - })(); - - module.exports = Header; - -}).call(this); diff --git a/lib/parser.js b/lib/parser.js deleted file mode 100644 index 9556918..0000000 --- a/lib/parser.js +++ /dev/null @@ -1,118 +0,0 @@ -// Generated by CoffeeScript 1.7.1 -(function() { - var EventEmitter, Header, Parser, fs, - __bind = function(fn, me){ return function(){ return fn.apply(me, arguments); }; }, - __hasProp = {}.hasOwnProperty, - __extends = function(child, parent) { for (var key in parent) { if (__hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; }; - - EventEmitter = require('events').EventEmitter; - - Header = require('./header'); - - fs = require('fs'); - - Parser = (function(_super) { - __extends(Parser, _super); - - function Parser(filename) { - this.filename = filename; - this.parseField = __bind(this.parseField, this); - this.parseRecord = __bind(this.parseRecord, this); - this.resume = __bind(this.resume, this); - this.pause = __bind(this.pause, this); - this.parse = __bind(this.parse, this); - } - - Parser.prototype.parse = function() { - this.emit('start', this); - this.header = new Header(this.filename); - this.header.parse((function(_this) { - return function(err) { - var bufLoc, loc, overflow, sequenceNumber, stream; - _this.emit('header', _this.header); - sequenceNumber = 0; - loc = _this.header.start; - bufLoc = _this.header.start; - overflow = null; - _this.paused = false; - stream = fs.createReadStream(_this.filename); - _this.readBuf = function() { - var buffer; - if (_this.paused) { - _this.emit('paused'); - return; - } - while (buffer = stream.read()) { - if (bufLoc !== _this.header.start) { - bufLoc = 0; - } - if (overflow !== null) { - buffer = overflow + buffer; - } - while (loc < (_this.header.start + _this.header.numberOfRecords * _this.header.recordLength) && (bufLoc + _this.header.recordLength) <= buffer.length) { - _this.emit('record', _this.parseRecord(++sequenceNumber, buffer.slice(bufLoc, bufLoc += _this.header.recordLength))); - } - loc += bufLoc; - if (bufLoc < buffer.length) { - overflow = buffer.slice(bufLoc, buffer.length); - } else { - overflow = null; - } - return _this; - } - }; - stream.on('readable', _this.readBuf); - return stream.on('end', function() { - return _this.emit('end'); - }); - }; - })(this)); - return this; - }; - - Parser.prototype.pause = function() { - return this.paused = true; - }; - - Parser.prototype.resume = function() { - this.paused = false; - this.emit('resuming'); - return this.readBuf(); - }; - - Parser.prototype.parseRecord = function(sequenceNumber, buffer) { - var field, loc, record, _fn, _i, _len, _ref; - record = { - '@sequenceNumber': sequenceNumber, - '@deleted': (buffer.slice(0, 1))[0] !== 32 - }; - loc = 1; - _ref = this.header.fields; - _fn = (function(_this) { - return function(field) { - return record[field.name] = _this.parseField(field, buffer.slice(loc, loc += field.length)); - }; - })(this); - for (_i = 0, _len = _ref.length; _i < _len; _i++) { - field = _ref[_i]; - _fn(field); - } - return record; - }; - - Parser.prototype.parseField = function(field, buffer) { - var value; - value = (buffer.toString('utf-8')).replace(/^\x20+|\x20+$/g, ''); - if (field.type === 'N') { - value = parseInt(value, 10); - } - return value; - }; - - return Parser; - - })(EventEmitter); - - module.exports = Parser; - -}).call(this); diff --git a/package.json b/package.json index e84016f..e68ac6a 100644 --- a/package.json +++ b/package.json @@ -1,24 +1,15 @@ { - "name": "node-dbf", - "version": "0.1.1", + "name": "dbf-parser", + "version": "0.0.1", "description": "An efficient dBase DBF file parser written in pure JavaScript", - "main": "./lib/parser.js", + "main": "./index.js", "repository": { "type": "git", - "url": "git://github.com/abstractvector/node-dbf.git" + "url": "git://github.com/tamtakoe/node-dbf.git" }, "keywords": [ "dBase", "dbf" ], - "author": "Matt Knight ", - "devDependencies": { - "coffee-script": ">=1.3.3" - }, - "dependencies": { - "iconv-lite": ">=0.2.11" - }, - "scripts": { - "prepublish": "cake build" - } + "author": "tamtakoe " } diff --git a/src/header.coffee b/src/header.coffee deleted file mode 100644 index 86080fb..0000000 --- a/src/header.coffee +++ /dev/null @@ -1,42 +0,0 @@ -fs = require 'fs' - -class Header - - constructor: (@filename) -> - return @ - - parse: (callback) -> - fs.readFile @filename, (err, buffer) => - throw err if err - - @type = (buffer.slice 0, 1).toString 'utf-8' - @dateUpdated = @parseDate (buffer.slice 1, 4) - @numberOfRecords = @convertBinaryToInteger (buffer.slice 4, 8) - @start = @convertBinaryToInteger (buffer.slice 8, 10) - @recordLength = @convertBinaryToInteger (buffer.slice 10, 12) - - @fields = (buffer.slice i, i+32 for i in [32 .. @start - 32] by 32).map @parseFieldSubRecord - - callback @ - - parseDate: (buffer) => - console.log @convertBinaryToInteger buffer.slice 0, 1 - year = 1900 + @convertBinaryToInteger buffer.slice 0, 1 - month = (@convertBinaryToInteger buffer.slice 1, 2) - 1 - day = @convertBinaryToInteger buffer.slice 2, 3 - - return new Date year, month, day - - parseFieldSubRecord: (buffer) => - header = { - name: ((buffer.slice 0, 11).toString 'utf-8').replace /[\u0000]+$/, '' - type: (buffer.slice 11, 12).toString 'utf-8' - displacement: @convertBinaryToInteger buffer.slice 12, 16 - length: @convertBinaryToInteger buffer.slice 16, 17 - decimalPlaces: @convertBinaryToInteger buffer.slice 17, 18 - } - - convertBinaryToInteger: (buffer) -> - return buffer.readInt32LE 0, true - -module.exports = Header \ No newline at end of file diff --git a/src/parser.coffee b/src/parser.coffee deleted file mode 100644 index 473763c..0000000 --- a/src/parser.coffee +++ /dev/null @@ -1,90 +0,0 @@ -{EventEmitter} = require 'events' -Header = require './header' -fs = require 'fs' - -class Parser extends EventEmitter - - constructor: (@filename) -> - - parse: => - @emit 'start', @ - - @header = new Header @filename - @header.parse (err) => - - @emit 'header', @header - - sequenceNumber = 0 - - loc = @header.start - bufLoc = @header.start - overflow = null - @paused = false - - stream = fs.createReadStream @filename - - @readBuf = => - - if @paused - - @emit 'paused' - - return - - while buffer = stream.read() - - if bufLoc isnt @header.start then bufLoc = 0 - - if overflow isnt null then buffer = overflow + buffer - - while loc < (@header.start + @header.numberOfRecords * @header.recordLength) && (bufLoc + @header.recordLength) <= buffer.length - - @emit 'record', @parseRecord ++sequenceNumber, buffer.slice bufLoc, bufLoc += @header.recordLength - - loc += bufLoc - - if bufLoc < buffer.length then overflow = buffer.slice bufLoc, buffer.length else overflow = null - - return @ - - stream.on 'readable',@readBuf - - stream.on 'end', () => - - @emit 'end' - - return @ - - pause: => - - @paused = true - - resume: => - - @paused = false - - @emit 'resuming' - - do @readBuf - - parseRecord: (sequenceNumber, buffer) => - record = { - '@sequenceNumber': sequenceNumber - '@deleted': (buffer.slice 0, 1)[0] isnt 32 - } - - loc = 1 - for field in @header.fields - do (field) => - record[field.name] = @parseField field, buffer.slice loc, loc += field.length - - return record - - parseField: (field, buffer) => - value = (buffer.toString 'utf-8').replace /^\x20+|\x20+$/g, '' - - if field.type is 'N' then value = parseInt value, 10 - - return value - -module.exports = Parser From ffcb419391d61f78dac82f4ee4ef2fb6d8435ec7 Mon Sep 17 00:00:00 2001 From: tamtakoe Date: Fri, 17 Apr 2015 02:59:55 +0300 Subject: [PATCH 2/2] some fixes --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 08c0da9..071c7be 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -node-dbf +dbf-parser ======== This is an event-based dBase file parser for very efficiently reading data from *.dbf files.