Using the “Oboe” Javascript library to speed up data processing
One of our node.js applications at work was designed to call an internal rest service, cache the whole response, and iterate over the data to separate relevant data from non-relevant data, and write the relevant data to file.
To speed things up a bit, we wanted to process, filter and write the data to file as it arrived. For this task, the excellent Javascript library “Oboe” came in handy:
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 |
var oboe = require('oboe'); var fs = require('fs'); var jsonFileStream = fs.createWriteStream('myfile.json')[...] var acceptHeader = { Accept: 'application/json' } var requestHeader = { url: "http://rest.example.local/myapp", headers: acceptHeader } oboe(requestHeader) .node('{ref hostname}', function(item) { handleItem(node, write_item_to_file); }) .done(function() { jsonFileStream.end(); }); function handleItem(item, callback) { if (isInterestingData(item)) { callback(item); } } function isInterestingData(item) {...} function write_item_to_file(item) {...} |
Keywords: HTTP response streaming