Saturday 25 August 2018

Fastest csv parser for Node?

Currently using the fast-csv node module for csv parsing and the chokidar module to check for file changes. There are about 3000 lines.I add each line from the csv file to an object called csvList using one of the columns as a key. I compare the list on the next read to the one already in memory.Is there a better / faster way of doing it? I noticed sometimes the csv file change is not detected or it's not read correctly.var fs = require('fs'); var chokidar = require('chokidar'); var csv = require('fast-csv'); let csvList = {}; var csvWatcher = chokidar.watch('file.csv', { ignored: /(^|[\/\\])\../, persistent: true }); csvWatcher.on('change', ()=> { runupdate(); }); function runupdate() { let csvStream = csv.fromPath("file.csv", { headers: myHeaders, trim: true, ignoreEmpty:true, strictColumnHandling:false}) .on("data", function(r) { if (r) { csvStream.pause(); if (!csvList[r.symbol]) { csvList[r.symbol] = {symbol : r.symbol, time : r.time, volume: r.volume } } else { if (csvList[r.symbol].time != r.time) { csvList[r.symbol] = { volume : r.volume, time : r.time} } csvStream.resume(); } }).on("end", function() { csvStream = null; }); }

Submitted August 25, 2018 at 03:36PM by mainst

No comments:

Post a Comment