Hello, I am trying to process a 100 megabyte (~ 900 thousand lines in a file) xml file in streaming mode
const XmlReader = require('xml-reader')
const file = fs.createReadStream(
'public/prices/middle-price_list.yml',
'utf8'
)
const parser = XmlReader.create({ stream: true })
file.on('data', (chunk) => {
parser.parse(chunk)
})
// Tried another option from the documentation
// file.on('data', (chunk) => {
// chunk.split('').forEach((char) => parser.parse(char))
// })
parser.on('tag:catalog', (data) => {
objShop.date = data.attributes.date
})
parser.on('tag:shop', (data) => {
objShop.name = data.children.filter(
(el) => el.name == 'name'
)[0]?.children[0]?.value
objShop.company = data.children.filter(
(el) => el.name == 'company'
)[0]?.children[0]?.value
objShop.url = data.children.filter(
(el) => el.name == 'url'
)[0]?.children[0]?.value
})
parser.on('tag:offer', (data) => {
objPrice.push({
title: data.children.filter((el) => el.name == 'name')[0]?.children[0]
?.value,
url: data.children.filter((el) => el.name == 'url')[0]?.children[0]
?.value,
picture: data.children.filter((el) => el.name == 'picture')[0]
?.children[0]?.value,
})
})
parser.on('done', (data) => {
console.log('objShop', objShop)
console.log('objPrice', objPrice.length)
})The file is processed normally, but during processing, the RAM is occupied by more than 1.5 gigabytes
what am I doing wrong?
Hello, I am trying to process a 100 megabyte (~ 900 thousand lines in a file) xml file in streaming mode
The file is processed normally, but during processing, the RAM is occupied by more than 1.5 gigabytes
what am I doing wrong?