w-json-stream

A tool for JSON parse and stringify by stream and web worker.

Documentation

To view documentation or get support, visit docs.

Installation

Using npm(ES6 module):

Note: w-json-stream is mainly dependent on from2 and jsonparse.

npm i w-json-stream

Example for samll data:

Link: [dev source code]

async function testSmall() {

    let obj = {
        a: 123,
        b: 145.67,
        c: 'test中文1',
        d: true,
        f: [11, 'xyz', false, [166, 197, 215]],
        g: {
            ga: 223,
            gb: 245.67,
            gc: 'test中文2',
            gd: [66, 97, 115],
        },
    }
    let arr = [
        {
            a: 123,
            b: 145.67,
            c: 'test中文1',
            d: true,
        },
        {
            f: [11, 'xyz', false, [166, 197, 215]],
            g: {
                ga: 223,
                gb: 245.67,
                gc: 'test中文2',
                gd: [66, 97, 115],
            },
        },
    ]

    let jobj = await json.stringify(obj)
    console.log(jobj)
    // => {"a":123,"b":145.67,"c":"test中文1","d":true,"f":[11,"xyz",false,[166,197,215]],"g":{"ga":223,"gb":245.67,"gc":"test中文2","gd":[66,97,115]}}

    let jarr = await json.stringify(arr)
    console.log(jarr)
    // => [{"a":123,"b":145.67,"c":"test中文1","d":true},{"f":[11,"xyz",false,[166,197,215]],"g":{"ga":223,"gb":245.67,"gc":"test中文2","gd":[66,97,115]}}]

    let robj = await json.parse(jobj)
    console.log(robj)
    // => {
    //     a: 123,
    //     b: 145.67,
    //     c: 'test中文1',
    //     d: true,
    //     f: [ 11, 'xyz', false, [ 166, 197, 215 ] ],
    //     g: { ga: 223, gb: 245.67, gc: 'test中文2', gd: [ 66, 97, 115 ] }
    // }

    let rarr = await json.parse(jarr)
    console.log(rarr)
    // => [
    //     { a: 123, b: 145.67, c: 'test中文1', d: true },
    //     {
    //       f: [ 11, 'xyz', false, [Array] ],
    //       g: { ga: 223, gb: 245.67, gc: 'test中文2', gd: [Array] }
    //     }
    // ]

}
testSmall()
    .catch((err) => {
        console.log(err)
    })

Example for large data:

Link: [dev source code]

async function testLarge() {

    let n = 700 //7000, 9000
    let lgRows = []
    for (let i = 1; i <= 1000; i++) {
        lgRows.push([i, 23.4, '567', 'abc', true, false, [], {}, [8, 9.01], { a: 12.34, b: 'xyz', i }])
    }
    let lgArr = []
    for (let i = 1; i <= n; i++) {
        lgArr.push({
            key: 'k' + i,
            value: {
                a: 123,
                b: 145.67,
                c: 'test中文1',
                d: true,
                f: [11, 'xyz', false, [166, 197, 215]],
                g: {
                    ga: 223,
                    gb: 245.67,
                    gc: 'test中文2',
                    gd: [66, 97, 115],
                    ge: JSON.parse(JSON.stringify(lgRows)),
                },
            }
        })
    }

    if (true) {

        try {
            let res = JSON.stringify(lgArr)
            console.log('JSON.stringify(lgArr)', res.length, res.substr(0, 200) + '...')
        }
        catch (err) {
            console.log('JSON.stringify(lgArr) catch', err)
            // n=7000 => catch RangeError: Invalid string length => Out Of Memory
            // n=9000 => FATAL ERROR: MarkCompactCollector: young object promotion failed Allocation failed - JavaScript heap out of memory
        }

    }

    if (true) {

        await json.stringify(lgArr)
            .then((res) => {
                console.log('json.stringify(lgArr) then', res.length, res.substr(0, 200) + '...')
            })
            .catch((err) => {
                console.log('json.stringify(lgArr) catch', err)
                // n=7000 => FATAL ERROR: Ineffective mark-compacts near heap limit Allocation failed - JavaScript heap out of memory
            })

    }

}
testLarge()
    .catch((err) => {
        console.log(err)
    })

Example for stream:

Link: [dev source code]

async function testStream() {

    async function testStringify() {
        return new Promise((resolve, reject) => {

            //fp
            let fp = './temp-stringify.json'

            //arr
            let n = 4000 //4000000
            let arr = []
            for (let i = 1; i <= n; i++) {
                arr.push({
                    key: 'k' + i,
                    value: {
                        a: 123,
                        b: 145.67,
                        c: 'test中文1',
                        d: true,
                        f: [11, 'xyz', false, [166, 197, 215]],
                        g: {
                            ga: 223,
                            gb: 245.67,
                            gc: 'test中文2',
                            gd: [66, 97, 115],
                        },
                    }
                })
            }

            //createStringifyStream
            let stringifyStream = json.createStringifyStream(arr)

            //createWriteStream
            let writerStream = fs.createWriteStream(fp)

            // //pipe
            // stringifyStream.pipe(writerStream)

            //stringifyStream onData
            stringifyStream.on('data', function(chunk) {
                writerStream.write(chunk, 'UTF8')
            })

            //stringifyStream onEnd
            stringifyStream.on('end', function() {
                writerStream.end()
            })

            //writerStream onFinish
            writerStream.on('finish', function() {
                console.log('writerStream finish')
                let res = fs.readFileSync(fp, 'utf8')
                fs.unlinkSync(fp)
                console.log('res.length', res.length)
                console.log('res', res.substr(0, 200) + '...')
                resolve()
            })

            //writerStream onError
            writerStream.on('error', function(err) {
                console.log('writerStream error', err)
            })

        })
    }

    async function testStringifys() {
        await testStringify()
        // writerStream finish
        // res.length 646894
        // res [{"key":"k1","value":{"a":123,"b":145.67,"c":"test中文1","d":true,"f":[11,"xyz",false,[166,197,215]],"g":{"ga":223,"gb":245.67,"gc":"test中文2","gd":[66,97,115]}}},{"key":"k2","value":{"a":123,"b":145.67,...
    }

    console.log('testStringify...')
    await testStringifys()
        .catch((err) => {
            console.log(err)
        })

    async function testParse(pre, filter) {
        return new Promise((resolve, reject) => {

            //fp
            let fp = './temp-write.json'

            //arr
            let arr = [
                {
                    key: 'k1',
                    value: {
                        a: 123,
                        b: 145.67,
                        c: 'test中文1',
                        d: true,
                        f: [11, 'xyz', false, [166, 197, 215]],
                        g: {
                            ga: 223,
                            gb: 245.67,
                            gc: 'test中文2',
                            gd: [66, 97, 115],
                        },
                    },
                },
                {
                    child: [
                        { name: 'child:abc', labels: ['child:label-abc'] },
                        { name: 'child:def', labels: ['child:label-def'] },
                        true,
                        false,
                        2020,
                        {
                            name: 'child:xyz',
                            query: {
                                labels: ['child:query:label-xyz']
                            }
                        },
                    ],
                },
                { name: 'abc', labels: ['label-abc'] },
                { name: 'def', labels: ['label-def'] },
                true,
                false,
                -1,
                {
                    name: 'xyz',
                    query: {
                        labels: ['query:label-xyz']
                    }
                },
            ]

            //jarr
            let jarr = JSON.stringify(arr)

            //writeFileSync
            fs.writeFileSync(fp, jarr, 'utf8')

            //createReadStream
            let readerStream = fs.createReadStream(fp)

            //createParseStream
            let parseStream = json.createParseStream(filter)

            //onData
            let res = []
            parseStream.on('data', function(chunk) {
                console.log(pre, 'parseStream data chunk=', chunk)
                res.push(chunk)
            })

            //onEnd
            parseStream.on('end', function() {
                console.log(pre, 'parseStream end res', res)
                fs.unlinkSync(fp)
                resolve()
            })

            //onError
            parseStream.on('error', function(err) {
                console.log(pre, 'parseStream error', err)
            })

            //pipe
            readerStream.pipe(parseStream)

        })
    }

    async function testParses() {

        let filter

        filter = '' //default
        console.log('test0', filter)
        await testParse('test0', filter)
        // test0
        // test0 parseStream data chunk= {
        //   value: {
        //     key: 'k1',
        //     value: {
        //       a: 123,
        //       b: 145.67,
        //       c: 'test中文1',
        //       d: true,
        //       f: [Array],
        //       g: [Object]
        //     }
        //   },
        //   key: 0
        // }
        // test0 parseStream data chunk= {
        //   value: { child: [ [Object], [Object], true, false, 2020, [Object] ] },
        //   key: 1
        // }
        // test0 parseStream data chunk= { value: { name: 'abc', labels: [ 'label-abc' ] }, key: 2 }
        // test0 parseStream data chunk= { value: { name: 'def', labels: [ 'label-def' ] }, key: 3 }
        // test0 parseStream data chunk= { value: true, key: 4 }
        // test0 parseStream data chunk= { value: false, key: 5 }
        // test0 parseStream data chunk= { value: -1, key: 6 }
        // test0 parseStream data chunk= { value: { name: 'xyz', query: { labels: [Array] } }, key: 7 }
        // test0 parseStream end res [
        //   { value: { key: 'k1', value: [Object] }, key: 0 },
        //   { value: { child: [Array] }, key: 1 },
        //   { value: { name: 'abc', labels: [Array] }, key: 2 },
        //   { value: { name: 'def', labels: [Array] }, key: 3 },
        //   { value: true, key: 4 },
        //   { value: false, key: 5 },
        //   { value: -1, key: 6 },
        //   { value: { name: 'xyz', query: [Object] }, key: 7 }
        // ]

        filter = '*.labels' //無emit, 只能監聽onData個別處理
        console.log('test1', filter)
        await testParse('test1', filter)
        // test1 *.labels
        // test1 parseStream data chunk= [ 'label-abc' ]
        // test1 parseStream data chunk= [ 'label-def' ]
        // test1 parseStream end res [ [ 'label-abc' ], [ 'label-def' ] ]

        filter = [true, 'labels', { emitPath: true }]
        console.log('test2', filter, 'true=*, 開啟emitPath才能塞回物件取得過濾後結果')
        await testParse('test2', filter)
        // test2 [ true, 'labels', { emitPath: true } ] true=*, 開啟emitPath才能塞回物件取得過濾後結果
        // test2 parseStream data chunk= { value: 'label-abc', path: [ 2, 'labels', 0 ] }
        // test2 parseStream data chunk= { value: 'label-def', path: [ 3, 'labels', 0 ] }
        // test2 parseStream end res [
        //   { value: 'label-abc', path: [ 2, 'labels', 0 ] },
        //   { value: 'label-def', path: [ 3, 'labels', 0 ] }
        // ]

        filter = [{ recurse: true }, 'labels', { emitPath: true }]
        console.log('test3', filter, '開啟recurse代表為任意子節點, 開啟emitPath才能塞回物件取得過濾後結果')
        await testParse('test3', filter)
        // test3 [ { recurse: true }, 'labels', { emitPath: true } ] 開啟recurse代表為任意子節點, 開啟emitPath才能塞回物件取得過濾後結果
        // test3 parseStream data chunk= { value: 'child:label-abc', path: [ 1, 'child', 0, 'labels', 0 ] }
        // test3 parseStream data chunk= { value: 'child:label-def', path: [ 1, 'child', 1, 'labels', 0 ] }
        // test3 parseStream data chunk= {
        // value: 'child:query:label-xyz',
        // path: [ 1, 'child', 5, 'query', 'labels', 0 ]
        // }
        // test3 parseStream data chunk= { value: 'label-abc', path: [ 2, 'labels', 0 ] }
        // test3 parseStream data chunk= { value: 'label-def', path: [ 3, 'labels', 0 ] }
        // test3 parseStream data chunk= { value: 'query:label-xyz', path: [ 7, 'query', 'labels', 0 ] }
        // test3 parseStream end res [
        // { value: 'child:label-abc', path: [ 1, 'child', 0, 'labels', 0 ] },
        // { value: 'child:label-def', path: [ 1, 'child', 1, 'labels', 0 ] },
        // {
        //     value: 'child:query:label-xyz',
        //     path: [ 1, 'child', 5, 'query', 'labels', 0 ]
        // },
        // { value: 'label-abc', path: [ 2, 'labels', 0 ] },
        // { value: 'label-def', path: [ 3, 'labels', 0 ] },
        // { value: 'query:label-xyz', path: [ 7, 'query', 'labels', 0 ] }
        // ]

    }

    console.log('testParse...')
    await testParses()
        .catch((err) => {
            console.log(err)
        })

}
testStream()
    .catch((err) => {
        console.log(err)
    })

In a browser(UMD module):

Note: w-json-stream does not dependent on any package.

[Necessary] Add script for w-json-stream.

<!-- for basic -->
<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/w-json-stream.umd.js"></script>

<!-- for web workers -->
<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/w-json-stream.wk.umd.js"></script>

stringify and parse for small data: ex-small.html [source code]

stringifyByStream and parseByStream for small data: ex-small-stream.html [source code]

stringify and parse for large data: ex-large.html [source code]

createParseStream: ex-stream-createParseStream.html [source code] * ReadableStream does not support IE11.

createParseStream with filter: ex-stream-createParseStreamWithFilter.html [source code]

createStringifyStream: ex-stream-createStringifyStream.html [source code] * WritableStream does not support IE11 and Firefox.

stringify and parse for large data in web worker: ex-large-webworker.html [source code] * WebWorkers(from blob) does not support IE11.

console.log('WebWorkers(from blob) does not support IE11')

let json = window['w-json-stream']
console.log(json)

console.log('build data...')

let n = 700 //7000, 9000
let lgRows = []
for (let i = 1; i <= 1000; i++) {
    lgRows.push([i, 23.4, '567', 'abc', true, false, [], {}, [8, 9.01], { a: 12.34, b: 'xyz', i }])
}
let carr = ''
let arr = []
for (let i = 1; i <= n; i++) {
    arr.push({
        key: 'k' + i,
        value: {
            a: 123,
            b: 145.67,
            c: 'test中文1',
            d: true,
            f: [11, 'xyz', false, [166, 197, 215]],
            g: {
                ga: 223,
                gb: 245.67,
                gc: 'test中文2',
                gd: [66, 97, 115],
                ge: JSON.parse(JSON.stringify(lgRows)),
            },
        }
    })
}

console.log('build data finish')

let t=0
let timer=setInterval(function(){
    console.log('t',t/10+'(s)')
    t++
    if(t>50){
        clearInterval(timer)
    }
},100)

let pmArr = json.stringify(arr) 
    .then(function(res){
        console.log('json.stringify(arr) then',res)
        carr = res
    })
    .catch(function(err){
        console.log('json.stringify(arr) catch',err)
    })

Promise.all([pmArr])
    .then(function(){

        json.parse(carr) 
            .then(function(r){
                console.log('json.parse(carr) then',r)
            })
            .catch(function(err){
                console.log('json.parse(carr) catch',err)
            })
        
    })
    .catch(function(err){
        console.log(err)
    })