@diotoborg/assumenda-rem-iure NPM

@diotoborg/assumenda-rem-iure

A tool for JSON parse and stringify by stream and web worker.

Documentation

To view documentation or get support, visit docs.

Installation

Using npm(ES6 module):

Note: @diotoborg/assumenda-rem-iure is mainly dependent on from2 and jsonparse.
npm i @diotoborg/assumenda-rem-iure

Example for samll data:

Link: [dev source code]

async function testSmall() {

let obj = {
    a: 123,
    b: 145.67,
    c: 'test中文1',
    d: true,
    f: [11, 'xyz', false, [166, 197, 215]],
    g: {
        ga: 223,
        gb: 245.67,
        gc: 'test中文2',
        gd: [66, 97, 115],
    },
}
let arr = [
    {
        a: 123,
        b: 145.67,
        c: 'test中文1',
        d: true,
    },
    {
        f: [11, 'xyz', false, [166, 197, 215]],
        g: {
            ga: 223,
            gb: 245.67,
            gc: 'test中文2',
            gd: [66, 97, 115],
        },
    },
]

let jobj = await json.stringify(obj)
console.log(jobj)
// => {"a":123,"b":145.67,"c":"test中文1","d":true,"f":[11,"xyz",false,[166,197,215]],"g":{"ga":223,"gb":245.67,"gc":"test中文2","gd":[66,97,115]}}

let jarr = await json.stringify(arr)
console.log(jarr)
// => [{"a":123,"b":145.67,"c":"test中文1","d":true},{"f":[11,"xyz",false,[166,197,215]],"g":{"ga":223,"gb":245.67,"gc":"test中文2","gd":[66,97,115]}}]

let robj = await json.parse(jobj)
console.log(robj)
// => {
//     a: 123,
//     b: 145.67,
//     c: 'test中文1',
//     d: true,
//     f: [ 11, 'xyz', false, [ 166, 197, 215 ] ],
//     g: { ga: 223, gb: 245.67, gc: 'test中文2', gd: [ 66, 97, 115 ] }
// }

let rarr = await json.parse(jarr)
console.log(rarr)
// => [
//     { a: 123, b: 145.67, c: 'test中文1', d: true },
//     {
//       f: [ 11, 'xyz', false, [Array] ],
//       g: { ga: 223, gb: 245.67, gc: 'test中文2', gd: [Array] }
//     }
// ]

} testSmall() .catch((err) => { console.log(err) })

#### Example for large data:
> **Link:** [[dev source code](https://github.com/diotoborg/assumenda-rem-iure/blob/master/g-large.mjs)]
```alias

async function testLarge() {

    let n = 700 //7000, 9000
    let lgRows = []
    for (let i = 1; i <= 1000; i++) {
        lgRows.push([i, 23.4, '567', 'abc', true, false, [], {}, [8, 9.01], { a: 12.34, b: 'xyz', i }])
    }
    let lgArr = []
    for (let i = 1; i <= n; i++) {
        lgArr.push({
            key: 'k' + i,
            value: {
                a: 123,
                b: 145.67,
                c: 'test中文1',
                d: true,
                f: [11, 'xyz', false, [166, 197, 215]],
                g: {
                    ga: 223,
                    gb: 245.67,
                    gc: 'test中文2',
                    gd: [66, 97, 115],
                    ge: JSON.parse(JSON.stringify(lgRows)),
                },
            }
        })
    }

    if (true) {

        try {
            let res = JSON.stringify(lgArr)
            console.log('JSON.stringify(lgArr)', res.length, res.substr(0, 200) + '...')
        }
        catch (err) {
            console.log('JSON.stringify(lgArr) catch', err)
            // n=7000 => catch RangeError: Invalid string length => Out Of Memory
            // n=9000 => FATAL ERROR: MarkCompactCollector: young object promotion failed Allocation failed - JavaScript heap out of memory
        }

    }

    if (true) {

        await json.stringify(lgArr)
            .then((res) => {
                console.log('json.stringify(lgArr) then', res.length, res.substr(0, 200) + '...')
            })
            .catch((err) => {
                console.log('json.stringify(lgArr) catch', err)
                // n=7000 => FATAL ERROR: Ineffective mark-compacts near heap limit Allocation failed - JavaScript heap out of memory
            })

    }

}
testLarge()
    .catch((err) => {
        console.log(err)
    })

Example for stream:

Link: [dev source code]

async function testStream() {

async function testStringify() {
    return new Promise((resolve, reject) => {

        //fp
        let fp = './temp-stringify.json'

        //arr
        let n = 4000 //4000000
        let arr = []
        for (let i = 1; i <= n; i++) {
            arr.push({
                key: 'k' + i,
                value: {
                    a: 123,
                    b: 145.67,
                    c: 'test中文1',
                    d: true,
                    f: [11, 'xyz', false, [166, 197, 215]],
                    g: {
                        ga: 223,
                        gb: 245.67,
                        gc: 'test中文2',
                        gd: [66, 97, 115],
                    },
                }
            })
        }

        //createStringifyStream
        let stringifyStream = json.createStringifyStream(arr)

        //createWriteStream
        let writerStream = fs.createWriteStream(fp)

        // //pipe
        // stringifyStream.pipe(writerStream)

        //stringifyStream onData
        stringifyStream.on('data', function(chunk) {
            writerStream.write(chunk, 'UTF8')
        })

        //stringifyStream onEnd
        stringifyStream.on('end', function() {
            writerStream.end()
        })

        //writerStream onFinish
        writerStream.on('finish', function() {
            console.log('writerStream finish')
            let res = fs.readFileSync(fp, 'utf8')
            fs.unlinkSync(fp)
            console.log('res.length', res.length)
            console.log('res', res.substr(0, 200) + '...')
            resolve()
        })

        //writerStream onError
        writerStream.on('error', function(err) {
            console.log('writerStream error', err)
        })

    })
}

async function testStringifys() {
    await testStringify()
    // writerStream finish
    // res.length 646894
    // res [{"key":"k1","value":{"a":123,"b":145.67,"c":"test中文1","d":true,"f":[11,"xyz",false,[166,197,215]],"g":{"ga":223,"gb":245.67,"gc":"test中文2","gd":[66,97,115]}}},{"key":"k2","value":{"a":123,"b":145.67,...
}

console.log('testStringify...')
await testStringifys()
    .catch((err) => {
        console.log(err)
    })

async function testParse(pre, filter) {
    return new Promise((resolve, reject) => {

        //fp
        let fp = './temp-write.json'

        //arr
        let arr = [
            {
                key: 'k1',
                value: {
                    a: 123,
                    b: 145.67,
                    c: 'test中文1',
                    d: true,
                    f: [11, 'xyz', false, [166, 197, 215]],
                    g: {
                        ga: 223,
                        gb: 245.67,
                        gc: 'test中文2',
                        gd: [66, 97, 115],
                    },
                },
            },
            {
                child: [
                    { name: 'child:abc', labels: ['child:label-abc'] },
                    { name: 'child:def', labels: ['child:label-def'] },
                    true,
                    false,
                    2020,
                    {
                        name: 'child:xyz',
                        query: {
                            labels: ['child:query:label-xyz']
                        }
                    },
                ],
            },
            { name: 'abc', labels: ['label-abc'] },
            { name: 'def', labels: ['label-def'] },
            true,
            false,
            -1,
            {
                name: 'xyz',
                query: {
                    labels: ['query:label-xyz']
                }
            },
        ]

        //jarr
        let jarr = JSON.stringify(arr)

        //writeFileSync
        fs.writeFileSync(fp, jarr, 'utf8')

        //createReadStream
        let readerStream = fs.createReadStream(fp)

        //createParseStream
        let parseStream = json.createParseStream(filter)

        //onData
        let res = []
        parseStream.on('data', function(chunk) {
            console.log(pre, 'parseStream data chunk=', chunk)
            res.push(chunk)
        })

        //onEnd
        parseStream.on('end', function() {
            console.log(pre, 'parseStream end res', res)
            fs.unlinkSync(fp)
            resolve()
        })

        //onError
        parseStream.on('error', function(err) {
            console.log(pre, 'parseStream error', err)
        })

        //pipe
        readerStream.pipe(parseStream)

    })
}

async function testParses() {

    let filter

    filter = '' //default
    console.log('test0', filter)
    await testParse('test0', filter)
    // test0
    // test0 parseStream data chunk= {
    //   value: {
    //     key: 'k1',
    //     value: {
    //       a: 123,
    //       b: 145.67,
    //       c: 'test中文1',
    //       d: true,
    //       f: [Array],
    //       g: [Object]
    //     }
    //   },
    //   key: 0
    // }
    // test0 parseStream data chunk= {
    //   value: { child: [ [Object], [Object], true, false, 2020, [Object] ] },
    //   key: 1
    // }
    // test0 parseStream data chunk= { value: { name: 'abc', labels: [ 'label-abc' ] }, key: 2 }
    // test0 parseStream data chunk= { value: { name: 'def', labels: [ 'label-def' ] }, key: 3 }
    // test0 parseStream data chunk= { value: true, key: 4 }
    // test0 parseStream data chunk= { value: false, key: 5 }
    // test0 parseStream data chunk= { value: -1, key: 6 }
    // test0 parseStream data chunk= { value: { name: 'xyz', query: { labels: [Array] } }, key: 7 }
    // test0 parseStream end res [
    //   { value: { key: 'k1', value: [Object] }, key: 0 },
    //   { value: { child: [Array] }, key: 1 },
    //   { value: { name: 'abc', labels: [Array] }, key: 2 },
    //   { value: { name: 'def', labels: [Array] }, key: 3 },
    //   { value: true, key: 4 },
    //   { value: false, key: 5 },
    //   { value: -1, key: 6 },
    //   { value: { name: 'xyz', query: [Object] }, key: 7 }
    // ]

    filter = '*.labels' //無emit, 只能監聽onData個別處理
    console.log('test1', filter)
    await testParse('test1', filter)
    // test1 *.labels
    // test1 parseStream data chunk= [ 'label-abc' ]
    // test1 parseStream data chunk= [ 'label-def' ]
    // test1 parseStream end res [ [ 'label-abc' ], [ 'label-def' ] ]

    filter = [true, 'labels', { emitPath: true }]
    console.log('test2', filter, 'true=*, 開啟emitPath才能塞回物件取得過濾後結果')
    await testParse('test2', filter)
    // test2 [ true, 'labels', { emitPath: true } ] true=*, 開啟emitPath才能塞回物件取得過濾後結果
    // test2 parseStream data chunk= { value: 'label-abc', path: [ 2, 'labels', 0 ] }
    // test2 parseStream data chunk= { value: 'label-def', path: [ 3, 'labels', 0 ] }
    // test2 parseStream end res [
    //   { value: 'label-abc', path: [ 2, 'labels', 0 ] },
    //   { value: 'label-def', path: [ 3, 'labels', 0 ] }
    // ]

    filter = [{ recurse: true }, 'labels', { emitPath: true }]
    console.log('test3', filter, '開啟recurse代表為任意子節點, 開啟emitPath才能塞回物件取得過濾後結果')
    await testParse('test3', filter)
    // test3 [ { recurse: true }, 'labels', { emitPath: true } ] 開啟recurse代表為任意子節點, 開啟emitPath才能塞回物件取得過濾後結果
    // test3 parseStream data chunk= { value: 'child:label-abc', path: [ 1, 'child', 0, 'labels', 0 ] }
    // test3 parseStream data chunk= { value: 'child:label-def', path: [ 1, 'child', 1, 'labels', 0 ] }
    // test3 parseStream data chunk= {
    // value: 'child:query:label-xyz',
    // path: [ 1, 'child', 5, 'query', 'labels', 0 ]
    // }
    // test3 parseStream data chunk= { value: 'label-abc', path: [ 2, 'labels', 0 ] }
    // test3 parseStream data chunk= { value: 'label-def', path: [ 3, 'labels', 0 ] }
    // test3 parseStream data chunk= { value: 'query:label-xyz', path: [ 7, 'query', 'labels', 0 ] }
    // test3 parseStream end res [
    // { value: 'child:label-abc', path: [ 1, 'child', 0, 'labels', 0 ] },
    // { value: 'child:label-def', path: [ 1, 'child', 1, 'labels', 0 ] },
    // {
    //     value: 'child:query:label-xyz',
    //     path: [ 1, 'child', 5, 'query', 'labels', 0 ]
    // },
    // { value: 'label-abc', path: [ 2, 'labels', 0 ] },
    // { value: 'label-def', path: [ 3, 'labels', 0 ] },
    // { value: 'query:label-xyz', path: [ 7, 'query', 'labels', 0 ] }
    // ]

}

console.log('testParse...')
await testParses()
    .catch((err) => {
        console.log(err)
    })

} testStream() .catch((err) => { console.log(err) })

### In a browser(UMD module):
> **Note:** @diotoborg/assumenda-rem-iure does not dependent on any package.

[Necessary] Add script for @diotoborg/assumenda-rem-iure.
```alias

<!-- for basic -->
<script src="https://cdn.jsdelivr.net/npm/@diotoborg/assumenda-rem-iure@1.0.21/dist/@diotoborg/assumenda-rem-iure.umd.js"></script>

<!-- for web workers -->
<script src="https://cdn.jsdelivr.net/npm/@diotoborg/assumenda-rem-iure@1.0.21/dist/@diotoborg/assumenda-rem-iure.wk.umd.js"></script>

stringify and parse for small data: ex-small.html [source code]
stringifyByStream and parseByStream for small data: ex-small-stream.html [source code]
stringify and parse for large data: ex-large.html [source code]
createParseStream: ex-stream-createParseStream.html [source code] * ReadableStream does not support IE11.
createParseStream with filter: ex-stream-createParseStreamWithFilter.html [source code]
createStringifyStream: ex-stream-createStringifyStream.html [source code] * WritableStream does not support IE11 and Firefox.
stringify and parse for large data in web worker: ex-large-webworker.html [source code] * WebWorkers(from blob) does not support IE11.

console.log('WebWorkers(from blob) does not support IE11')

let json = window'@diotoborg/assumenda-rem-iure' console.log(json)

console.log('build data...')

let n = 700 //7000, 9000 let lgRows = [] for (let i = 1; i <= 1000; i++) { lgRows.push([i, 23.4, '567', 'abc', true, false, [], {}, 8, 9.01, { a: 12.34, b: 'xyz', i }]) } let carr = '' let arr = [] for (let i = 1; i <= n; i++) { arr.push({ key: 'k' + i, value: { a: 123, b: 145.67, c: 'test中文1', d: true, f: [11, 'xyz', false, 166, 197, 215], g: { ga: 223, gb: 245.67, gc: 'test中文2', gd: 66, 97, 115, ge: JSON.parse(JSON.stringify(lgRows)), }, } }) }

console.log('build data finish')

let t=0 let timer=setInterval(function(){ console.log('t',t/10+'(s)') t++ if(t>50){ clearInterval(timer) } },100)

let pmArr = json.stringify(arr) .then(function(res){ console.log('json.stringify(arr) then',res) carr = res }) .catch(function(err){ console.log('json.stringify(arr) catch',err) })