Multi-tasks NPM

multi-tasks

Install:

npm install multi-tasks

How to use:

//see examples/example0
let multiTasks = require('multi-tasks').multiTasks;


//Step1, If you have tasks that need to be executed simultaneously, please provide them as an array, multi-tasks will automatically split them and execute.
let alltasks = [];
for(let i=0;i<50;i++){
    alltasks.push({
        name: `task-${i}`,
        data: `This prop is for a subtask`
    });
};

//Step2, Provide a function to process a certain sub-task and return the result data
let processTask = (task, helper)=>{
    let {taskCount} = task;

    return 'a result which is not a promise'; 
};

//Step2.1, or return a promise
let processTask = (task, helper)=>{
    let {taskCount} = task;

    return new Promise((resolve, reject)=>{
        resolve({
            data:`task${taskCount} complete`
        })
    })
};

//Step2.2, dynamically create a new task if find something new while processing
let processTask = (task, helper)=>{
    let {taskCount} = task;

    
    if(taskCount % 2 === 0){
        helper.createNewTasks({//You can dynamically create a new task if find something new while processing
            msg:'a new task'
        });
        return;
    }

    return new Promise((resolve, reject)=>{
        resolve({
            data:`task${taskCount} complete`
        })
    })
};

//Step2.3, this example shows how to handle exceptions/errors
let processTask = (task, helper)=>{
    let {taskCount} = task;

    //This is the demo of exceptions/errors, they will be captured and saved in the results/errors folder
    if(taskCount===3) throw 'exception';
    if(taskCount===4) return Promise.reject({err:'a test error'});
    if(taskCount===5) aaa =  bbb;

    if(taskCount===6) return 'a result which is not a promise'; //Return a non-promise result is OK

    //by default, you should return a promise, all returned data can be found in the results/succ folder
    //but to return a non-promise result is also OK, see above
    return new Promise((resolve, reject)=>{
        setTimeout(()=>{
            if(taskCount % 2 === 0){
                helper.createNewTasks({//You can dynamically create a new task if find something new while processing
                    msg:'a new task'
                });
                resolve()
            }else{
                resolve({
                    data:`task${taskCount} complete`
                })
            }
        }, 10)
    })
}

//Step3, run!
multiTasks.start({
    tasks: alltasks, 
    processTask, 
    taskRootFolder: `../examples-tmp-data/example0`, //a directory to store progress and results files, you can check the progress here
    taskId: 'my-task',
    numberOfWorkers: 3, //Assign how many workers are working in parallel
    //autoCloseAfterCompletion: true, //if you have dynamically generated new tasks, put this as false
    shouldTerminate,//return true if you need to terminate the whole process
    onFinish: (report)=>{
        console.log('finish callback', report);
    }
});

Have a try:

node examples/example0/run
node examples/example1/run

Changelog:

2.0.2 new feature: support shouldTerminate
2.0.1 Avoid possible I/O conflicts.
2.0.0 Rewritten with a new architecture to support dynamic tasks.
1.2.8 Fix: create task folder failed on MacOS
1.2.7 Small updates
1.2.6 Support onFinish event
1.2.5 Rename numberOfWorks to numberOfWorkers, the old one are still supported ;-)
1.2.4 Fix: opt.numberOfWorkers not work
1.2.3 Update README
1.2.2 Update README and examples
1.2.1 Handle exceptions and errors in subtasks
1.2.0 Simplified usage by providing the function way and support return Promise
1.1.4 Remove make-dir
1.1.3 Simplified usage, see example0
1.1.2
1.1.1 Rename files, updated changelog
1.1.0 Simplified the usage of a customized Consumer, see example0
1.0.8 Fix examples
1.0.7 Remove moment
1.0.6 Performance optimization