@cruisecritic/s3-multipart-uploader v1.0.5

S3 Multipart Uploader

Wrapper around AWS SDKs v3 S3 multipart uploader. This tries to simplify the process. It doesn't allow for much customization but should "just work."

Process adapted from https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpu-upload-object.html

Example

const s3Uploader = new S3MultipartUploader<CsvRow>(s3Client, bucket, filename, {
    transformer: stringify, // Transformer to transform data into a string needed for file
    checkInterval: 10000, // How many rows should be processed before we check calculated file size.
    logger, // If you pass a logger instance this job will add debug logging for what it's doing
});

// We want to catch any error so that we can abort the upload
try {
    // Start the upload process
    await s3Uploader.start();

    // Optional: Add header row
    s3Uploader.add({
        field1: field1,
        field2: field2,
        ...
    });

    let result;
    do {
        result = // get results
        result.forEach(({ field1, field2 }) => {
            // Add each row of the result
            s3Uploader.add({
                field1,
                field2,
                ...
            });
        });
    } while (result.length);

    // Finish processing data. This will wait for all uploads to finish and tell s3 the upload is finished
    await s3Uploader.finish();
} catch (e) {
    if (e instanceof Error) {
        logger.error('Error uploading', e);
    }
    // Tell s3 to abort the upload 
    await s3Uploader.abort();
}