Aggregate-fn NPM

aggregate-fn

Aggregate fn is a Javascript utility that groups together multiple async operations. It is really useful when you want to merge many API requests, avoiding rate-limits.

Features

This utility can:

aggregate requests up to a configured upper limit
aggregate requests up to a max configured delay
provide working metrics

const { fn, flush, cancel } = aggregateFn(aggregableFn, {
  maxWaitTime: 100,
  maxItems: 10,
  stats: console.log
})

aggregate-fn libray model

Documentation

APIs

Install

npm install aggregate-fn

Examples

Simple test

In the following code snippet multiple async operations are aggregated together. Full source code available here simple.js

import { aggregateFn } from 'aggregate-fn'
import { promisify } from 'util'
const sleep = promisify(setTimeout)

// this is a way to gather metrics
const statsPrinter = stats => console.log(`Stats: ${JSON.stringify(stats)}`)

// Here you can execute your async task (e.g. call one API)
const myAggregableAsyncFn = async requests => {
  const responses = requests.map(request => request * 2)
  return responses
}

// Init aggregateFn
const { fn, flush, cancel } = aggregateFn(myAggregableAsyncFn, {
  maxWaitTime: 200,
  maxItems: 2,
  stats: statsPrinter
})

//Executes multiple parallel async functions
Promise.all([
  (async function () {
    const res = await fn(1)
    console.log(`Request #1: ${res}`)
  }()),

  (async function () {
    const res = await fn(2)
    console.log(`Request #2: ${res}`)
  }()),

  (async function () {
    const res = await fn(3)
    console.log(`Request #3: ${res}`)
  }()),

  (async function () {
    await sleep(500)
    const res = await fn(4)
    console.log(`Request #4: ${res}`)
  }())
])

process.on('SIGINT', () => flush())
process.on('exit', () => cancel())

Calling a bulk service

In the following code snippet we reduce the number of requests toward a Spotify service, in order to have a better usage of Spotify's quota. Full source code available here: spotify.js

import { aggregateFn } from 'aggregate-fn'

// https://developer.spotify.com/documentation/general/guides/authorization/client-credentials/
const SPOTIFY_CLIENTID = process.env.SPOTIFY_CLIENTID
const SPOTIFY_SECRET = process.env.SPOTIFY_SECRET

// Init aggregateFn
const { fn: getArtist, flush, cancel } = aggregateFn(_getArtistBulk, {
  maxWaitTime: 1000,
  maxItems: 10,
  stats: stats => console.log(`Stats: ${JSON.stringify(stats)}`)
})

// Executes multiple parallels async functions
Promise.all([
  (async function () {
    const info = await getArtist('1dfeR4HaWDbWqFHLkxsg1d')
    console.log(`Request #1: ${info}`)
  }()),

  (async function () {
    const info = await getArtist('3fMbdgg4jU18AjLCKBhRSm')
    console.log(`Request #2: ${info}`)
  }())
])

process.on('SIGINT', () => flush())
process.on('exit', () => cancel())

// ** Spotify APIs ** //
async function _getArtistBulk (artistIds) {
  const token = await _getSpotifyToken(SPOTIFY_CLIENTID, SPOTIFY_SECRET)
  const req = await fetch('https://api.spotify.com/v1/artists?ids=' + artistIds.join(','), {
    headers: {
      Authorization: `Bearer ${token}`,
      'Content-Type': 'application/json'
    }
  })

  const json = await req.json()
  if (!req.ok || !Array.isArray(json.artists)) throw new Error('Unable to retrieve Spotify data. Check your CLIENT_ID and SECRET.')
  return json.artists.map(({ name }) => name)
}

async function _getSpotifyToken (clientid, secret) {
  const req = await fetch('https://accounts.spotify.com/api/token', {
    method: 'POST',
    headers: {
      Authorization: 'Basic ' + (Buffer.from(clientid + ':' + secret).toString('base64')),
      'content-type': 'application/x-www-form-urlencoded'
    },
    body: 'grant_type=client_credentials'
  })

  const json = await req.json()
  if (!req.ok || !json.access_token) throw new Error('Unable to retrieve Spotify token. Check your CLIENT_ID and SECRET.')
  return json.access_token
}