@flowblade/source-duckdb v0.8.5

@flowblade/source-duckdb

Install

yarn add @flowblade/source-duckdb @flowblade/core @duckdb/node-api

Note that at this time @duckdb/node-api is still in alpha. To install use the latest tag, ie: @duckdb/node-api@1.3.0-alpha.21

Query the database

import { DuckdbDatasource, sql } from '@flowblade/source-duckdb';

// See setup below
import { ds } from "./config.ts";

const params = {
    min: 10,
    max: 99,
    name: 'test',
    createdAt: new Date().toISOString(),
};

type Row = { id: number; name: 'test'; createdAt: Date };

const rawSql = sql<Row>`

      WITH products(productId, createdAt)
          AS MATERIALIZED (
               FROM RANGE(1,100) SELECT 
               range::INT,
               TIMESTAMPTZ '2025-01-01 12:30:00.123456789+01:00'
          )
      
      SELECT productId, 
             ${params.name} as name,
             createdAt
             
      FROM products 
      WHERE productId BETWEEN ${params.min}::INTEGER AND ${params.max}::INTEGER
      AND createdAt < ${params.createdAt}::TIMESTAMPTZ
    `;

const result = await ds.query(rawSql);

const { data, meta, error } = result;

if (data) {
    // Typed as Row[]
    console.log(data);
}
if (error) {
    // Typed as QError
    console.log(error);
}

// Optionally: map over the data to transform it
const { data: mappedData } = result.map((row) => {
    return {
        id: row.productId,
        key: `key-${row.productId}`
    }
});

Create a duckdb instance

import os from 'node:os';

import { type DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';
import { DuckdbDatasource } from '@flowblade/source-duckdb';

// Create a connection to a DuckDB instance
const createConnection = async (
  maxThreads = 4
): Promise<DuckDBConnection> => {
  const availableThreads = os.availableParallelism();
  const maxParallelism = Math.min(maxThreads, availableThreads - 1);
  const threads = availableThreads > 1 ? maxParallelism : undefined;

  const instance = await DuckDBInstance.create(':memory:', {
    // Choose between READ_ONLY or READ_WRITE
    // Note that in READ_WRITE mode concurrency is limited to 1
    // See: https://duckdb.org/docs/connect/concurrency.html
    access_mode: 'READ_WRITE',
    max_memory: '64MB',
    // Using more threads may require additional memory
    ...(threads ? { threads: threads.toString(10) } : {}),
  });
  return await instance.connect();
};

const duckdb = await createConnection();

// Create a duckdb datasource
export const ds = new DuckdbDatasource({ connection: duckdb });

Compatibility

Contributors

Contributions are welcome. Have a look to the CONTRIBUTING document.

Sponsors

Sponsor, coffee, or star – All is spent for quality time with loved ones. Thanks ! 🙏❤️

Special thanks to

License

MIT © Sébastien Vanvelthem and contributors.