4.1.1 • Published 3 years ago

bigquery-streamer v4.1.1

Weekly downloads
39
License
ISC
Repository
github
Last release
3 years ago

Bigquery Streamer

This is a streamer to transport Redis records to Google BigQuery using stream insert mode.

Features

  • Fast, lightweight
  • Scalable, multiple streamers can play well with each others
  • Auto daily table/dataset creation, supports schema, ttl
  • Feedback based throttling
  • Basic schema validation
  • Archive to Google Cloud Storage if payload size exceeds limits
  • Auto correct invalid records
  • Failure handlings with exponential retries
  • Compatible with node.js >= 12

Get Started

Install
npm install bigquery-streamer --save
Start Streamer
var Streamer = require('bigquery-streamer');

var config = {...};
var streamer = new Streamer(config);
streamer.start();

Config

{
	"gcloud": {
		"projectId": "PROJECT_ID",
		"credentials": {
			"client_email": "CLIENT_EMAIL",
			"private_key": "PRIVATE_KEY"
		},
		"autoRetry": true,
		"maxRetries": 10
	},
	"redis": {
		"port": 6379,
		"host": "localhost",
		"db": 1
	},
	"streamer": {
		"max_row_size": 16000,
		"dataset_namespace": "development",
		"table_name_format": "YYYYMMDD_KEY",
		"key_prefix": "logs",
		"scan_interval": 5000,
		"send_batch_size": 100,
		"senders": 64,
		"enable_stat": true,
		"stat_interval": 10000,
		"table_definitions": {
			"general": {
				"v2": {
					"fields": [
						{
							"name": "name",
							"type": "STRING"
						},
						{
							"name": "hostname",
							"type": "STRING"
						},
						{
							"name": "pid",
							"type": "INTEGER"
						},
						{
							"name": "level",
							"type": "INTEGER"
						},
						{
							"name": "msg",
							"type": "STRING"
						},
						{
							"name": "time",
							"type": "TIMESTAMP"
						},
						{
							"name": "v",
							"type": "INTEGER"
						},
						{
							"name": "cargo",
							"type": "STRING"
						},
						{
							"name": "err",
							"type": "RECORD",
							"fields": [
								{
									"name": "name",
									"type": "STRING"
								},
								{
									"name": "message",
									"type": "STRING"
								},
								{
									"name": "stack",
									"type": "STRING"
								}
							]
						}
					],
					"strippable_fields": [
						"msg",
						"cargo"
					],
					"ttl": 100
				}
			}
		}
	}
}

Debug

Debug-logger is used. With environment variable DEBUG=*, all debug info will direct to stdout.

4.1.1

3 years ago

4.1.0

4 years ago

4.0.0

4 years ago

3.2.5

4 years ago

3.2.4

4 years ago

3.2.3

4 years ago

3.2.2

6 years ago

3.2.1

6 years ago

3.2.0

8 years ago

3.1.3

8 years ago

3.1.2

8 years ago

3.1.1

8 years ago

3.1.0

8 years ago

3.0.0

8 years ago

1.1.3

9 years ago

1.1.2

9 years ago

1.1.1

9 years ago

1.1.0

9 years ago