gatsby-cdn-search-plugin v0.0.39

Gatsby cdn search plugin

This plugin is in beta and still in work

Give us any feedback, open issues for any questions or ideas

Mongo query compatible search plugin for gatsby.

It is a no cost way to add search to your site.

Key technology is http2/http3 protocols, CDN, mongo-like query and N-GRAM search.

The plugin supports mongo-like query syntax with custom n-gram search.

Idea of this plugin is simple.

• calculated indices in build phase of gatsby apps. (n-gram, text-lex, simple)
• split the indices by chunk
• create range diapason indices as "table of contents" the chunk
• save all chunk and "table of contents" on CDN as assets
• in runtime plugin restore indices and efficiently on-demand loaded chunk over http2 protocol
• http2 multiplexing multiple requests over a single TCP connection.

The plugin has native support React via Hook "useCdnCursorQuery".

Also, you can trace your request with log-level.

import { useCdnCursorQuery, log } from 'gatsby-cdn-search-plugin'
log.enableAll(); // full logging 

Live demo

Kaggle dataset "Used Car Auction Prices" 500 000 row

live demo

source code on github

source of data

Plugins config

    plugins = [
    'gatsby-plugin-offline',
    {
      resolve: require.resolve("./cdn-indice-plugin"),
      options: {
        id: 'cars',
        chunkSize: 6000,
        dataChunkSize: 60,
        indices: [  
          {
              id: 'model',
              column: 'model',
              type: 'simple', 
          },
          { id: 'make', column: 'make' },
          { id: 'year', column: 'year' },
          { id: 'state', column: 'state' },
          { 
              id: 'id-state',
              column: 'state',
              algoritm: 'english',
              type: 'text-lex' 
          },
          { 
              id: 'ngram',
              type: "n-gram",
              actuationLimit: 1,
              actuationLimitAuto: false, 
              gramLen: 3, 
              toLowcase: true, 
              algoritm: 'english',
              stopWords: ["and"],
              columns: ['model', 'make', 'color'] 
          }
        ],
        idAttr: 'id',
        normalizer: ({ data }) => { 
          return data.recentCars
            .map(( {id, ...node} ) => ({ id: id.replace('Car__',''), ...node }));
        },
        graphQL: `query MyQuery { 
          recentCars(cursor: 0, limit: 500000){
              id
              color
              make
              mmr
              model
              seller
              sellingprice
              state
              transmission
              trim
              vin
              year
        }
      }`
      }
    }
    ]

Usage stateful React hook

import { useCdnCursorStatelessQuery, log } from 'gatsby-cdn-search-plugin'
log.enableAll(); // full logging 


const makeQuery = (search) => { // Different query strategy. It is depends of length search word
  if (search.length >= 4) {
    return { $ngram: search }; // n-gram 
  } else if (!!search.length) {
    return {
      $or: [
        { model: { $regex: new RegExp(`^${search}`, 'i'), }, }, // regexp by two columns
        { make: { $regex: new RegExp(`^${search}`, 'i'), }, }
      ],
    };
  } else {
    return undefined;
  }
}
const [state, dispatch] = useReducer(reducer, initialState);

const query = useMemo(() => makeQuery(state.search), [state.search]);

const {hasNext, next, fetching, all, page} = useCdnCursorStatefulQuery('cars', query, {year: 1}, 0, 30); // hook return cursor of data

const load = useMemo(() => {
    if (hasNext && !fetching) {
          next(); // load next slice of data 
    }}, [hasNext, fetching, next]);

Usage stateless React hook (more complicated)

import { useCdnCursorStatelessQuery, log } from 'gatsby-cdn-search-plugin'
log.enableAll(); // full logging 

const initialState = {
    loading: false,
    search: '',
    list: [],
    page: 0,
};

function reducer(state, action) {
    switch (action.type) {
        case 'pageUp':
            return { ...state, page: state.page + 1 }
        case 'type':
            return { ...state, search: action.value }
        case 'loading':
            return { ...state, loading: true }
        case 'load':
            return { ...state, loading: false, list: action.list, page: 0 };
        case 'indice':
            return { ...state, indice: action.value }
        case 'loadMore':
            return { ...state, loading: false, list: [...state.list, ...action.list] };
        default:
            throw new Error();
    }
}

const makeQuery = (search) => { // Different query strategy. It is depends of length search word
  if (search.length >= 4) {
    return { $ngram: search, year: { $lte: 2014 } }; // n-gram and  year <= 2014
  } else if (!!search.length) {
    return {
      $or: [
        { model: { $regex: new RegExp(`^${search}`, 'i'), }, }, // regexp by two columns
        { make: { $regex: new RegExp(`^${search}`, 'i'), }, }
      ],
    };
  } else {
    return { year: { $lte: 2014 } }; // only date predicate
  }
}
const [state, dispatch] = useReducer(reducer, initialState);

const query = useMemo(() => makeQuery(state.search), [state.search]);

const cursor = useCdnCursorQuery('cars', query, {year: 1}, 0, 30); // hook return cursor of data

  useEffect(() => {
    (async () => {
        let list = await cursor.next(); // load first slice of data
        dispatch({ type: 'load', list });
    })();
  }, [state.search, cursor])

  useEffect(() => {
    (async () => {
      if (await cursor.hasNext()) {
        let list = await cursor.next(); // load next slice of data 
        dispatch({ type: 'loadMore', list })
      }
    })()
  }, [state.page]);

Usage find api exactly

      import { restoreDb } from 'gatsby-cdn-search-plugin'

      const db = await restoreDb('cars');
      let result;
      if (search.length >= 4) {
        result = await db.find({ $ngram: search, year: { $gte: 2014 } }, undefined, 0, offset);
      } else if (!!search.length) {
        result = await db.find({ model: { $regex: new RegExp(`^${search}`, 'i'), }, year: { $gte: 2014 } }, undefined, 0, offset);
      } else {
        result = await db.find({ year: { $gte: 2014 } }, undefined, 0, offset);
      }

Usage cursor api exactly

      import { restoreDb } from 'gatsby-cdn-search-plugin'

      let cursor;
      const searchFetch = async (search, skip = 0, limit = 30) => {
        const db = await restoreDb('cars');
        if(cursor){
          cursor.finish();
        }
        if (search.length >= 4) {
          cursor =  db.cursor({ $ngram: search, year: { $gte: 2014 } }, undefined, skip, limit);
        } else if (!!search.length) {
          cursor =  db.cursor({
            $or: [
              { model: { $regex: new RegExp(`^${search}`, 'i'), }, },
              { make: { $regex: new RegExp(`^${search}`, 'i'), }, }
            ],
          }, undefined, skip, limit);
        } else {
          cursor = db.cursor({ year: { $gte: 2014 } }, undefined, skip, limit);
        }
        return await cursor.next();
      }

Plugin options

NgramIndicesOption

SimpleIndicesOption

TextLexIndicesOption