N8n-nodes-test22-twitter NPM

run the database server run the live lookup server run n8n

Banner image

n8n-nodes-starter

This repo contains example nodes to help you get started building your own custom integrations for n8n. It includes the node linter and other dependencies.

To make your custom node available to the community, you must create it as an npm package, and submit it to the npm registry.

Prerequisites

You need the following installed on your development machine:

git
Node.js and npm. Minimum version Node 16. You can find instructions on how to install both using nvm (Node Version Manager) for Linux, Mac, and WSL here. For Windows users, refer to Microsoft's guide to Install NodeJS on Windows.
Install n8n with:
```
npm install n8n -g
```
Recommended: follow n8n's guide to set up your development environment.

Using this starter

These are the basic steps for working with the starter. For detailed guidance on creating and publishing nodes, refer to the documentation.

Generate a new repository from this template repository.

Clone your new repo:

git clone https://github.com/<your organization>/<your-repo-name>.git

Run npm i to install dependencies.
Open the project in your editor.
Browse the examples in /nodes and /credentials. Modify the examples, or replace them with your own nodes.
Update the package.json to match your details.
Run npm run lint to check for errors or npm run lintfix to automatically fix errors when possible.
Test your node locally. Refer to Run your node locally for guidance.
Replace this README with documentation for your node. Use the README_TEMPLATE to get started.
Update the LICENSE file to use your details.
Publish your package to npm.

More information

Refer to our documentation on creating nodes for detailed information on building your own nodes.

License

MIT

DB_HOST= DB_PORT= DB_NAME= DB_USER= DB_PASSWORD= INDEED_FIRST_CTK= INDEED_SECOND_CTK= INDEED_FIRST_SURF= INDEED_SECOND_SURF= INDEED_FIRST_PPID= INDEED_SECOND_PPID= INDEED_FIRST_CSRF= INDEED_SECOND_CSRF= INDEED_FIRST_JSESSIONID= INDEED_SECOND_JSESSIONID= INDEED_FIRST_CF_CLEARANCE= INDEED_SECOND_CF_CLEARANCE=

{ "name": "suhub-jobs", "module": "index.ts", "version": "1.0.0", "type": "module", "devDependencies": { "@types/axios": "^0.14.0", "@types/body-parser": "^1.19.5", "@types/bun": "latest", "@types/express": "^5.0.0", "@types/pg": "^8.11.10", "@types/request": "^2.48.12", "@types/request-promise-native": "^1.0.21", "@types/winston": "^2.4.4", "drizzle-kit": "^0.26.2", "ts-node": "^10.9.2", "tsx": "^4.19.1", "vitest": "^2.1.3" }, "peerDependencies": { "typescript": "^5.0.0" }, "dependencies": { "@types/cheerio": "^0.22.35", "@types/node-cron": "^3.0.11", "@types/puppeteer": "^7.0.4", "@types/swagger-ui-express": "^4.1.7", "@types/yamljs": "^0.2.34", "axios": "^1.7.7", "body-parser": "^1.20.3", "callsite": "^1.0.0", "cheerio": "^1.0.0", "csv": "^6.3.10", "dotenv": "^16.4.5", "drizzle-orm": "^0.35.3", "express": "^4.21.1", "express-validator": "^7.2.0", "fuzzball": "^2.1.3", "https-proxy-agent": "^7.0.5", "moment": "^2.30.1", "moment-timezone": "^0.5.46", "node-cron": "^3.0.3", "node-html-parser": "^6.1.13", "p-retry": "^6.2.0", "pg": "^8.13.0", "pino": "^9.5.0", "puppeteer": "^23.9.0", "puppeteer-extra": "^3.3.6", "puppeteer-extra-plugin-stealth": "^2.11.2", "request": "^2.88.2", "request-promise-native": "^1.0.9", "socks-proxy-agent": "^8.0.4", "swagger-ui-express": "^5.0.1", "tslog": "^4.9.3", "uuid": "^11.0.2", "vite": "^5.4.10", "winston": "^3.15.0", "winston-daily-rotate-file": "^5.0.0", "yamljs": "^0.3.0" }, "scripts": { "dev": "bun src/server.ts", "staging": "NODE_ENV=staging bun src/server.ts", "test": "vitest", "scrapeBayt": "bun src/scraping/baytScrape/scrape.ts", "scrapeLinkedin": "bun src/scraping/linkedinScrape/scrape.ts", "scrapeIndeed": "bun src/scraping/indeedScraping/scrape.ts", "scrapeNaukri": "bun src/scraping/naukriScrape/scrape.ts", "scrapeAll": "bun src/scraping/scrapeAll.ts", "csvSegmented": "bun src/csv_scripts/getSegmentedJobs.ts", "csvNotSegmented": "bun src/csv_scripts/getNotSegmentedJobs.ts", "csvDuplicates": "bun src/csv_scripts/getDuplicateJobs.ts", "segment": "bun src/segmentation/segmentationScript.ts" } }

import type { IndeedCookies } from "../../Types/Cookies";

const IndeedTestingCookies: Array = { CTK: process.env.INDEED_FIRST_CTK ?? "", SURF: process.env.INDEED_FIRST_SURF ?? "", PPID: process.env.INDEED_FIRST_PPID ?? "", CSRF: process.env.INDEED_FIRST_CSRF ?? "", cf_clearance: process.env.INDEED_FIRST_CF_CLEARANCE ?? "", JSESSIONID: process.env.INDEED_FIRST_JSESSIONID ?? "", }, { CTK: process.env.INDEED_SECOND_CTK ?? "", SURF: process.env.INDEED_SECOND_SURF ?? "", PPID: process.env.INDEED_SECOND_PPID ?? "", CSRF: process.env.INDEED_SECOND_CSRF ?? "", cf_clearance: process.env.INDEED_SECOND_CF_CLEARANCE ?? "", JSESSIONID: process.env.INDEED_SECOND_JSESSIONID ?? "", }, ;

export { IndeedTestingCookies };

import puppeteer from "puppeteer-extra"; import StealthPlugin from "puppeteer-extra-plugin-stealth";

import logger from "../../../../utils/helpers/logger"; import { IndeedTestingCookies } from "../Cookies/testingCookies.ts"; import type { IndeedCookies } from "../../Types/Cookies.ts"; import randomDelay from "../../../utils/helpers/randomDelay.ts"; import fs from "fs"; import path from "path";

interface PuppeteerCookie { name: string; value: string; domain: string; path?: string; expires?: number; httpOnly?: boolean; secure?: boolean; sameSite?: "Strict" | "Lax" | "None"; }

async function ensurePuppeteerConfig() { const configPath = path.join(process.cwd(), ".npmrc"); const configLine = "puppeteer_download_host=https://storage.googleapis.com";

if ( !fs.existsSync(configPath) || !fs.readFileSync(configPath, "utf8").includes(configLine) ) { fs.appendFileSync(configPath, \n${configLine}\n); } }

puppeteer.use(StealthPlugin());

async function getIndeedJobDetails( jobId: string | undefined, location: string | undefined, cookies: IndeedCookies ): Promise<string | null, boolean, string> { try { const browser: any = await puppeteer.launch({ headless: true, defaultViewport: null, args: "--no-sandbox", "--disable-setuid-sandbox", "--disable-infobars", "--window-position=0,0", "--ignore-certifcate-errors", "--ignore-certifcate-errors-spki-list", "--disable-blink-features=AutomationControlled", , }); const page = await browser.newPage();

const cookiesArray: PuppeteerCookie[] = [
  { name: "CTK", value: cookies.CTK, domain: `.indeed.com` },
  { name: "SURF", value: cookies.SURF, domain: `.indeed.com` },
  { name: "PPID", value: cookies.PPID, domain: `.indeed.com` },
  { name: "CSRF", value: cookies.CSRF, domain: `.indeed.com` },
  { name: "JSESSIONID", value: cookies.JSESSIONID, domain: `.indeed.com` },
  {
    name: "cf_clearance",
    value: cookies.cf_clearance,
    domain: `.indeed.com`,
  },
];

await page.setCookie(...cookiesArray);

await page.goto(`https://${location}.indeed.com/viewjob?jk=${jobId}`, {
  waitUntil: "domcontentloaded",
});

let elementFound = false;
for (let i = 0; i < 2; i++) {
  await randomDelay();
  try {
    const element = await page.$eval(
      "h1",
      (el) => el.textContent?.trim() === "Additional Verification Required"
    );
    elementFound = element;

    if (elementFound) {
      // await page.screenshot({ path: `screenshot-0${i}.png` }); // leave for debugging purposes
      await randomDelay();
      for (let l = 0; l < 9; l++) {
        await randomDelay();
        await page.keyboard.press("Tab");
      }
      await randomDelay();
      await page.keyboard.press("Space");
    }

    console.log(`Attempt ${i + 1} failed, retrying...`);
    // await page.screenshot({ path: `screenshot-${i}.png` }); // leave for debugging purposes
    await randomDelay();
    await randomDelay();
    await randomDelay();
    await randomDelay();
  } catch (error) {
    if (i === 2) {
      console.log("All attempts exhausted");
    } else {
      await randomDelay();
    }
  }
}
await randomDelay();

const jobDescription = await page.$("#jobDescriptionText");
const jobDescriptionMarkup = await page.evaluate(
  (el) => el?.innerHTML,
  jobDescription
);

const logoElement = await page.$(".jobsearch-JobInfoHeader-logo");
const logoUrl = await page.evaluate((el) => el?.src, logoElement);

const applyButton = await page.$('button[aria-label="Apply now "]');
const closed = !applyButton;

await browser.close();
return [jobDescriptionMarkup, closed, logoUrl];

} catch (error: any) { logger.error("Failed Getting Job Description: ", error); return null, false, ""; } }

export default getIndeedJobDetails;

export const waitThreeMinutes = (): Promise => { return new Promise((resolve) => { setTimeout(resolve, 45000); }); };

const randomDelay = (): Promise => new Promise((resolve) => setTimeout(resolve, Math.floor(Math.random() * 2500) + 500) );

export default randomDelay;

DB_HOST=46.101.147.190 DB_PORT=5432 DB_NAME=suhub_external_jobs_testing DB_USER=suhub DB_PASSWORD=suhub@external.123 INDEED_FIRST_CTK="1i9jm0vauj6uf800" INDEED_SECOND_CTK="1ibo39nehgcvs800" INDEED_FIRST_SURF="uf5fBVto2uL1fMMYph7rLIzrlbtRX3HL" INDEED_SECOND_SURF="3AA54fbFbuKqCHFP0KmRlD91dCwB8yES" INDEED_FIRST_PPID="eyJraWQiOiI2YTIyY2MxYi1mODc2LTQ0NDYtOWUwZC0xMGU5Y2ZmNjk2NTgiLCJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiJ9.eyJzdWIiOiI3ZDQxMDNiYWU3NDUzNWEzIiwibGFzdF9hdXRoX3RpbWUiOjE3MjgzMTEyMzk4MTIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJwaG9uZV9zY29wZSI6W10sImF1dGgiOiJnb29nbGUiLCJjcmVhdGVkIjoxNjM4Mjk4MzQxMDAwLCJpc3MiOiJodHRwczovL3NlY3VyZS5pbmRlZWQuY29tIiwibGFzdF9hdXRoX2xldmVsIjoiU1RST05HIiwibG9nX3RzIjoxNzI4MzExMjM5ODEyLCJhdWQiOiJjMWFiOGYwNGYiLCJwaG9uZV92ZXJpZmllZCI6dHJ1ZSwicmVtX21lIjp0cnVlLCJwaG9uZV9udW1iZXIiOiIrMjQ5OTExMDkwNDk4IiwiZXhwIjoxNzMyMjg0OTM3LCJpYXQiOjE3MzIyODMxMzcsImVtYWlsIjoiZmllbGRyNjJAZ21haWwuY29tIn0.AFTOHpjqanVfw1K90iC7I4dsYdd25l_k-kDvGxbExqMrhzpZEFnTGGCh7EPvZ9_Hw9czLx7mewWl4gMe-t7Ucw" INDEED_SECOND_PPID="eyJraWQiOiI2YTIyY2MxYi1mODc2LTQ0NDYtOWUwZC0xMGU5Y2ZmNjk2NTgiLCJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiJ9.eyJzdWIiOiI2MGIyMjkyYWRhZDA1YzZlIiwibGFzdF9hdXRoX3RpbWUiOjE3MzE4MjI3NTk3MjQsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJhdXRoIjoiZ29vZ2xlIiwiY3JlYXRlZCI6MTczMTgyMjc1OTAwMCwiaXNzIjoiaHR0cHM6Ly9zZWN1cmUuaW5kZWVkLmNvbSIsImxhc3RfYXV0aF9sZXZlbCI6IlNUUk9ORyIsImxvZ190cyI6MTczMTgyMjc1OTcyNCwiYXVkIjoiYzFhYjhmMDRmIiwicmVtX21lIjp0cnVlLCJleHAiOjE3MzE4MjQ1NTksImlhdCI6MTczMTgyMjc1OSwiZW1haWwiOiJtb2p0YWJhYWhtZWQxODA5QGdtYWlsLmNvbSJ9.v8XuXvKL6xW9BwzHeND942pQYKNpCx7A_xUad_FuKCA6uyTaMQVsotg9ycS4nJKFrslAjlH9MJgCZM73muUVvg" INDEED_FIRST_CSRF="10t7RMJYtEL0tnpJPBSVD1fn1LJaJRIy" INDEED_SECOND_CSRF="Dwjy05H6iixN6Y2AVmUnuDk4QPHhSxW6" INDEED_FIRST_JSESSIONID="8018A9DDDAC5BCC9D649F5C89DC8D11D" INDEED_SECOND_JSESSIONID="5634585A0268145093895C826CF3846F" INDEED_FIRST_CF_CLEARANCE="JSgPow4gGLCgoOUKM.kdj8enblPVK0sX2EMFO7bc8QU-1732283027-1.2.1.1-wmJ.8rGPJ21KdjKTT1YEriBwiXhtkKH.zjHfi8HFb8zIpk52t7hingROOcGSvXRAes4YFp63iREP9Ni2RNPi.1E95QEo76YD4mdVcfVxabrGLJqWVyOhnw.zP4cU9M3dq4j80LsE6lNv1YNR4pOuk9l1vKy6lkSFkgPE2Zcll620wWJ4HLieWZk9aOfz4nNrPE6Kx31ztolPnyYfWZdOcrP1FDX5qKS2SDPTyLpbc3yxsFISavNqSyhfHKaQSKvZlbyLqbarX876FAhz8HXJFYjRbpd8wg4yWAoxXZRUf8gxusmkm8sMZgOwM4BrVx1ZuYO1dntnfJtQDbTJHGiCMnZL3orzW6k7eXfMTeimDlHNgXvvZfq4q1w3KmozfeC5ON3Mc.YE7i6_dgs_SFW2B.Di9RY_DuMLsXvW.yQza_o" INDEED_SECOND_CF_CLEARANCE="SnneQ9y_S4twQcKhUTVbe8p4AhjQhs6F4oZGaB9L1tE-1731822636-1.2.1.1-hZcCmzAOcpgDasFeogXW06aN4.mIC8mycEZr1weUvZdH4oGkdHX_5khKsXCQbhfREImVZgtZ8YsQBgx7iq8C4WnsFlAGEK9e_3EWZn7oTPrXiLrmQjUqdYCHU2xUxq.jbVXvXq05sGFRFfxdI0Fr9eruiAUZ76sl_nSAn8q0A0nIlgCVNPt6uDCkR12FMZ.QDyr9.8cdHjkbR8wKhEHG.8tJ.ShJ5.SU.FIEcv.8TRGbd66BrIRJZSuqbd0CBF41iowt94UJFlY9s0N9xKdZ_FT9VsSO2iNGMCX4bvxLCSTohFJJh.lIFul53YOumLxFns5j1GWJg21QNOxn_mqb4pY0TqeBmElYy3D3mMAAb4pqTkKdwzttEtv7CXLAWH78hbkJ_xqMKiEKCaAuom4PDa9yQ10eW2m70koX4W10sF8"

n8n-community-node-package

cron dotenv instagram-node instagram-private-api mysql mysql2 puppeteer puppeteer-extra puppeteer-extra-plugin-stealth request-promise

0.1.0

7 months ago