diff --git a/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts b/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts index e062fb4c43d..52ff991605f 100644 --- a/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts +++ b/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts @@ -6,6 +6,7 @@ import { parse } from 'css-what' import { SelectorType } from 'cheerio' import { ICommonObject, INodeOutputsValue, IDocument, INode, INodeData, INodeParams } from '../../../src/Interface' import { handleEscapeCharacters, webCrawl, xmlScrape } from '../../../src/utils' +import { checkDenyList } from '../../../src/httpSecurity' class Cheerio_DocumentLoaders implements INode { label: string @@ -148,6 +149,7 @@ class Cheerio_DocumentLoaders implements INode { async function cheerioLoader(url: string): Promise { try { + await checkDenyList(url) let docs: IDocument[] = [] if (url.endsWith('.pdf')) { if (process.env.DEBUG === 'true') diff --git a/packages/components/nodes/documentloaders/Playwright/Playwright.ts b/packages/components/nodes/documentloaders/Playwright/Playwright.ts index fcef1faddf3..1a268b38844 100644 --- a/packages/components/nodes/documentloaders/Playwright/Playwright.ts +++ b/packages/components/nodes/documentloaders/Playwright/Playwright.ts @@ -10,6 +10,7 @@ import { test } from 'linkifyjs' import { omit } from 'lodash' import { handleEscapeCharacters, INodeOutputsValue, webCrawl, xmlScrape } from '../../../src' import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' +import { checkDenyList } from '../../../src/httpSecurity' class Playwright_DocumentLoaders implements INode { label: string @@ -189,6 +190,7 @@ class Playwright_DocumentLoaders implements INode { async function playwrightLoader(url: string): Promise { try { + await checkDenyList(url) let docs = [] const executablePath = process.env.PLAYWRIGHT_EXECUTABLE_PATH diff --git a/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts b/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts index c22b05de94f..2f12b7dece6 100644 --- a/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts +++ b/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts @@ -6,6 +6,7 @@ import { omit } from 'lodash' import { PuppeteerLifeCycleEvent } from 'puppeteer' import { handleEscapeCharacters, INodeOutputsValue, webCrawl, xmlScrape } from '../../../src' import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' +import { checkDenyList } from '../../../src/httpSecurity' class Puppeteer_DocumentLoaders implements INode { label: string @@ -180,6 +181,7 @@ class Puppeteer_DocumentLoaders implements INode { async function puppeteerLoader(url: string): Promise { try { + await checkDenyList(url) let docs: Document[] = [] const executablePath = process.env.PUPPETEER_EXECUTABLE_PATH