crawlee-proxyport

SDK > crawlee-proxyport

`crawlee-proxyport`

crawlee-proxyport Node.js TypeScript package provides easy way to use rotating proxy for Crawlee scraping framework.

npmjs.com/package/crawlee-proxyport - page on npmjs.com

github.com/proxyport/crawlee-proxyport - source code on github

Prerequisites

Installation

$ npm i crawlee-proxyport

Getting Started

Before you get your first proxy, you need to assign an API Key.

 import { CheerioCrawler, ProxyConfiguration } from 'crawlee';
import { ProxyProvider } from 'crawlee-proxyport';
 
const startUrls = ['https://example.com'];
 
const crawler: CheerioCrawler = new CheerioCrawler({
    useSessionPool: true,
    persistCookiesPerSession: true,
    proxyConfiguration: new ProxyConfiguration({
        newUrlFunction: (sId) => proxyProvider.newUrlFunction(sId)
    }),
    maxRequestRetries: 20,
    sessionPoolOptions:{
        sessionOptions: {
            // you may want to play with this number, we recommend to use value between 10 and 50
            maxUsageCount: 20,
        },
    },
    async requestHandler({ request, $, log }) {
        const title = $('title').text();
        log.info(`Title of ${request.loadedUrl} is '${title}'`);
    },
});
 
const proxyProvider = new ProxyProvider(<API_KEY>, crawler);
 
await crawler.run(startUrls);

	import { CheerioCrawler, ProxyConfiguration } from 'crawlee';
	import { ProxyProvider } from 'crawlee-proxyport';

	const startUrls = ['https://example.com'];

	const crawler: CheerioCrawler = new CheerioCrawler({
	useSessionPool: true,
	persistCookiesPerSession: true,
	proxyConfiguration: new ProxyConfiguration({
	newUrlFunction: (sId) => proxyProvider.newUrlFunction(sId)
	}),
	maxRequestRetries: 20,
	sessionPoolOptions:{
	sessionOptions: {
	// you may want to play with this number, we recommend to use value between 10 and 50
	maxUsageCount: 20,
	},
	},
	async requestHandler({ request, $, log }) {
	const title = $('title').text();
	log.info(`Title of ${request.loadedUrl} is '${title}'`);
	},
	});

	const proxyProvider = new ProxyProvider(<API_KEY>, crawler);

	await crawler.run(startUrls);