feat: Added crawler template.

This commit is contained in:
Spencer Brower
2025-09-12 12:15:27 -04:00
parent 313051a9bd
commit a95cf11b08
8 changed files with 3921 additions and 0 deletions

View File

@@ -0,0 +1,28 @@
import { PlaywrightCrawler, Dataset } from 'crawlee';
async function main() {
const startUrls = [
'https://ipchicken.com',
];
await crawler.run(startUrls);
}
const crawler = new PlaywrightCrawler({
requestHandler: async ({ request, page, enqueueLinks, log }) => {
const basic = await page.locator('p[align=center] b').innerText();
const ip = basic.split('\n')[0]
log.info(`Your ip is: '${ip}'`);
},
// maxRequestsPerCrawl: 50,
launchContext: {
launchOptions: {
executablePath: process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH,
},
},
headless: true,
});
await main();