Langbase

Website Crawler

This example demonstrates how to crawl a website using Langbase website crawler tool.

Website Crawler Example

Website Crawler Example

import 'dotenv/config';
import {Langbase} from 'langbase';

const langbase = new Langbase({
	apiKey: process.env.LANGBASE_API_KEY!,
});

/**
 * Crawls specified URLs using spider.cloud service.
 *
 * Get your API key from the following link and set it in .env file.
 *
 * @link https://spider.cloud/docs/quickstart
 */
async function main() {
	const results = await langbase.tools.crawl({
		url: ['https://langbase.com', 'https://langbase.com/about'],
		maxPages: 1,
		apiKey: process.env.CRAWL_KEY,
	});

	console.log(results);
}

main();