Rate Limit hit

// Attempt to scrape the shadcn-vue components from the website using firecrawl

import FirecrawlApp from '@mendable/firecrawl-js';
import fs from 'node:fs';
import 'dotenv/config'

const firecrawl = new FirecrawlApp({
    apiKey: process.env.FIRECRAWL_API_KEY,
    version: 'v1'
});

const vue_components_website = {
    "shadcn-vue": "https://www.shadcn-vue.com/docs/components/accordion.html",
    "primevue": "https://primevue.org/autocomplete/",
    "bootstrap-vue": "https://bootstrap-vue.org/docs/components/",
    "vuetify": "https://vuetifyjs.com/en/components/all/"
}

const delay = ms => new Promise(res => setTimeout(res, ms));

async function scrapeUIComponents() {
    try {
        console.log("Starting scraping...");

        // Create directory called components
        if (!fs.existsSync("components")) {
            fs.mkdirSync("components");
        }

        for (const [key, value] of Object.entries(vue_components_website)) {
            // Make a directory for each component according to the key
            if (!fs.existsSync(`components/${key}`)) {
                fs.mkdirSync(`components/${key}`);
            }

            // Scrape the website
            const result = await firecrawl.crawlUrl(value);

            // Write the result to a file
            fs.writeFileSync(`components/${key}/components.json`, JSON.stringify(result, null, 2));

            await delay(60000);
        }
    } catch (error) {
        console.error("An error has occurred:", error);
    }
}

scrapeUIComponents();

// Attempt to scrape the shadcn-vue components from the website using firecrawl

import FirecrawlApp from '@mendable/firecrawl-js';
import fs from 'node:fs';
import 'dotenv/config'

const firecrawl = new FirecrawlApp({
    apiKey: process.env.FIRECRAWL_API_KEY,
    version: 'v1'
});

const vue_components_website = {
    "shadcn-vue": "https://www.shadcn-vue.com/docs/components/accordion.html",
    "primevue": "https://primevue.org/autocomplete/",
    "bootstrap-vue": "https://bootstrap-vue.org/docs/components/",
    "vuetify": "https://vuetifyjs.com/en/components/all/"
}

const delay = ms => new Promise(res => setTimeout(res, ms));

async function scrapeUIComponents() {
    try {
        console.log("Starting scraping...");

        // Create directory called components
        if (!fs.existsSync("components")) {
            fs.mkdirSync("components");
        }

        for (const [key, value] of Object.entries(vue_components_website)) {
            // Make a directory for each component according to the key
            if (!fs.existsSync(`components/${key}`)) {
                fs.mkdirSync(`components/${key}`);
            }

            // Scrape the website
            const result = await firecrawl.crawlUrl(value);

            // Write the result to a file
            fs.writeFileSync(`components/${key}/components.json`, JSON.stringify(result, null, 2));

            await delay(60000);
        }
    } catch (error) {
        console.error("An error has occurred:", error);
    }
}

scrapeUIComponents();