Crawl is not respecting limit crawl option
I set a limit of 500 for my crawl but I find that it keeps crawling beyond 500 pages and I have to interrupt it on my end. My code is below. Am I doing something wrong?
app = FirecrawlApp(api_key=FIRECRAWL_API_KEY)
start_time = time.time()
crawl_url = "https://www.lsu.edu/majors"
params = {
"crawlerOptions": {
"limit": 500,
"maxDepth": 2,
"ignoreSitemap": False,
"ignoreRobots": False,
},
"pageOptions": {
"onlyMainContent": True,
"parsePDF": True,
"removeTags": ["script", "style", "nav", "header", "footer",
".advertisement", ".sidebar", ".nav", ".menu",
"#comments", "img", "svg", "iframe", "video",
"audio"]
},
}
urls = []
job_id = app.crawl_url(crawl_url, params=params, wait_until_done=False)
app = FirecrawlApp(api_key=FIRECRAWL_API_KEY)
start_time = time.time()
crawl_url = "https://www.lsu.edu/majors"
params = {
"crawlerOptions": {
"limit": 500,
"maxDepth": 2,
"ignoreSitemap": False,
"ignoreRobots": False,
},
"pageOptions": {
"onlyMainContent": True,
"parsePDF": True,
"removeTags": ["script", "style", "nav", "header", "footer",
".advertisement", ".sidebar", ".nav", ".menu",
"#comments", "img", "svg", "iframe", "video",
"audio"]
},
}
urls = []
job_id = app.crawl_url(crawl_url, params=params, wait_until_done=False)
