CD
Cloudflare Developers•7d ago
Gab

aiSearch does not vectorize files with Custom customMetadata inside the R2

Problem at: aiSearch does not vectorize files with Custom customMetadata inside the R2, I test without the customMetadata and it worked well, when I try to start sync, it shows an "unknown error at the files with metadata) Is customMetada allowed at r2 files inside aiSearch? Why is this happening? code snnipet that im using to scrape and upload the files inside my R2 
app.get("/site", async (c: AppContext) => {
    const firecrawl = new Firecrawl({apiKey: c.env.FIRECRAWL_API_KEY, apiUrl: "https://crawler.mysite.dev"})
    const crawlResponse = await firecrawl.crawl(
        "https://docs.example.org",
        {
            sitemap: "include",
            limit: 3,
            timeout: 10,
            scrapeOptions: {
                formats: ["markdown"]
            }
        }
    )

    await Promise.all(
        (crawlResponse.data ?? []).map((item) => {
            const fileName = item.metadata?.title
                ?.replace(/[^a-z0-9]/gi, "_")
                || `untitled_${Date.now()}`

            return c.env.R2.put(
                `jsonata/${fileName}_${crypto.randomUUID()}.md`,
                item.markdown ?? "",

                {
                    httpMetadata: { contentType: "text/markdown" }, 
                    customMetadata: { // IF I COMMENT THIS PARAM OBJECT, THE AISEACH WORKERS WELL
                        ...Object.fromEntries(
                            Object.entries(item.metadata ?? {}).map(([k, v]) => [k, String(v)])
                        ),
                        crawledAt: new Date().toISOString(),
                    },
                }
            )
        })
    )
    return c.json(crawlResponse)
})
app.get("/site", async (c: AppContext) => {
    const firecrawl = new Firecrawl({apiKey: c.env.FIRECRAWL_API_KEY, apiUrl: "https://crawler.mysite.dev"})
    const crawlResponse = await firecrawl.crawl(
        "https://docs.example.org",
        {
            sitemap: "include",
            limit: 3,
            timeout: 10,
            scrapeOptions: {
                formats: ["markdown"]
            }
        }
    )

    await Promise.all(
        (crawlResponse.data ?? []).map((item) => {
            const fileName = item.metadata?.title
                ?.replace(/[^a-z0-9]/gi, "_")
                || `untitled_${Date.now()}`

            return c.env.R2.put(
                `jsonata/${fileName}_${crypto.randomUUID()}.md`,
                item.markdown ?? "",

                {
                    httpMetadata: { contentType: "text/markdown" }, 
                    customMetadata: { // IF I COMMENT THIS PARAM OBJECT, THE AISEACH WORKERS WELL
                        ...Object.fromEntries(
                            Object.entries(item.metadata ?? {}).map(([k, v]) => [k, String(v)])
                        ),
                        crawledAt: new Date().toISOString(),
                    },
                }
            )
        })
    )
    return c.json(crawlResponse)
})
No description
0 Replies
No replies yetBe the first to reply to this messageJoin

Did you find this page helpful?