Concurrent fetch calls

const fetchSitemap = () =>
  Effect.tryPromise(() =>
    client.get(url, params, { headers }).then((response) => response.data)
  );

const parseSitemap = (data: string) =>
  Effect.tryPromise(() => parseStringPromise(data));

const fetchUrl = (url: string) =>
  Effect.tryPromise(() => {
    console.log(`Fetching URL: ${url}`);
    return client.get(url, params, { headers });
  });

const processUrlsWithConcurrency = (urls: string[], concurrencyLimit: number) =>
  Effect.gen(function* () {
    yield* Effect.all(
      urls.map(
        (u) =>
          fetchUrl(u).pipe(
            Effect.map(() => true),
            Effect.catchAll((error) =>
              Effect.logError(`Error fetching URL ${url}: ${error.message}`)
            )
          ),
        { concurrency: concurrencyLimit }
      )
    );
  });

const program = Effect.gen(function* () {
  // Fetch the sitemap
  yield* Effect.logInfo("Fetching sitemap...");
  const sitemapData = yield* fetchSitemap();

  // Parse the sitemap
  const parsedSitemap = yield* parseSitemap(sitemapData as string);
  yield* Effect.logInfo("Parsed sitemap");
  // Extract URLs
  const urls = parsedSitemap.urlset.url.map(
    (entry: any) => entry.loc[0] as string
  );

  // Fetch each URL with concurrency limit
  yield* processUrlsWithConcurrency(urls, 10);
});

Effect.runPromise(program).then(() => Effect.logInfo("All tasks completed"));