I am trying to extract some data from a html page using html rewriter: ``` var htmlRewriter = new H

I am trying to extract some data from a html page using html rewriter:

var htmlRewriter = new HTMLRewriter();

                var title = '';
                var description = '';
                var favicon = '';
                var ogImage = '';

                htmlRewriter.on('title', {
                    text: (text) => {
                        title = text;
                    }
                });

                htmlRewriter.on('meta[name="description"]', {
                    element: (element) => {
                        description = element.getAttribute('content');
                    }
                });

                htmlRewriter.on('link[rel="icon"]', {
                    element: (element) => {
                        favicon = element.getAttribute('href');
                    }
                });

                htmlRewriter.on('meta[property="og:image"]', {
                    element: (element) => {
                        ogImage = element.getAttribute('content');
                    }
                });

                var content_response = new Response(html, {
                    headers: {
                        'Content-Type': 'text/html',
                    },
                });

                await (await htmlRewriter.transform(content_response)).text();

                console.log('meta data', { title, description, favicon, ogImage });

                return { title, description, favicon, ogImage };

var htmlRewriter = new HTMLRewriter();

                var title = '';
                var description = '';
                var favicon = '';
                var ogImage = '';

                htmlRewriter.on('title', {
                    text: (text) => {
                        title = text;
                    }
                });

                htmlRewriter.on('meta[name="description"]', {
                    element: (element) => {
                        description = element.getAttribute('content');
                    }
                });

                htmlRewriter.on('link[rel="icon"]', {
                    element: (element) => {
                        favicon = element.getAttribute('href');
                    }
                });

                htmlRewriter.on('meta[property="og:image"]', {
                    element: (element) => {
                        ogImage = element.getAttribute('content');
                    }
                });

                var content_response = new Response(html, {
                    headers: {
                        'Content-Type': 'text/html',
                    },
                });

                await (await htmlRewriter.transform(content_response)).text();

                console.log('meta data', { title, description, favicon, ogImage });

                return { title, description, favicon, ogImage };

var htmlRewriter = new HTMLRewriter();

                var title = '';
                var description = '';
                var favicon = '';
                var ogImage = '';

                htmlRewriter.on('title', {
                    text: (text) => {
                        title = text;
                    }
                });

                htmlRewriter.on('meta[name="description"]', {
                    element: (element) => {
                        description = element.getAttribute('content');
                    }
                });

                htmlRewriter.on('link[rel="icon"]', {
                    element: (element) => {
                        favicon = element.getAttribute('href');
                    }
                });

                htmlRewriter.on('meta[property="og:image"]', {
                    element: (element) => {
                        ogImage = element.getAttribute('content');
                    }
                });

                var content_response = new Response(html, {
                    headers: {
                        'Content-Type': 'text/html',
                    },
                });

                await (await htmlRewriter.transform(content_response)).text();

                console.log('meta data', { title, description, favicon, ogImage });

                return { title, description, favicon, ogImage };

var htmlRewriter = new HTMLRewriter();

                var title = '';
                var description = '';
                var favicon = '';
                var ogImage = '';

                htmlRewriter.on('title', {
                    text: (text) => {
                        title = text;
                    }
                });

                htmlRewriter.on('meta[name="description"]', {
                    element: (element) => {
                        description = element.getAttribute('content');
                    }
                });

                htmlRewriter.on('link[rel="icon"]', {
                    element: (element) => {
                        favicon = element.getAttribute('href');
                    }
                });

                htmlRewriter.on('meta[property="og:image"]', {
                    element: (element) => {
                        ogImage = element.getAttribute('content');
                    }
                });

                var content_response = new Response(html, {
                    headers: {
                        'Content-Type': 'text/html',
                    },
                });

                await (await htmlRewriter.transform(content_response)).text();

                console.log('meta data', { title, description, favicon, ogImage });

                return { title, description, favicon, ogImage };

I keep geting the error: An RPC stub was not disposed properly. You must call dispose() on all stubs in order to let the other side know that you are no longer using them. You cannot rely on the garbage collector for this because it may take arbitrarily long before actually collecting unreachable objects. As a shortcut, calling dispose() on the result of an RPC call disposes all stubs within it.

AdvanyOP•10/29/24, 12:00 PM

btw, this is inside a step, if I remove wait .text(), it works but I don't get the response...

AAdvany I am trying to extract some data from a html page using html rewriter: ``` var ...

AdvanyOP•10/29/24, 12:14 PM

rewrote it using cheerio

makrdev•10/29/24, 1:06 PM

'create' method doesn't return the 'id' of created instance. Is this expected?

const {id} = await WORKFLOW.create();
console.log(id) // RpcProperty {}

const {id} = await WORKFLOW.create();
console.log(id) // RpcProperty {}

This issue exist in local env. It works on prod.

Jani•10/29/24, 1:43 PM

I’ve been testing out long-running workflows and wanted to share some observations:

Testing persistence outside of an execution loop: my main goal was to see if workflows keep their state properly when running in a loop (storing a reference of a step result outside of a loop) with longer sleep intervals (around 30 minutes). This seemed to work - state reference was not lost even when it most definitely hibernated during those long waits.
Switching versions mid-sleep: during the second sleep cycle, I deployed a new version of the workflow. Surprisingly, it moved the execution to the new version instead of sticking with the original one and even disregarded the current active sleep cycle. Shouldn’t it use the version it was invoked with? (source)
Dashboard wall time: the workflow mostly sat in sleep mode, but the dashboard showed a total wall time of almost 64 minutes, which doesn’t seem right given the actual activity.

Adding more details here in a thread.

Mmakrdev 'create' method doesn't return the 'id' of created instance. Is this expected? ...

Diogo Ferreira•10/29/24, 1:45 PM

Yes, this is a known issue on local env. We are working on a fix for it

Gyurmatag•10/29/24, 4:44 PM

Hey guys!
I want to get all my meetings from Google calendar API from 2022. I want to fetch them week by week. For example like this:

for (const { weekStart, weekEnd } of weeklyIntervals) {
      const events = await step.do(
        `fetchMeetings for ${weekStart} - ${weekEnd}`,
        { retries: { limit: 3, delay: 2000 } },
        async () => {
          const response = await fetch(
            `https://www.googleapis.com/calendar/v3/calendars/primary/events?timeMin=${weekStart}&timeMax=${weekEnd}&orderBy=startTime&singleEvents=true`,
            {
              headers: { Authorization: `Bearer ${accessToken}` },
            }
          );

          if (!response.ok) throw new Error("Failed to fetch events");
          const data = await response.json();
          return data.items.map((event: { summary: string }) => event.summary);
        }
      );

      // Log summaries for this week
      events.forEach((summary: string, index: number) =>
        console.log(`Event ${index + 1} on week ${weekStart} - ${weekEnd}: ${summary}`)
      );
    }

for (const { weekStart, weekEnd } of weeklyIntervals) {
      const events = await step.do(
        `fetchMeetings for ${weekStart} - ${weekEnd}`,
        { retries: { limit: 3, delay: 2000 } },
        async () => {
          const response = await fetch(
            `https://www.googleapis.com/calendar/v3/calendars/primary/events?timeMin=${weekStart}&timeMax=${weekEnd}&orderBy=startTime&singleEvents=true`,
            {
              headers: { Authorization: `Bearer ${accessToken}` },
            }
          );

          if (!response.ok) throw new Error("Failed to fetch events");
          const data = await response.json();
          return data.items.map((event: { summary: string }) => event.summary);
        }
      );

      // Log summaries for this week
      events.forEach((summary: string, index: number) =>
        console.log(`Event ${index + 1} on week ${weekStart} - ${weekEnd}: ${summary}`)
      );
    }

Will this solution work? Should I include the loop inside the step? (I am in free tier) Does the paid plan have unlimited steps?

marbemac•10/29/24, 5:21 PM

Man what a solid foundation you guys have put together with the workflows beta! One questions so far:

Any creative ways you can think of to handle the "human in the loop" pattern with workflows? Basically where at some point in the workflow we want to "sleep" until a human has taken some sort of action (approve / deny, etc). Inngest manages this with their

waitForEvent

waitForEvent

function (https://www.inngest.com/docs/features/inngest-functions/steps-workflows/wait-for-event), which I know CF doesn't have, but wondering if there are other creative ways that are not too hacky to support this kind of functionality with CF.

Mmarbemac Man what a solid foundation you guys have put together with the workflows beta! ...

Hello, I’m Allie!•10/29/24, 5:45 PM

Break it into two separate Workflows maybe? Not entirely sure on the logistics there(would probably depend on your setup)

Mmarbemac Man what a solid foundation you guys have put together with the workflows beta! ...

Diogo Ferreira•10/29/24, 5:52 PM

For the moment, we do not support

waitForEvent

waitForEvent

steps or events as entrypoint (focus on the "for the moment"

). One way you could achieve the intended behaviour, would be store the humam action in a KV or D1, and periodically check for that action, sleeping between checks

DDiogo Ferreira For the moment, we do not support `waitForEvent` steps or events as entrypoint (...

nclevenger•10/29/24, 5:56 PM

I was just experimenting with this myself - I was about to try opening a websocket to the Slack API in Socket Mode and try to wrap the onmessageonmessage handler with a promise that could be awaited in the

step

step

https://api.slack.com/apis/socket-mode

Slack API

Socket Mode

How to use Socket Mode with SDKs.

GGyurmatag Hey guys! I want to get all my meetings from Google calendar API from 2022. I wa...

Diogo Ferreira•10/29/24, 5:57 PM

That approach should work. The for loop should be outside the step, that way you will benefit from the persistancy of previous successful step executions, sparing on the API calls.

Recommendation. Give it a spin!

DDiogo Ferreira That approach should work. The for loop should be outside the step, that way you...

Gyurmatag•10/29/24, 7:18 PM

Okay, Thank you! But What about the maximum 256 steps? I will definitely have more than that. Does the paid tier have limits regarding the steps?

i40west•10/29/24, 9:17 PM

What is the function passed to

step.do

step.do

supposed to return? If I return anything at all from it, it causes the workflow to throw an exception after finishing successfully, every time, and I can't figure out what I'm doing wrong

zuegno•10/30/24, 3:59 AM

Hello, I'm really excited to try Workflows. I tried to add a Workflow to a Remix application. I used the Remix example Framework guide in the Pages docs. Got this error on wrangler deploy:

[ERROR] Your Worker depends on the following Workflows, which are not exported in your entrypoint file: DemgEventWorkflow.

  You should export these objects from your entrypoint, build/worker/index.js

[ERROR] Your Worker depends on the following Workflows, which are not exported in your entrypoint file: DemgEventWorkflow.

  You should export these objects from your entrypoint, build/worker/index.js

Any ideas on how to fix are appreciated!

zuegno•10/30/24, 4:01 AM

My worker is built in a functions/[[path]].tsfunctions/[[path]].ts file that looks like:

import { createPagesFunctionHandler } from "@remix-run/cloudflare-pages";

// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore - the server build file is generated by `remix vite:build`
// eslint-disable-next-line import/no-unresolved
import * as build from "../build/server";

export const onRequest = createPagesFunctionHandler({ build });

import { createPagesFunctionHandler } from "@remix-run/cloudflare-pages";

// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore - the server build file is generated by `remix vite:build`
// eslint-disable-next-line import/no-unresolved
import * as build from "../build/server";

export const onRequest = createPagesFunctionHandler({ build });

zuegno•10/30/24, 4:20 AM

Project was started with https://developers.cloudflare.com/workers/frameworks/framework-guides/remix/

Cloudflare Docs

Remix | Cloudflare Workers docs

Create a Remix application and deploy it to Cloudflare Workers with Workers Assets.

DiamondDragon•10/30/24, 6:22 AM

noticing if you have a long output in the trace online it will go off screen (copy button disappears)

DiamondDragon•10/30/24, 6:23 AM

also, maybe I missed it, but I haven't figured out how to run a workflow that's been modified locally, as opposed to the one already deployed. Is this where we're at with the beta or did i miss how to?

Zzuegno Hello, I'm really excited to try Workflows. I tried to add a Workflow to a Remix...

Cayter•10/30/24, 7:25 AM

Remix with Vite currently relies on Wrangler's getPlatformProxy() that uses miniflare under the hood. This setup means Remix is running on Node.js and accessing the Env through HTTP getPlatformProxy().
By following this PR, you can leverage more features specifically designed for Cloudflare Workers, remix with cloudflare pages will have bumpy journey ahead
Run the following commands to set up your development environment:
- remix vite:devremix vite:dev on port 3000
- wrangler dev --define "process.env.NODE_ENV:'production'" --log-level=info --port=3001 --test-scheduled app/worker.tswrangler dev --define "process.env.NODE_ENV:'production'" --log-level=info --port=3001 --test-scheduled app/worker.ts

Note: worker.tsworker.ts should be ESM-based for Cloudflare Workers.

wrangler dev

wrangler dev

enables you to develop queues, crons, and workflows within workerdworkerd environment instead of NodeJS.

the error you're encountering is because getPlatformProxy()getPlatformProxy() doesn’t work with workflows. If you have

[[workflows]]

[[workflows]]

in wrangler.toml, Miniflare will refer to it, resulting in the error

the development experience for a Vite-powered app has limitations, especially with newer features. This should improve once Vite v6 is released, and Wrangler is updated to support it

Zzuegno Hello, I'm really excited to try Workflows. I tried to add a Workflow to a Remix...

Matt Silverlock•10/30/24, 1:53 PM

You will need to deploy a Worker with your Workflow; you can’t bundle it into your Pages (Remix) app.

DiamondDragon•10/30/24, 4:12 PM

I'm getting a Type instantiation is excessively deep and possibly infinite.Type instantiation is excessively deep and possibly infinite. in a step. I don't think it's a very complex type being returned

DDiamondDragon noticing if you have a long output in the trace online it will go off screen (co...

Diogo Ferreira•10/30/24, 5:24 PM

Thanks for reporting it!

Avi•10/30/24, 11:24 PM

is there support for scheduling a workflow to run periodically, like a cron job?

Avi•10/30/24, 11:24 PM

or would I use a worker with a cron trigger to initiate a workflow?

Avi•10/30/24, 11:24 PM

oh the docs address this, yes, nvm

Avi•10/30/24, 11:28 PM

ok in that case

Jacob•10/30/24, 11:28 PM

If you want to run it more often than once a minute, you could have your workflow schedule its next run rather than an external trigger. Would just need to jump-start it initially

Avi•10/30/24, 11:28 PM

what does the workflow concurrency limit mean? does that mean that if I have 5k scheduled, 100 will be initally scheduled to run, and as they conplete,m ore will be scheduled?

Avi•10/30/24, 11:28 PM

basically 100 max parallelism?

JJacob If you want to run it more often than once a minute, you could have your workflo...

Avi•10/30/24, 11:29 PM

my usecase is that i'd like to run code every night at midnight, and id love to NOT have to create a separate cloudflare worker with a

scheduled

scheduled

handler for this

Avi•10/30/24, 11:29 PM

just one instance of that code

Avi•10/30/24, 11:29 PM

essentially a global singleton

Jacob•10/30/24, 11:29 PM

The same Worker can export both the scheduled handler and the Workflow

Jacob•10/30/24, 11:30 PM

So it can all be in one .ts file

JJacob The same Worker can export both the scheduled handler and the Workflow

Avi•10/30/24, 11:30 PM

yeah, but that worker is actually just a DO stub wrapper

Jacob•10/30/24, 11:31 PM

wdym?

Avi•10/30/24, 11:31 PM

i guess we could do it there though

JJacob wdym?

Avi•10/30/24, 11:31 PM

i mean the "worker" is literally just the minimum boilerplate/wrapper to wrap a DO stub

Avi•10/30/24, 11:31 PM

but that doesnt mean we cant add a cron scheduled handler to it

Jacob•10/30/24, 11:33 PM

Yeah I’d just put the scheduled handler in the same worker that you’re wanting to trigger Workflows from

Jacob•10/30/24, 11:39 PM

Another trick I’ve found with crons (in general) that hit external APIs is to not schedule right at the top of the hour - some APIs I’ve used can actually slow down a bit because of everyone’s crons hitting them at the exact same time