Production workers 100% errors

Gagan Suie · 2024-04-02T20:32:43.953Z

Randomly started getting 500 errors in production env workers `{ "outcome": "exception", "scriptName": "mage-api", "diagnosticsChannelEvents": [], "exceptions": [ { "name": "Error", "message": "The script will never generate a response.", "timestamp": 1712089582194 } ],`

J

James•4/2/24, 9:30 PM

Did you change anything? What does your worker do? Do you have a minimal reproduction?

G

Gagan SuieOP•4/2/24, 9:36 PM

nothing was changed

G

Gagan SuieOP•4/2/24, 9:36 PM

i've tried reverting to previous deployments and redeploying as well

G

Gagan SuieOP•4/2/24, 9:37 PM

what do you mean minimal reproduction? to reproduce the error, just visit our website and any api trigered will give that response

G

Gagan SuieOP•4/2/24, 9:37 PM

https://mage.stream

Mage - A collaborative streaming platform

Mage is a collaborative streaming platform for gamers and developers

J

James•4/2/24, 9:40 PM

I mean some code that others can run - a minimal example - to reproduce the issue

J

James•4/2/24, 9:40 PM

There's not been any widespread reports I'm aware of, so likely going to require some debug and digging into where your worker is failing

G

Gagan SuieOP•4/2/24, 10:14 PM

i finally figured out the error. it was related to packages. I reverted packages from a few days ago and its working now.
Now i need to investigate which package was causing issues.......

G

Gagan SuieOP•4/2/24, 10:14 PM

these are the working packages:

 "dependencies": {
        "@cloudflare/itty-router-openapi": "^1.0.6",
        "@miniflare/d1": "^2.14.1",
        "@tsndr/cloudflare-worker-jwt": "^2.2.1",
        "itty-router": "^4.0.11",
        "moment": "^2.29.4",
        "stripe": "^14.1.0",
        "tslib": "^2.5.0",
        "worker-auth-providers": "^0.0.13"
    },
    "devDependencies": {
        "@cloudflare/workers-types": "^4.20240208.0",
        "@types/jest": "^29.5.0",
        "@typescript-eslint/eslint-plugin": "7.3.1",
        "@typescript-eslint/parser": "7.2.0",
        "esbuild": "^0.20.0",
        "eslint": "^8.37.0",
        "eslint-config-prettier": "^9.0.0",
        "eslint-config-typescript": "^3.0.0",
        "jest": "^29.5.0",
        "jest-environment-miniflare": "^2.13.0",
        "miniflare": "^3.0.0",
        "prettier": "3.2.5",
        "ts-jest": "^29.1.0",
        "ts-node": "^10.9.1",
        "typescript": "^5.0.3",
        "wrangler": "^3.23.0"
    }

 "dependencies": {
        "@cloudflare/itty-router-openapi": "^1.0.6",
        "@miniflare/d1": "^2.14.1",
        "@tsndr/cloudflare-worker-jwt": "^2.2.1",
        "itty-router": "^4.0.11",
        "moment": "^2.29.4",
        "stripe": "^14.1.0",
        "tslib": "^2.5.0",
        "worker-auth-providers": "^0.0.13"
    },
    "devDependencies": {
        "@cloudflare/workers-types": "^4.20240208.0",
        "@types/jest": "^29.5.0",
        "@typescript-eslint/eslint-plugin": "7.3.1",
        "@typescript-eslint/parser": "7.2.0",
        "esbuild": "^0.20.0",
        "eslint": "^8.37.0",
        "eslint-config-prettier": "^9.0.0",
        "eslint-config-typescript": "^3.0.0",
        "jest": "^29.5.0",
        "jest-environment-miniflare": "^2.13.0",
        "miniflare": "^3.0.0",
        "prettier": "3.2.5",
        "ts-jest": "^29.1.0",
        "ts-node": "^10.9.1",
        "typescript": "^5.0.3",
        "wrangler": "^3.23.0"
    }

 "dependencies": {
        "@cloudflare/itty-router-openapi": "^1.0.6",
        "@miniflare/d1": "^2.14.1",
        "@tsndr/cloudflare-worker-jwt": "^2.2.1",
        "itty-router": "^4.0.11",
        "moment": "^2.29.4",
        "stripe": "^14.1.0",
        "tslib": "^2.5.0",
        "worker-auth-providers": "^0.0.13"
    },
    "devDependencies": {
        "@cloudflare/workers-types": "^4.20240208.0",
        "@types/jest": "^29.5.0",
        "@typescript-eslint/eslint-plugin": "7.3.1",
        "@typescript-eslint/parser": "7.2.0",
        "esbuild": "^0.20.0",
        "eslint": "^8.37.0",
        "eslint-config-prettier": "^9.0.0",
        "eslint-config-typescript": "^3.0.0",
        "jest": "^29.5.0",
        "jest-environment-miniflare": "^2.13.0",
        "miniflare": "^3.0.0",
        "prettier": "3.2.5",
        "ts-jest": "^29.1.0",
        "ts-node": "^10.9.1",
        "typescript": "^5.0.3",
        "wrangler": "^3.23.0"
    }

 "dependencies": {
        "@cloudflare/itty-router-openapi": "^1.0.6",
        "@miniflare/d1": "^2.14.1",
        "@tsndr/cloudflare-worker-jwt": "^2.2.1",
        "itty-router": "^4.0.11",
        "moment": "^2.29.4",
        "stripe": "^14.1.0",
        "tslib": "^2.5.0",
        "worker-auth-providers": "^0.0.13"
    },
    "devDependencies": {
        "@cloudflare/workers-types": "^4.20240208.0",
        "@types/jest": "^29.5.0",
        "@typescript-eslint/eslint-plugin": "7.3.1",
        "@typescript-eslint/parser": "7.2.0",
        "esbuild": "^0.20.0",
        "eslint": "^8.37.0",
        "eslint-config-prettier": "^9.0.0",
        "eslint-config-typescript": "^3.0.0",
        "jest": "^29.5.0",
        "jest-environment-miniflare": "^2.13.0",
        "miniflare": "^3.0.0",
        "prettier": "3.2.5",
        "ts-jest": "^29.1.0",
        "ts-node": "^10.9.1",
        "typescript": "^5.0.3",
        "wrangler": "^3.23.0"
    }

and this version was causing errors:

    "dependencies": {
        "@cloudflare/itty-router-openapi": "^1.0.6",
        "@miniflare/d1": "^2.14.2",
        "@tsndr/cloudflare-worker-jwt": "^2.2.1",
        "itty-router": "^5.0.6",
        "moment": "^2.29.4",
        "stripe": "^14.1.0",
        "tslib": "^2.5.0",
        "worker-auth-providers": "^0.0.13"
    },
    "devDependencies": {
        "@cloudflare/workers-types": "^4.20240329.0",
        "@types/jest": "^29.5.0",
        "@typescript-eslint/parser": "7.5.0",
        "@typescript-eslint/eslint-plugin": "7.5.0",
        "esbuild": "^0.20.0",
        "eslint": "^8.37.0",
        "eslint-config-prettier": "^9.0.0",
        "eslint-config-typescript": "^3.0.0",
        "jest": "^29.5.0",
        "jest-environment-miniflare": "^2.13.0",
        "miniflare": "^3.20240329.0",
        "prettier": "3.2.5",
        "ts-jest": "^29.1.0",
        "ts-node": "^10.9.1",
        "typescript": "^5.0.3",
        "wrangler": "^3.44.0"
    }

    "dependencies": {
        "@cloudflare/itty-router-openapi": "^1.0.6",
        "@miniflare/d1": "^2.14.2",
        "@tsndr/cloudflare-worker-jwt": "^2.2.1",
        "itty-router": "^5.0.6",
        "moment": "^2.29.4",
        "stripe": "^14.1.0",
        "tslib": "^2.5.0",
        "worker-auth-providers": "^0.0.13"
    },
    "devDependencies": {
        "@cloudflare/workers-types": "^4.20240329.0",
        "@types/jest": "^29.5.0",
        "@typescript-eslint/parser": "7.5.0",
        "@typescript-eslint/eslint-plugin": "7.5.0",
        "esbuild": "^0.20.0",
        "eslint": "^8.37.0",
        "eslint-config-prettier": "^9.0.0",
        "eslint-config-typescript": "^3.0.0",
        "jest": "^29.5.0",
        "jest-environment-miniflare": "^2.13.0",
        "miniflare": "^3.20240329.0",
        "prettier": "3.2.5",
        "ts-jest": "^29.1.0",
        "ts-node": "^10.9.1",
        "typescript": "^5.0.3",
        "wrangler": "^3.44.0"
    }

    "dependencies": {
        "@cloudflare/itty-router-openapi": "^1.0.6",
        "@miniflare/d1": "^2.14.2",
        "@tsndr/cloudflare-worker-jwt": "^2.2.1",
        "itty-router": "^5.0.6",
        "moment": "^2.29.4",
        "stripe": "^14.1.0",
        "tslib": "^2.5.0",
        "worker-auth-providers": "^0.0.13"
    },
    "devDependencies": {
        "@cloudflare/workers-types": "^4.20240329.0",
        "@types/jest": "^29.5.0",
        "@typescript-eslint/parser": "7.5.0",
        "@typescript-eslint/eslint-plugin": "7.5.0",
        "esbuild": "^0.20.0",
        "eslint": "^8.37.0",
        "eslint-config-prettier": "^9.0.0",
        "eslint-config-typescript": "^3.0.0",
        "jest": "^29.5.0",
        "jest-environment-miniflare": "^2.13.0",
        "miniflare": "^3.20240329.0",
        "prettier": "3.2.5",
        "ts-jest": "^29.1.0",
        "ts-node": "^10.9.1",
        "typescript": "^5.0.3",
        "wrangler": "^3.44.0"
    }

    "dependencies": {
        "@cloudflare/itty-router-openapi": "^1.0.6",
        "@miniflare/d1": "^2.14.2",
        "@tsndr/cloudflare-worker-jwt": "^2.2.1",
        "itty-router": "^5.0.6",
        "moment": "^2.29.4",
        "stripe": "^14.1.0",
        "tslib": "^2.5.0",
        "worker-auth-providers": "^0.0.13"
    },
    "devDependencies": {
        "@cloudflare/workers-types": "^4.20240329.0",
        "@types/jest": "^29.5.0",
        "@typescript-eslint/parser": "7.5.0",
        "@typescript-eslint/eslint-plugin": "7.5.0",
        "esbuild": "^0.20.0",
        "eslint": "^8.37.0",
        "eslint-config-prettier": "^9.0.0",
        "eslint-config-typescript": "^3.0.0",
        "jest": "^29.5.0",
        "jest-environment-miniflare": "^2.13.0",
        "miniflare": "^3.20240329.0",
        "prettier": "3.2.5",
        "ts-jest": "^29.1.0",
        "ts-node": "^10.9.1",
        "typescript": "^5.0.3",
        "wrangler": "^3.44.0"
    }

J

James•4/2/24, 10:22 PM

I would guess

itty-router

itty-router

itty-router

itty-router since that's a major bump - did you check the breaking changes?

GGagan Suie nothing was changed

J

James•4/2/24, 10:22 PM

Generally I'd recommend double checking this in future

Glad you figured it out though!

JJames I would guess `itty-router` since that's a major bump - did you check the breaki...

G

Gagan SuieOP•4/2/24, 11:15 PM

you were right sir. great deduction!

-i've never seen this error message and everything on google said it was an infinite loop issue on cloudflares side. And cloudflare had some outages today. Also a few people resolved the issue by reverting some random packages they had.
-i did revert to previous deployments, but i guess i didn't go back far enough.
-i have mergify and dependabot that automatically created and deployed new package versions (they dont deploy to production until i manually merge the changes). And i guess itty-router slipped under the radar this time!

J

James•4/2/24, 11:26 PM

Awesome to hear! You might want to spin up a few tests for your worker using the new vitest integration to catch these kind of things in those automation version bumps in future: https://blog.cloudflare.com/workers-vitest-integration

J

James•4/2/24, 11:27 PM

If you're doing automatic dependency bumps, having some basic tests for if your worker still runs is something I would very strongly advise

J

James•4/2/24, 11:27 PM

Even if it's just a smoke test that hits

that would probably have caught this hanging issue from the version bump, and prevented the PR from merging

G

Gagan SuieOP•4/2/24, 11:56 PM

i will investigate this tonight

G

Gagan SuieOP•4/2/24, 11:56 PM

thank you!

GGagan Suie you were right sir. great deduction! -i've never seen this error message and e...

K

Kevin•4/3/24, 1:00 AM

I definitely recommend the v4 --> v5 Migration Guide:
https://itty.dev/itty-router/migrations/v4-v5

But the biggest thing you have to do is to swap any

.handle()

.handle()

.handle()

.handle() references to

.fetch()

.fetch()

.fetch()

.fetch(), as

.handle()

.handle()

.handle()

.handle() is now fully deprecated (v4 allowed both).

Otherwise, when you ask for

.handle()

.handle()

.handle()

.handle() (no longer found), the Proxy catches it and assumes you're trying to register a route, not handle a request.

Mayhem ensues.

Migration Guides > v4 to v5 - itty-router - itty.dev

Ultra-small, powerful helpers for modern serverless APIs.

G

Gagan SuieOP•4/3/24, 1:00 AM

yep. that was the issue. im looking to actually migrate to @Cloudflare/itty-router-openapi

G

Gagan SuieOP•4/3/24, 1:01 AM

but im having trouble restricting the /docs endpoint

GGagan Suie yep. that was the issue. im looking to actually migrate to @Cloudflare/itty-rout...

K

Kevin•4/3/24, 1:01 AM

That one should be safe (they're still on v4). I'll be helping them get fully onto the new v5 stuff soon

G

Gagan SuieOP•4/3/24, 1:02 AM

do you know if something like this is allowed?

router.get('/docs', async () => await isAdmin(request, env))

router.get('/docs', async () => await isAdmin(request, env))

router.get('/docs', async () => await isAdmin(request, env))

router.get('/docs', async () => await isAdmin(request, env))

K

Kevin•4/3/24, 1:03 AM

it depends on what's in

isAdmin

isAdmin

isAdmin

isAdmin

K

Kevin•4/3/24, 1:03 AM

(also, no need for the async/await in there)

G

Gagan SuieOP•4/3/24, 1:05 AM

isAdmin returns true if user is an admin

G

Gagan SuieOP•4/3/24, 1:05 AM

false otherwise

K

Kevin•4/3/24, 1:05 AM

assuming

isAdmin

isAdmin

isAdmin

isAdmin is something like this:

const isAdmin = ({ headers }) => {
  if (headers.get('Authorization') !== 'foo') {
    throw new StatusError(401, 'Not Authorized')

    // alernatively
    return error(401)
  }

  // no return here allows further handlers
}

const isAdmin = ({ headers }) => {
  if (headers.get('Authorization') !== 'foo') {
    throw new StatusError(401, 'Not Authorized')

    // alernatively
    return error(401)
  }

  // no return here allows further handlers
}

const isAdmin = ({ headers }) => {
  if (headers.get('Authorization') !== 'foo') {
    throw new StatusError(401, 'Not Authorized')

    // alernatively
    return error(401)
  }

  // no return here allows further handlers
}

const isAdmin = ({ headers }) => {
  if (headers.get('Authorization') !== 'foo') {
    throw new StatusError(401, 'Not Authorized')

    // alernatively
    return error(401)
  }

  // no return here allows further handlers
}

K

Kevin•4/3/24, 1:06 AM

yeah that's the prob... in itty, any handler that returns anything, stops the flow and responds (with whatever that was). The exception is explicitly returning undefined (that's allowed).

K

Kevin•4/3/24, 1:06 AM

So instead, you want it to not return anything to pass onto the next handlers/routes, or return (or throw) to break out early.

K

Kevin•4/3/24, 1:07 AM

It's an inversion on the express patterns, where you specifically had to allow next handlers to pass. Allows for typically far simpler/shorter handler code, but requires some rethinking (and has some gotchas)

G

Gagan SuieOP•4/3/24, 1:10 AM

the isAdmin middlware code doesn't even hit

G

Gagan SuieOP•4/3/24, 1:10 AM

    const router = OpenAPIRouter()
    router.get('/docs', () => isAdmin(request, env))

    const router = OpenAPIRouter()
    router.get('/docs', () => isAdmin(request, env))

    const router = OpenAPIRouter()
    router.get('/docs', () => isAdmin(request, env))

    const router = OpenAPIRouter()
    router.get('/docs', () => isAdmin(request, env))

K

Kevin•4/3/24, 1:10 AM

If you're trying to cover an entire branch, don't forget the wildcard.

K

Kevin•4/3/24, 1:11 AM

that route will only match

/docs

/docs

, not

/docs/foo

/docs/foo

/docs/foo

/docs/foo

K

Kevin•4/3/24, 1:11 AM

router.get('/docs/*', isAdmin)

router.get('/docs/*', isAdmin)

router.get('/docs/*', isAdmin)

router.get('/docs/*', isAdmin)

G

Gagan SuieOP•4/3/24, 1:11 AM

the swaggerui dashboard is still visible.

G

Gagan SuieOP•4/3/24, 1:13 AM

but now /docs is appearing in swaggerui and when i try to execute it, i get unauthorized.

GGagan Suie ` const router = OpenAPIRouter() router.get('/docs', () => isAdmin(reques...

K

Kevin•4/3/24, 1:13 AM

// where is the request, env coming from here?
router.get('/docs', () => isAdmin(request, env)) 

// here isAdmin naturally has access to the request, env, etc passed from router.fetch
router.get('/docs', isAdmin)

// where is the request, env coming from here?
router.get('/docs', () => isAdmin(request, env)) 

// here isAdmin naturally has access to the request, env, etc passed from router.fetch
router.get('/docs', isAdmin)

// where is the request, env coming from here?
router.get('/docs', () => isAdmin(request, env)) 

// here isAdmin naturally has access to the request, env, etc passed from router.fetch
router.get('/docs', isAdmin)

// where is the request, env coming from here?
router.get('/docs', () => isAdmin(request, env)) 

// here isAdmin naturally has access to the request, env, etc passed from router.fetch
router.get('/docs', isAdmin)

K

Kevin•4/3/24, 1:14 AM

have a repo for me to take a look at?

G

Gagan SuieOP•4/3/24, 1:15 AM

the frontend is opensource, the backend is not unfortunately

G

Gagan SuieOP•4/3/24, 1:15 AM

but you can watch my live stream on this very platform:
https://mage.stream/channel/18

Mage - A collaborative streaming platform

Mage is a collaborative streaming platform for gamers and developers

G

Gagan SuieOP•4/3/24, 1:16 AM

can you see my stream?

K

Kevin•4/3/24, 1:17 AM

I can

G

Gagan SuieOP•4/3/24, 1:17 AM

brilliant!

K

Kevin•4/3/24, 1:17 AM

was just logging in

K

Kevin•4/3/24, 1:47 AM

ok, to put your own upstream catch before OpenAPI:

import { Router, error } from 'itty-router'
import { OpenAPIRouter } from '@cloudflare/itty-router-openapi'

const rootRouter = Router()
const router = OpenAPIRouter()

// example middleware
const isAdmin = (request) => {
  if (!request.headers.get('Authorization')) {
    return error(401)
  }
}

rootRouter
  // we register the auth catch on the root router
  .all('/docs/*', isAdmin)

  // then attach the next router to handle all requests
  .all('*', router.fetch)

export default {
  fetch: router.fetch
}

import { Router, error } from 'itty-router'
import { OpenAPIRouter } from '@cloudflare/itty-router-openapi'

const rootRouter = Router()
const router = OpenAPIRouter()

// example middleware
const isAdmin = (request) => {
  if (!request.headers.get('Authorization')) {
    return error(401)
  }
}

rootRouter
  // we register the auth catch on the root router
  .all('/docs/*', isAdmin)

  // then attach the next router to handle all requests
  .all('*', router.fetch)

export default {
  fetch: router.fetch
}

import { Router, error } from 'itty-router'
import { OpenAPIRouter } from '@cloudflare/itty-router-openapi'

const rootRouter = Router()
const router = OpenAPIRouter()

// example middleware
const isAdmin = (request) => {
  if (!request.headers.get('Authorization')) {
    return error(401)
  }
}

rootRouter
  // we register the auth catch on the root router
  .all('/docs/*', isAdmin)

  // then attach the next router to handle all requests
  .all('*', router.fetch)

export default {
  fetch: router.fetch
}

import { Router, error } from 'itty-router'
import { OpenAPIRouter } from '@cloudflare/itty-router-openapi'

const rootRouter = Router()
const router = OpenAPIRouter()

// example middleware
const isAdmin = (request) => {
  if (!request.headers.get('Authorization')) {
    return error(401)
  }
}

rootRouter
  // we register the auth catch on the root router
  .all('/docs/*', isAdmin)

  // then attach the next router to handle all requests
  .all('*', router.fetch)

export default {
  fetch: router.fetch
}

G

Gagan SuieOP•4/3/24, 2:38 AM

that didn't work either. i learned how to handle it properly via their parameters:

    const router: any = OpenAPIRouter({
        docs_url: env.ENV === 'local' ? '/docs' : 'null',
        redoc_url: env.ENV === 'local' ? '/redocs' : 'null'
    })

    const router: any = OpenAPIRouter({
        docs_url: env.ENV === 'local' ? '/docs' : 'null',
        redoc_url: env.ENV === 'local' ? '/redocs' : 'null'
    })

    const router: any = OpenAPIRouter({
        docs_url: env.ENV === 'local' ? '/docs' : 'null',
        redoc_url: env.ENV === 'local' ? '/redocs' : 'null'
    })

    const router: any = OpenAPIRouter({
        docs_url: env.ENV === 'local' ? '/docs' : 'null',
        redoc_url: env.ENV === 'local' ? '/redocs' : 'null'
    })

i was sending empty string. but it required a null string.

K

Kevin•4/3/24, 2:41 AM

oh super weird...

KKevin oh super weird...

G

Gagan SuieOP•4/3/24, 3:24 AM

hey thanks again for helping me remove a lot of fluff from the routes!

K

Kevin•4/3/24, 4:21 AM

Happy to help!

Similar Threads

Similar Threads

Similar Threads