AI Gateway is incorrectly considering a streaming `refusal` on the Anthropic API to be successful. h

AI Gateway is incorrectly considering a streaming

refusal

refusal

refusal

refusal on the Anthropic API to be successful. https://docs.anthropic.com/en/docs/test-and-evaluate/strengthen-guardrails/handle-streaming-refusals#implementation-guide

Here's is a sample response:

{
  "id": "msg_01HCDu5LRGeP2o7s2xGmxyx8",
  "type": "message",
  "role": "assistant",
  "model": "claude-opus-4-1-20250805",
  "content": "",
  "stop_reason": null,
  "stop_sequence": null,
  "usage": {
    "input_tokens": 9,
    "cache_creation_input_tokens": 16203,
    "cache_read_input_tokens": 0,
    "cache_creation": {
      "ephemeral_5m_input_tokens": 16203,
      "ephemeral_1h_input_tokens": 0
    },
    "output_tokens": 0
  },
  "streamed_data": [
    {
      "nonce": "e8071956",
      "type": "message_start",
      "message": {
        "id": "msg_01HCDu5LRGeP2o7s2xGmxyx8",
        "type": "message",
        "role": "assistant",
        "model": "claude-opus-4-1-20250805",
        "content": [],
        "stop_reason": null,
        "stop_sequence": null,
        "usage": {
          "input_tokens": 9,
          "cache_creation_input_tokens": 16203,
          "cache_read_input_tokens": 0,
          "cache_creation": {
            "ephemeral_5m_input_tokens": 16203,
            "ephemeral_1h_input_tokens": 0
          },
          "output_tokens": 0
        }
      }
    },
    {
      "type": "message_delta",
      "delta": {
        "stop_reason": "refusal",
        "stop_sequence": null
      },
      "usage": {
        "input_tokens": 9,
        "cache_creation_input_tokens": 16203,
        "cache_read_input_tokens": 0,
        "output_tokens": 0
      }
    },
    {
      "nonce": "8aeda459",
      "type": "message_stop"
    }
  ]
}

{
  "id": "msg_01HCDu5LRGeP2o7s2xGmxyx8",
  "type": "message",
  "role": "assistant",
  "model": "claude-opus-4-1-20250805",
  "content": "",
  "stop_reason": null,
  "stop_sequence": null,
  "usage": {
    "input_tokens": 9,
    "cache_creation_input_tokens": 16203,
    "cache_read_input_tokens": 0,
    "cache_creation": {
      "ephemeral_5m_input_tokens": 16203,
      "ephemeral_1h_input_tokens": 0
    },
    "output_tokens": 0
  },
  "streamed_data": [
    {
      "nonce": "e8071956",
      "type": "message_start",
      "message": {
        "id": "msg_01HCDu5LRGeP2o7s2xGmxyx8",
        "type": "message",
        "role": "assistant",
        "model": "claude-opus-4-1-20250805",
        "content": [],
        "stop_reason": null,
        "stop_sequence": null,
        "usage": {
          "input_tokens": 9,
          "cache_creation_input_tokens": 16203,
          "cache_read_input_tokens": 0,
          "cache_creation": {
            "ephemeral_5m_input_tokens": 16203,
            "ephemeral_1h_input_tokens": 0
          },
          "output_tokens": 0
        }
      }
    },
    {
      "type": "message_delta",
      "delta": {
        "stop_reason": "refusal",
        "stop_sequence": null
      },
      "usage": {
        "input_tokens": 9,
        "cache_creation_input_tokens": 16203,
        "cache_read_input_tokens": 0,
        "output_tokens": 0
      }
    },
    {
      "nonce": "8aeda459",
      "type": "message_stop"
    }
  ]
}

{
  "id": "msg_01HCDu5LRGeP2o7s2xGmxyx8",
  "type": "message",
  "role": "assistant",
  "model": "claude-opus-4-1-20250805",
  "content": "",
  "stop_reason": null,
  "stop_sequence": null,
  "usage": {
    "input_tokens": 9,
    "cache_creation_input_tokens": 16203,
    "cache_read_input_tokens": 0,
    "cache_creation": {
      "ephemeral_5m_input_tokens": 16203,
      "ephemeral_1h_input_tokens": 0
    },
    "output_tokens": 0
  },
  "streamed_data": [
    {
      "nonce": "e8071956",
      "type": "message_start",
      "message": {
        "id": "msg_01HCDu5LRGeP2o7s2xGmxyx8",
        "type": "message",
        "role": "assistant",
        "model": "claude-opus-4-1-20250805",
        "content": [],
        "stop_reason": null,
        "stop_sequence": null,
        "usage": {
          "input_tokens": 9,
          "cache_creation_input_tokens": 16203,
          "cache_read_input_tokens": 0,
          "cache_creation": {
            "ephemeral_5m_input_tokens": 16203,
            "ephemeral_1h_input_tokens": 0
          },
          "output_tokens": 0
        }
      }
    },
    {
      "type": "message_delta",
      "delta": {
        "stop_reason": "refusal",
        "stop_sequence": null
      },
      "usage": {
        "input_tokens": 9,
        "cache_creation_input_tokens": 16203,
        "cache_read_input_tokens": 0,
        "output_tokens": 0
      }
    },
    {
      "nonce": "8aeda459",
      "type": "message_stop"
    }
  ]
}

{
  "id": "msg_01HCDu5LRGeP2o7s2xGmxyx8",
  "type": "message",
  "role": "assistant",
  "model": "claude-opus-4-1-20250805",
  "content": "",
  "stop_reason": null,
  "stop_sequence": null,
  "usage": {
    "input_tokens": 9,
    "cache_creation_input_tokens": 16203,
    "cache_read_input_tokens": 0,
    "cache_creation": {
      "ephemeral_5m_input_tokens": 16203,
      "ephemeral_1h_input_tokens": 0
    },
    "output_tokens": 0
  },
  "streamed_data": [
    {
      "nonce": "e8071956",
      "type": "message_start",
      "message": {
        "id": "msg_01HCDu5LRGeP2o7s2xGmxyx8",
        "type": "message",
        "role": "assistant",
        "model": "claude-opus-4-1-20250805",
        "content": [],
        "stop_reason": null,
        "stop_sequence": null,
        "usage": {
          "input_tokens": 9,
          "cache_creation_input_tokens": 16203,
          "cache_read_input_tokens": 0,
          "cache_creation": {
            "ephemeral_5m_input_tokens": 16203,
            "ephemeral_1h_input_tokens": 0
          },
          "output_tokens": 0
        }
      }
    },
    {
      "type": "message_delta",
      "delta": {
        "stop_reason": "refusal",
        "stop_sequence": null
      },
      "usage": {
        "input_tokens": 9,
        "cache_creation_input_tokens": 16203,
        "cache_read_input_tokens": 0,
        "output_tokens": 0
      }
    },
    {
      "nonce": "8aeda459",
      "type": "message_stop"
    }
  ]
}

Anthropic

Streaming refusals - Anthropic

daveOP•9/19/25, 5:18 AM

Is it not possible to delete dynamic routes?

.r20•9/19/25, 7:03 PM

ive been wondering the same thing too i cant find a way to do it

Beefp•9/19/25, 7:07 PM

Any plans to support OpenAIs responses API? (and offer compatibility with other models) - would be an absolute game changer compared to other AI gateways

.r20•9/19/25, 9:44 PM

it already supports responses api

.r20•9/19/25, 9:44 PM

just use the compat endpoint they give you for the base url and you can just do client.response like normal

.r20•9/19/25, 9:44 PM

no changes necessary

..r20 no changes necessary

SmallShen•9/20/25, 12:14 AM

I think the compat endpoint only support completions api?

.r20•9/20/25, 12:56 AM

ive used responses api with ai gateway before i just dont remember which endpoint i used tbh

karimulm•9/22/25, 5:18 AM

Dynamic routes menu missing pagination. No pagination controls available, making page 2+ data inaccessible.

Ddave Is it not possible to delete dynamic routes?

Puliczek•9/22/25, 10:24 AM

same problem

Puliczek•9/22/25, 10:25 AM

models/gemini-2.5-flash-lite is not found for API version v1, or is not supported for generateContent. Call ListModels to see the list of available models and their supported methods

models/gemini-2.5-flash-lite is not found for API version v1, or is not supported for generateContent. Call ListModels to see the list of available models and their supported methods

How to call gemini-2.5-flash-lite?

Puliczek•9/22/25, 11:05 AM

btw gemini-2.5-flash is working

Puliczek•9/22/25, 11:07 AM

but i need gemini-2.5-flash-lite

Puliczek•9/22/25, 11:17 AM

gemini-2.5-flash-lite model works when I use @google/generative-ai. So problem solved, i think now i can't use own cf-metadata.

daveOP•9/22/25, 8:56 PM

HTTP 525 issue with Anthropic models with AI Gateway it seems?

SmallShen•9/23/25, 5:24 AM

I think there is a performance issue for ai gateway ui dashboard, if the streaming response is large, clicking(expand) the log would lag for a sec.

SmallShen•9/23/25, 5:26 AM

There is also a ui bug in dynamic routes dashboard, the background color is incorrect when in dark theme.

.r20•9/25/25, 11:40 PM

whats the prompts tab for in ai gateway?

.r20•9/25/25, 11:40 PM

its not showing anything

noway5566•9/27/25, 2:49 PM

When I test the AI Gateway with stream: true, the response is not delivered in a real-time streaming fashion. Instead, the output only appears all at once after the upstream service finishes processing. In other words, even though I set "stream": true and use Accept: text/event-stream, the gateway buffers the response and returns it all at once instead of streaming it incrementally.

Here’s the sample code I used for testing:

curl -N -X POST https://gateway.ai.cloudflare.com/v1/{account_id}/{gateway_id}/compat/chat/completions \
  --header 'Authorization: Bearer {GOOGLE_GENERATIVE_AI_API_KEY}' \
  --header 'Content-Type: application/json' \
  --header 'Accept: text/event-stream' \
  --header 'Cache-Control: no-cache' \
  --data '{
    "model": "google-ai-studio/gemini-2.0-flash",
    "messages": [
      {
        "role": "user",
        "content": "What is Cloudflare?"
      }
    ],
    "stream": true,
    "temperature": 0.7,
    "max_tokens": 1000
  }' | while IFS= read -r line; do
    echo "[$(date '+%H:%M:%S.%3N')] $line"
  done

curl -N -X POST https://gateway.ai.cloudflare.com/v1/{account_id}/{gateway_id}/compat/chat/completions \
  --header 'Authorization: Bearer {GOOGLE_GENERATIVE_AI_API_KEY}' \
  --header 'Content-Type: application/json' \
  --header 'Accept: text/event-stream' \
  --header 'Cache-Control: no-cache' \
  --data '{
    "model": "google-ai-studio/gemini-2.0-flash",
    "messages": [
      {
        "role": "user",
        "content": "What is Cloudflare?"
      }
    ],
    "stream": true,
    "temperature": 0.7,
    "max_tokens": 1000
  }' | while IFS= read -r line; do
    echo "[$(date '+%H:%M:%S.%3N')] $line"
  done

Could you confirm if AI Gateway currently supports real-time streaming passthrough, or if it’s expected behavior that the response is buffered and only returned once the upstream completes?

Acterion•9/27/25, 5:32 PM

New to AI gateway
Trying to set it up with direct connection with OpenWeb-UI hosted on my subdomain (using CF tunnel to my own infra) and constatnly getting CORS errors when sending OPTIONS request to /completions endpoint. Seems like CF either dropping Allow-Cross-Origin headers or/and not working correctly with OPTIONS requests.
Did anyone had success configuring OpenWebUI to work with Gateway before?

mongj•9/28/25, 7:16 AM

hey does anyone know if the AI gateway supports sending audio files in a multi-part form request for STT models like whisper?

I was testing out the groq endpoint but consistently got 400s for valid audio files. URL input seems to work correctly.

mongj•9/28/25, 7:16 AM

here's a basic request I tested with

curl -X POST https://gateway.ai.cloudflare.com/v1/<cloudflare-acco
unt-id>/<gateway-name>/groq/audio/transcriptions \
  -H "Content-Type: multipart/form-data" \
  -H "Authorization: Bearer <token>" \
  -F "model=whisper-large-v3-turbo" \
  -F "file=@/path/to/audio.mp3"

curl -X POST https://gateway.ai.cloudflare.com/v1/<cloudflare-acco
unt-id>/<gateway-name>/groq/audio/transcriptions \
  -H "Content-Type: multipart/form-data" \
  -H "Authorization: Bearer <token>" \
  -F "model=whisper-large-v3-turbo" \
  -F "file=@/path/to/audio.mp3"

mongj•9/28/25, 7:16 AM

and the response

{"error":{"message":"could not process file - is it a valid media file?","type":"invalid_request_error"}}

{"error":{"message":"could not process file - is it a valid media file?","type":"invalid_request_error"}}

mongj•9/28/25, 7:16 AM

Using a url pointing to an uploaded version of the same audio file works as intended.

CodingCoop | Co-Founder Nullshot•9/28/25, 1:04 PM

How can I get access to this - applied a few weeks ago and no response.

About to lean into open router but would love to partner:

https://blog.cloudflare.com/ai-gateway-aug-2025-refresh/

wondering if joining the startup program would help push this through?

The Cloudflare Blog

AI Gateway now gives you access to your favorite AI models, dynamic...

AI Gateway simplifies AI app development with unified billing, secure key storage, and dynamic routing. Gain observability and control over costs, API keys, and traffic, connecting to major AI providers through a single endpoint.

Leszek•9/29/25, 7:14 AM

Hey, I've created my first AI Search instance, with AI Gateway where I added the OpenAI key.
I've added my first document and I get an

error

error

while building embeddings during the sync.

I've chosen an OpenAI embedding model. The error I am getting is about missing token... It appears, AI Search is not using the token I have provided within the AI Gateway.

{
  "error": {
    "message": "You didn't provide an API key. You need to provide your API key in an Authorization header using Bearer auth (i.e. Authorization: Bearer YOUR_KEY), or as the password field (with blank username) if you're accessing the API from your browser and are prompted for a username and password. You can obtain an API key from https://platform.openai.com/account/api-keys.",
    "type": "invalid_request_error",
    "param": null,
    "code": null
  }
}

{
  "error": {
    "message": "You didn't provide an API key. You need to provide your API key in an Authorization header using Bearer auth (i.e. Authorization: Bearer YOUR_KEY), or as the password field (with blank username) if you're accessing the API from your browser and are prompted for a username and password. You can obtain an API key from https://platform.openai.com/account/api-keys.",
    "type": "invalid_request_error",
    "param": null,
    "code": null
  }
}

LLeszek Hey, I've created my first AI Search instance, with AI Gateway where I added the...

misterPaul•9/29/25, 11:26 AM

have you set up "Authenticated Gateway" on the settings page and passed the token in the cf-aig-authorization header as a bearer token

misterPaul•9/29/25, 11:38 AM

Go to your AI Search instance > Settings > Models. Is the Gateway field pointing to the correct AI Gateway where you stored the key?
In that specific AI Gateway, under "Provider Keys", is the OpenAI key definitely there and valid?

Those are the two obvious things to check, after that check the "Authenticated Gateway" setting I mentioned

Leszek•9/29/25, 11:46 AM

ave you set up "Authenticated Gateway" on the settings page

yes

and passed the token in the cf-aig-authorization header as a bearer token

no. I found no place for that.

I checked again right now and everything works.... So I guess it was an issue on the Cloudflare side.

Thank you for your help!

.r20•9/29/25, 5:39 PM

not sure if this is a bug but ai gateway created datasets dont support using metadata filters

.r20•9/29/25, 5:39 PM

daveOP•9/30/25, 3:57 PM

If I update a Secret Store value that's managed by AI Gateway, should that work?

Nnoway5566 When I test the AI Gateway with stream: true, the response is not delivered in a...

Brandon Studio•10/1/25, 11:44 AM

Same issue here. Could anyone help?

Brandon Studio•10/1/25, 11:44 AM

I have a video record

Nnoway5566 When I test the AI Gateway with stream: true, the response is not delivered in a...

daveOP•10/1/25, 10:14 PM

This is a regression afaik, iirc @Kathy might remember we talked about it and I think there was an internal ticket?

Aac We are having issues where OpenAI requests are now getting blocked because they ...

sarme•10/2/25, 5:15 PM

Same this happened to us sudden 18% error rate with 0 transparency over which worker location was involved. Also saw it in Hackernews today.

cosbgn•10/3/25, 8:33 AM

What kind of rate limits apply when using unified billing on a partner provider, e.g. sonnet 4.5?

cosbgn•10/3/25, 8:34 AM

I think by default anthropic has something like 60 requests per minute, but I woudn't be using my own api key as I would use the gateway-billing

cosbgn•10/3/25, 8:35 AM

So I'm wondering if I can safely move all my traffic there and cloudflare will automatically use another api key if one hits a limit

Zetax•10/4/25, 7:08 PM

Does Cloudflaare AI gateway allow me to call workers ai from any application? If not, that would be such a great feature

ElishaKay•10/8/25, 7:10 AM

Am I supposed to get logs of ai-search within API gateway?

I'm only seeing logs of my embedding calls - but not on my llm_completion (inference) calls

muhtasim•10/8/25, 8:46 AM

how to get llm usage (neuron) via api?

stephen•10/8/25, 9:27 PM

AI Gateway BYOK Issue: Google Vertex AI Imagen Authentication Fails
Problem
Cloudflare AI Gateway's BYOK (Provider Keys) feature successfully authenticates third-party Vertex AI models (Mistral) but fails to authenticate Google's native models (Imagen), despite using the same service account configuration.

---

What Works

Mistral OCR via AI Gateway BYOK:

await env.AI.gateway("my-gateway").run({
provider: "google-vertex-ai",
endpoint: "v1/projects/my-project/locations/us-central1/publishers/mistralai/models/mistral-ocr-2505:rawPredict",
query: { model: "mistral-ocr-2505", document: {...} }
})
//

200 OK - Works perfectly

---

What Fails

Imagen 4 via AI Gateway BYOK:

await env.AI.gateway("my-gateway").run({
provider: "google-vertex-ai",
endpoint: "v1/projects/my-project/locations/us-central1/publishers/google/models/imagen-4.0-generate-001:predict",
query: {
instances: [{ prompt: "test image" }],
parameters: { sampleCount: 1 }
}
})
//

401 from Google: "CREDENTIALS_MISSING"

---

Configurations Tried

Service account has Vertex AI User IAM role

Added "region": "us-central1" to service account JSON

Tested both :predict and :rawPredict endpoints - both fail

Verified credentials work when calling Vertex AI directly
---

Observation
Third-party publishers (publishers/mistralai/): BYOK authentication works
Google publishers (publishers/google/): BYOK authentication fails

The AI Gateway appears to not generate or pass OAuth2 credentials correctly for Google's native Vertex AI models.

---

Question: Is this a known limitation of BYOK with Google Vertex AI native models, or should this be working?

Sstephen AI Gateway BYOK Issue: Google Vertex AI Imagen Authentication Fails Problem Clou...

Marco Ancona•10/8/25, 9:53 PM

Not only Vertex. In the last 4 hours, we saw that all requests to Google AI Studio BYOK via AI Gateway are failing with the error "400 API key not valid. Please pass a valid API key."

We tried to generate a new key and tested the key locally, confirming it works. Still, the error occurs when running via AI Gateway.

Calls to OpenAI via AI Gateway work.

AI Gateway is incorrectly considering a streaming `refusal` on the Anthropic API to be successful. h

Similar Threads

Similar Threads

Similar Threads