is stream a POST endpoint or GET endpoint (locally)?

Is stream a POST endpoint or GET endpoint. Trying to run the handler locally before hosting it on runpod with streaming but it's not working.

Noticed the example here: https://doc.runpod.io/reference/llama2-13b-chat#streaming-token-outputs
where /stream is a GET endpoint

But when I check the fastapi swagger API it shows it's a POST endpoint, which ofc doesn't stream but rather return all results together.

Here is my simplified handler:

import runpod
import asyncio

async def handler(job):
    for count in range(3):
        result = f"This is the {count} generated output."
        yield result
        await asyncio.sleep(5)


runpod.serverless.start(
    {
        "handler": handler,
    }
)

import runpod
import asyncio

async def handler(job):
    for count in range(3):
        result = f"This is the {count} generated output."
        yield result
        await asyncio.sleep(5)


runpod.serverless.start(
    {
        "handler": handler,
    }
)

and by client which fails with /stream (GET) (method not allowed error) locally.

import requests

def benchmark_response():
    headers = {
        "Content-Type": "application/json"
    }
    response = requests.post(
        url='http://localhost:8000/run', 
        headers=headers, json={'whatever': 1}, timeout=600,
    )
    
    job_id = response.json()['id']
    print(job_id)
    url = f'http://localhost:8000/stream/{job_id}'

    while True:
        get_status = requests.get(url, headers=headers)
        print(get_status.text)


if __name__ == '__main__':
    benchmark_response()

import requests

def benchmark_response():
    headers = {
        "Content-Type": "application/json"
    }
    response = requests.post(
        url='http://localhost:8000/run', 
        headers=headers, json={'whatever': 1}, timeout=600,
    )
    
    job_id = response.json()['id']
    print(job_id)
    url = f'http://localhost:8000/stream/{job_id}'

    while True:
        get_status = requests.get(url, headers=headers)
        print(get_status.text)


if __name__ == '__main__':
    benchmark_response()

Runpod•2y ago•

15 replies

goku

is stream a POST endpoint or GET endpoint (locally)?

import runpod
import asyncio

async def handler(job):
    for count in range(3):
        result = f"This is the {count} generated output."
        yield result
        await asyncio.sleep(5)


runpod.serverless.start(
    {
        "handler": handler,
    }
)

import runpod
import asyncio

async def handler(job):
    for count in range(3):
        result = f"This is the {count} generated output."
        yield result
        await asyncio.sleep(5)


runpod.serverless.start(
    {
        "handler": handler,
    }
)

and by client which fails with /stream (GET) (method not allowed error) locally.

import requests

def benchmark_response():
    headers = {
        "Content-Type": "application/json"
    }
    response = requests.post(
        url='http://localhost:8000/run', 
        headers=headers, json={'whatever': 1}, timeout=600,
    )
    
    job_id = response.json()['id']
    print(job_id)
    url = f'http://localhost:8000/stream/{job_id}'

    while True:
        get_status = requests.get(url, headers=headers)
        print(get_status.text)


if __name__ == '__main__':
    benchmark_response()

import requests

def benchmark_response():
    headers = {
        "Content-Type": "application/json"
    }
    response = requests.post(
        url='http://localhost:8000/run', 
        headers=headers, json={'whatever': 1}, timeout=600,
    )
    
    job_id = response.json()['id']
    print(job_id)
    url = f'http://localhost:8000/stream/{job_id}'

    while True:
        get_status = requests.get(url, headers=headers)
        print(get_status.text)


if __name__ == '__main__':
    benchmark_response()

is stream a POST endpoint or GET endpoint (locally)?

Similar Threads

is stream a POST endpoint or GET endpoint (locally)?

Similar Threads

Similar Threads

Similar Threads