RunPod•2w ago

The total token limit at 131

I use vLLM and set max model length to 8000 a2048 but out is just 131 (total out + in ), although i have set max tokens to 2048. I try with 2 models and result is the same.

8 Replies

Jason•2w ago

Hi, you need to use the right input for max output token

{
  "input": {
    "messages": [
      {
        "role": "system",
        "content": "Your are an ai assistant."
      },
      {
        "role": "user",
        "content": "Explain llm models"
      }
    ],
"sampling_params": {
    "max_tokens": 3000,
    "temperature": 0.7,
    "top_p": 0.95,
    "n": 1,
    "stream": false,
    "stop": [],
    "presence_penalty": 0,
    "frequency_penalty": 0,
    "logit_bias": {},
    "best_of": 1
}
  }
}

{
  "input": {
    "messages": [
      {
        "role": "system",
        "content": "Your are an ai assistant."
      },
      {
        "role": "user",
        "content": "Explain llm models"
      }
    ],
"sampling_params": {
    "max_tokens": 3000,
    "temperature": 0.7,
    "top_p": 0.95,
    "n": 1,
    "stream": false,
    "stop": [],
    "presence_penalty": 0,
    "frequency_penalty": 0,
    "logit_bias": {},
    "best_of": 1
}
  }
}