Pod ran out of CPU RAM

#!/bin/bash

. /mnt/disk/vllm-venv/bin/activate

export LD_LIBRARY_PATH=/usr/local/cuda-12.4/lib64:$LD_LIBRARY_PATH
export PATH=/usr/local/cuda-12.4/bin:$PATH
export CUDACXX=/usr/local/cuda-12.4/bin/nvcc

export RAY_memory_monitor_refresh_ms=0
#export OMP_NUM_THREADS=4

export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
#export PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync
export VLLM_FLASHINFER_FORCE_TENSOR_CORES=1
export VLLM_USE_FLASHINFER_SAMPLER=1
#export VLLM_USE_RAY_SPMD_WORKER=1
#export VLLM_USE_RAY_COMPILED_DAG=1
#export VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL=1
export VLLM_USE_TRITON_FLASH_ATTN=1
export VLLM_USE_TRITON_AWQ=1
export VLLM_ATTENTION_BACKEND=FLASHINFER
export VLLM_USE_V1=0
export VLLM_ENABLE_V1_MULTIPROCESSING=1
export VLLM_ENABLE_MOE_ALIGN_BLOCK_SIZE_TRITON=1
export VLLM_CUDA_MEM_ALIGN_KV_CACHE=1
#export VLLM_MLA_DISABLE=1

set -e

cd /mnt/disk/models/

vllm serve "./Qwen2.5-72B-Instruct-AWQ" \
--served-model-name="qwen2.5-72b" \
--max-model-len="14000" \
--dtype="auto" \
--gpu-memory-utilization="0.993" \
--distributed-executor-backend="mp" \
--enable-chunked-prefill=false \
--kv-cache-dtype=fp8_e5m2 \
--quantization="awq_marlin" \
--enforce-eager \
--scheduling-policy="priority" \
--tensor-parallel-size="2" \
--swap-space="1" \
--enable-prefix-caching \
--disable-log-requests \
--disable-log-stats \
--enable-auto-tool-choice \
--tool-call-parser hermes \
--host="0.0.0.0" --port="5000"

#!/bin/bash

. /mnt/disk/vllm-venv/bin/activate

export LD_LIBRARY_PATH=/usr/local/cuda-12.4/lib64:$LD_LIBRARY_PATH
export PATH=/usr/local/cuda-12.4/bin:$PATH
export CUDACXX=/usr/local/cuda-12.4/bin/nvcc

export RAY_memory_monitor_refresh_ms=0
#export OMP_NUM_THREADS=4

export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
#export PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync
export VLLM_FLASHINFER_FORCE_TENSOR_CORES=1
export VLLM_USE_FLASHINFER_SAMPLER=1
#export VLLM_USE_RAY_SPMD_WORKER=1
#export VLLM_USE_RAY_COMPILED_DAG=1
#export VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL=1
export VLLM_USE_TRITON_FLASH_ATTN=1
export VLLM_USE_TRITON_AWQ=1
export VLLM_ATTENTION_BACKEND=FLASHINFER
export VLLM_USE_V1=0
export VLLM_ENABLE_V1_MULTIPROCESSING=1
export VLLM_ENABLE_MOE_ALIGN_BLOCK_SIZE_TRITON=1
export VLLM_CUDA_MEM_ALIGN_KV_CACHE=1
#export VLLM_MLA_DISABLE=1

set -e

cd /mnt/disk/models/

vllm serve "./Qwen2.5-72B-Instruct-AWQ" \
--served-model-name="qwen2.5-72b" \
--max-model-len="14000" \
--dtype="auto" \
--gpu-memory-utilization="0.993" \
--distributed-executor-backend="mp" \
--enable-chunked-prefill=false \
--kv-cache-dtype=fp8_e5m2 \
--quantization="awq_marlin" \
--enforce-eager \
--scheduling-policy="priority" \
--tensor-parallel-size="2" \
--swap-space="1" \
--enable-prefix-caching \
--disable-log-requests \
--disable-log-stats \
--enable-auto-tool-choice \
--tool-call-parser hermes \
--host="0.0.0.0" --port="5000"

# Select model and load it.
MODEL_ID = "/workspace/models/Qwen2.5-72B-Instruct"
device_map = calculate_offload_device_map(
    MODEL_ID,
    reserve_for_hessians=False,
    num_gpus=1,
    torch_dtype=torch.bfloat16,
    trust_remote_code=True,
)
model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID,
    device_map=device_map,
    torch_dtype=torch.bfloat16,
    trust_remote_code=True
)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

# Select model and load it.
MODEL_ID = "/workspace/models/Qwen2.5-72B-Instruct"
device_map = calculate_offload_device_map(
    MODEL_ID,
    reserve_for_hessians=False,
    num_gpus=1,
    torch_dtype=torch.bfloat16,
    trust_remote_code=True,
)
model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID,
    device_map=device_map,
    torch_dtype=torch.bfloat16,
    trust_remote_code=True
)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

# Apply quantization
oneshot(
    model=model,
    dataset=ds,
    recipe=recipe,
    max_seq_length=MAX_SEQUENCE_LENGTH,
    num_calibration_samples=NUM_CALIBRATION_SAMPLES,
)

# Save quantized model
SAVE_DIR = MODEL_ID.split("/")[1] + "-FP8-KV"
model.save_pretrained(SAVE_DIR, save_compressed=True)

# Apply quantization
oneshot(
    model=model,
    dataset=ds,
    recipe=recipe,
    max_seq_length=MAX_SEQUENCE_LENGTH,
    num_calibration_samples=NUM_CALIBRATION_SAMPLES,
)

# Save quantized model
SAVE_DIR = MODEL_ID.split("/")[1] + "-FP8-KV"
model.save_pretrained(SAVE_DIR, save_compressed=True)