92 lines
2.2 KiB
Plaintext
92 lines
2.2 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "928261ae",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from vllm import LLM, SamplingParams"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "a35fc2b4",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"model_id = \"./llms/OLMoE-1B-7B-0924-Instruct\"\n",
|
|
"\n",
|
|
"llm = LLM(\n",
|
|
" model=model_id,\n",
|
|
" # cpu_offload_gb=4,\n",
|
|
" tensor_parallel_size=2,\n",
|
|
" # gpu_memory_utilization=0.90,\n",
|
|
" max_model_len=4096,\n",
|
|
")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "6a708f11",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"sampling_params = SamplingParams(\n",
|
|
" temperature=0.6,\n",
|
|
" top_p=0.95,\n",
|
|
" top_k=20,\n",
|
|
" max_tokens=1024,\n",
|
|
")\n",
|
|
"\n",
|
|
"# Prepare the input to the model\n",
|
|
"prompt = \"Give me a short introduction to large language models.\"\n",
|
|
"messages = [\n",
|
|
" {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
|
|
" {\"role\": \"user\", \"content\": prompt}\n",
|
|
"]\n",
|
|
"\n",
|
|
"# Generate outputs\n",
|
|
"outputs = llm.chat(\n",
|
|
" messages, \n",
|
|
" sampling_params=sampling_params,\n",
|
|
" # chat_template_kwargs={\"enable_thinking\": True}, # Set to False to strictly disable thinking\n",
|
|
")\n",
|
|
"\n",
|
|
"# Print the outputs.\n",
|
|
"for out in outputs:\n",
|
|
" # out.prompt is the input prompt; out.outputs is a list of completion choices\n",
|
|
" print(\"=== PROMPT ===\")\n",
|
|
" print(out.prompt)\n",
|
|
" print(\"=== COMPLETION ===\")\n",
|
|
" print(out.outputs[0].text)\n",
|
|
" print(\"\\n---\\n\")"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "moe-explore",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.12.11"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|