Building AI Agents with smol-agents, QWEN, and vLLM

# Create and activate venv
python3 -m venv .venv
source .venv/bin/activate

# Upgrade pip
pip install -U pip

# Install runtime deps
pip install smolagents openai requests vllm

            
# Start vLLM server
python -m vllm.entrypoints.openai.api_server \
--model Qwen/Qwen3-4B-Instruct-2507 `# Model name/path to load (4B parameter LLM)` \
--host 127.0.0.1 `# Local bind address for the server` \
--port 8000 `# Port for the server to listen on` \
--dtype float16 `# Data type for model weights (float16 for reduced VRAM usage)` \
--max-model-len 8192 `# Max context length in tokens for input/output` \
--gpu-memory-utilization 0.95 `# Fraction of GPU VRAM to use (95% of 12GB)` \
--swap-space 20 `# GB of system RAM for offloading when VRAM is full` \
--max-num-seqs 1 `# Max parallel sequences to process at once` \
--max-num-batched-tokens 8192 `# Max tokens processed in a single batch`
            
          

from smolagents import CodeAgent, tool, OpenAIServerModel
from openai import OpenAI

# Direct OpenAI client for simple chat
client = OpenAI(
    base_url="http://127.0.0.1:8000/v1",
    api_key="EMPTY",
)

# smolagents setup for tool usage
model = OpenAIServerModel(
    model_id="Qwen/Qwen3-4B-Instruct-2507",
    api_base="http://127.0.0.1:8000/v1",
    api_key="EMPTY",
    timeout=60,
)

@tool
def ping() -> str:
    """Return a simple string to prove tool-calling works."""
    return "pong"

# CodeAgent for tool usage
agent = CodeAgent(
    tools=[ping],
    model=model,
    verbosity_level=2,
)

def simple_chat(message: str) -> str:
    """Simple chat without any agent overhead."""
    response = client.chat.completions.create(
        model="Qwen/Qwen3-4B-Instruct-2507",
        messages=[{"role": "user", "content": message}],
        max_tokens=100
    )
    return response.choices[0].message.content

if __name__ == "__main__":
    print("=== Hello Test ===")
    print(simple_chat("Hello"))
    print("\n=== Ping Tool Test ===")
    print(agent.run("Use the ping tool"))

python agent_hello.py

import os
import json
import asyncio
import requests
from openai import OpenAI

# --- Config ---
BASE_URL = "http://127.0.0.1:8000/v1"
MODEL = "Qwen/Qwen3-4B-Instruct-2507"
DATA_DIR = "data"
USERS_FILE = os.path.join(DATA_DIR, "users.json")

os.makedirs(DATA_DIR, exist_ok=True)

client = OpenAI(base_url=BASE_URL, api_key="not-needed")


# --- Tools ---
def ping() -> str:
    """Return a simple pong string."""
    return "pong"


def fetch_users(limit: int = 5) -> str:
    """
    Fetch dummy users from https://dummyjson.com.
    Save results to data/users.json and return a summary.
    """
    url = f"https://dummyjson.com/users?limit={limit}"
    resp = requests.get(url)
    resp.raise_for_status()
    payload = resp.json()

    with open(USERS_FILE, "w") as f:
        json.dump(payload, f, indent=2)

    users = payload.get("users", [])
    first_names = [u["firstName"] + " " + u["lastName"] for u in users[:3]]
    more = max(0, len(users) - 3)
    summary = f"Users: {len(users)} total; first: {', '.join(first_names)}"
    if more:
        summary += f" … +{more} more"

    return f"Saved to: {USERS_FILE}\n{summary}"


def load_user_context() -> str:
    """Load saved users.json and return as string for RAG context."""
    if not os.path.exists(USERS_FILE):
        return ""
    try:
        with open(USERS_FILE) as f:
            data = json.load(f)
        return json.dumps(data, indent=2)
    except Exception as e:
        return f"(Could not load JSON from {USERS_FILE}: {e})"


# --- Chat wrapper ---
def chat_with_model(prompt: str, context: str = "") -> str:
    """Send a message to the model, optionally with RAG context."""
    messages = []
    if context:
        messages.append({"role": "system", "content": f"Context:\n{context}"})
    messages.append({"role": "user", "content": prompt})

    resp = client.chat.completions.create(model=MODEL, messages=messages)
    return resp.choices[0].message.content


# --- REPL loop ---
async def repl():
    print("⚡ Async Agent (users only). Type 'exit' to quit.")
    print("Examples:")
    print("  ping")
    print("  fetch users")
    print("  fetch users 3\n")

    while True:
        text = input("you> ").strip()
        if text.lower() in {"exit", "quit"}:
            break

        if text == "ping":
            print("[ping…]")
            print(ping())

        elif text.startswith("fetch users"):
            parts = text.split()
            limit = int(parts[2]) if len(parts) > 2 else 5
            print(f"[fetch users {limit}…]")
            print(fetch_users(limit))

        else:
            print("[chat…]")
            context = load_user_context()
            reply = chat_with_model(text, context=context)
            print(reply)


if __name__ == "__main__":
    asyncio.run(repl())

python agent_async_fn.py

you> ping
[ping…]
pong

you> fetch users 3
[fetch users 3…]
Saved to: data/users.json
Users: 3 total; first: John Doe, Jane Smith, Alice Johnson

Fetched JSON:
{
  "users": [
    { "firstName": "John", "lastName": "Doe", "email": "john@example.com" },
    { "firstName": "Jane", "lastName": "Smith", "email": "jane@example.com" },
    { "firstName": "Alice", "lastName": "Johnson", "email": "alice@example.com" }
  ]
}

you> how old is John Doe?
[chat…]
He is 25 years old in the dummy dataset.