Small Language Fashions with Hugging Face transformers Library + smolLM3

# Python 3.10 or newer required
python --version

# Create and activate a digital surroundings
python -m venv smollm-env
supply smollm-env/bin/activate       # macOS / Linux
smollm-envScriptsactivate          # Home windows

# Set up all dependencies
pip set up 
  "transformers>=4.53.0" 
  "torch>=2.3.0" 
  "speed up>=0.30.0" 
  "bitsandbytes>=0.43.0" 
  "sentencepiece" 
  "trl>=0.9.0" 
  "peft>=0.11.0" 
  "datasets>=2.19.0"

# device_check.py
# Run this earlier than anything to verify your setup and choose the appropriate dtype.

def detect_device():
    """
    Detect the most effective out there compute machine.
    Returns (device_str, dtype_str, load_kwargs) to be used with from_pretrained.
    """
    strive:
        import torch
    besides ImportError:
        elevate RuntimeError("PyTorch not discovered. Set up with: pip set up torch")

    if torch.cuda.is_available():
        vram_gb = torch.cuda.get_device_properties(0).total_memory / 1e9
        print(f"CUDA GPU detected: {torch.cuda.get_device_name(0)} ({vram_gb:.1f} GB VRAM)")
        # bfloat16 is advisable for SmolLM3 -- it is the coaching dtype
        return "cuda", torch.bfloat16, {"device_map": "auto", "torch_dtype": torch.bfloat16}

    elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
        print("Apple Silicon MPS detected")
        # MPS helps float16 however not all bfloat16 ops -- use float16 on Apple Silicon
        return "mps", torch.float16, {"device_map": "mps", "torch_dtype": torch.float16}

    else:
        print("No GPU discovered -- working on CPU (slower however practical)")
        return "cpu", torch.float32, {"device_map": "cpu", "torch_dtype": torch.float32}


if __name__ == "__main__":
    machine, dtype, kwargs = detect_device()
    print(f"Gadget : {machine}")
    print(f"Dtype  : {dtype}")
    print(f"Kwargs : {kwargs}")

CUDA GPU detected: NVIDIA GeForce RTX 3060 (12.0 GB VRAM)
Gadget : cuda
Dtype  : torch.bfloat16
Kwargs : {'device_map': 'auto', 'torch_dtype': torch.bfloat16}

# first_inference.py
# Conditions: transformers>=4.53.0, torch, speed up
# Run: python first_inference.py

import re
import torch
from transformers import AutoTokenizer, AutoModelForCausalLM

MODEL_ID = "HuggingFaceTB/SmolLM3-3B"

# ── 1. Load tokenizer and mannequin ───────────────────────────────────────────────

print(f"Loading {MODEL_ID}...")

tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

mannequin = AutoModelForCausalLM.from_pretrained(
    MODEL_ID,
    torch_dtype=torch.bfloat16,    # Match the coaching dtype; use float16 on Apple Silicon
    device_map="auto",             # Spreads throughout all out there GPUs, or CPU if none
)
mannequin.eval()

print(f"Mannequin loaded on: {mannequin.machine}")

# ── 2. Era helper ──────────────────────────────────────────────────────

def generate(messages: listing[dict], max_new_tokens: int = 512) -> str:
    """
    Apply the SmolLM3 chat template, tokenize, generate, and decode.
    Strips the ... block from the output routinely
    so callers at all times obtain the ultimate reply solely.
    """
    # apply_chat_template codecs messages utilizing SmolLM3's built-in chat template
    textual content = tokenizer.apply_chat_template(
        messages,
        tokenize=False,
        add_generation_prompt=True,
    )
    inputs = tokenizer(textual content, return_tensors="pt").to(mannequin.machine)

    with torch.no_grad():
        output_ids = mannequin.generate(
            **inputs,
            max_new_tokens=max_new_tokens,
            temperature=0.6,   # Really useful by the SmolLM3 staff for balanced output
            top_p=0.95,        # Nucleus sampling -- retains output targeted with out being repetitive
            do_sample=True,
        )

    # Decode solely the newly generated tokens, not the enter immediate
    new_tokens = output_ids[0][inputs["input_ids"].form[-1]:]
    uncooked = tokenizer.decode(new_tokens, skip_special_tokens=True)

    # Strip the chain-of-thought block if current.
    # In assume mode the mannequin prefixes its response with ....
    # Callers often solely want the ultimate reply that follows.
    last = re.sub(r".*?", "", uncooked, flags=re.DOTALL).strip()
    return last


# ── 3. Examine assume vs no_think on the identical immediate ──────────────────────────

immediate = "A buyer is charged twice for a similar order. What are three concrete steps assist ought to take?"

# no_think: quick, direct reply -- good for high-throughput classification and replies
no_think_messages = [
    {"role": "system", "content": "/no_think"},
    {"role": "user",   "content": prompt},
]

# assume: reasoning hint earlier than reply -- good for complicated selections and edge circumstances
think_messages = [
    {"role": "system", "content": "/think"},
    {"role": "user",   "content": prompt},
]

print("n── no_think mode ──")
print(generate(no_think_messages, max_new_tokens=256))

print("n── assume mode ──")
print(generate(think_messages, max_new_tokens=512))

python first_inference.py

# ticket_router.py
# Conditions: transformers>=4.53.0, torch, speed up
# Run: python ticket_router.py

import re
import json
import torch
from dataclasses import dataclass
from transformers import AutoTokenizer, AutoModelForCausalLM

MODEL_ID      = "HuggingFaceTB/SmolLM3-3B"
ESCALATE_AT   = 0.70   # Tickets with confidence under this go to a human agent

# ── Information class for a routing consequence ──────────────────────────────────────────

@dataclass
class RoutingResult:
    ticket: str
    class: str           # billing | technical | account | basic
    confidence: float       # 0.0-1.0 self-reported by the mannequin
    reply: str              # Generated in the identical language because the ticket
    escalate: bool          # True when confidence < ESCALATE_AT
    raw_output: str         # Full mannequin output for debugging


# ── System immediate ─────────────────────────────────────────────────────────────

SYSTEM_PROMPT = """You're a multilingual buyer assist router for a SaaS firm.
Your job is to categorise assist tickets and draft a useful, skilled reply.

Guidelines:
- Detect the language of the ticket routinely.
- Classify into EXACTLY ONE of: billing, technical, account, basic.
- Reply within the SAME language because the ticket.
- Price your confidence actually from 0.0 to 1.0. Low confidence means the ticket is ambiguous or outdoors your information.
- Reply ONLY with a single JSON object -- no preamble, no rationalization outdoors the JSON.

Required format:
{"class": "", "confidence": <0.0-1.0>, "reply": ""}"""


# ── Router class ──────────────────────────────────────────────────────────────

class TicketRouter:
    def __init__(self, model_id: str = MODEL_ID):
        print(f"Loading {model_id}...")
        self.tokenizer = AutoTokenizer.from_pretrained(model_id)
        self.mannequin = AutoModelForCausalLM.from_pretrained(
            model_id,
            torch_dtype=torch.bfloat16,
            device_map="auto",
        )
        self.mannequin.eval()
        print(f"Prepared on {self.mannequin.machine}")

    def _call_model(self, ticket: str) -> str:
        """
        Format the ticket right into a chat message, run inference in no_think mode
        (sooner for classification), and return the uncooked decoded output.
        """
        messages = [
            {"role": "system", "content": SYSTEM_PROMPT},
            {"role": "user",   "content": ticket},
        ]
        textual content = self.tokenizer.apply_chat_template(
            messages,
            tokenize=False,
            add_generation_prompt=True,
            enable_thinking=False,   # Quick path -- no chain-of-thought for routine classification
        )
        inputs = self.tokenizer(textual content, return_tensors="pt").to(self.mannequin.machine)

        with torch.no_grad():
            output_ids = self.mannequin.generate(
                **inputs,
                max_new_tokens=256,
                temperature=0.3,   # Decrease temp for classification -- extra deterministic output
                top_p=0.9,
                do_sample=True,
            )

        new_tokens = output_ids[0][inputs["input_ids"].form[-1]:]
        return self.tokenizer.decode(new_tokens, skip_special_tokens=True).strip()

    def _parse_output(self, uncooked: str) -> dict:
        """
        Extract the JSON object from the mannequin's output.
        Falls again to a default 'basic' class with zero confidence if parsing fails.
        This prevents a JSON parse failure from crashing the pipeline.
        """
        # Discover any JSON object within the output, even when surrounded by stray textual content
        match = re.search(r"{.*?}", uncooked, re.DOTALL)
        if not match:
            return {"class": "basic", "confidence": 0.0, "reply": uncooked}
        strive:
            return json.hundreds(match.group())
        besides json.JSONDecodeError:
            return {"class": "basic", "confidence": 0.0, "reply": uncooked}

    def route(self, ticket: str) -> RoutingResult:
        """
        Route a single ticket. Returns a RoutingResult with classification,
        confidence, reply, and escalation flag.
        """
        uncooked = self._call_model(ticket)
        parsed = self._parse_output(uncooked)

        class   = parsed.get("class", "basic")
        confidence = float(parsed.get("confidence", 0.0))
        reply      = parsed.get("reply", "Thanks for reaching out. We'll observe up shortly.")

        return RoutingResult(
            ticket=ticket,
            class=class,
            confidence=confidence,
            reply=reply,
            escalate=confidence < ESCALATE_AT,
            raw_output=uncooked,
        )

    def route_batch(self, tickets: listing[str]) -> listing[RoutingResult]:
        """Route a listing of tickets sequentially. Returns ends in enter order."""
        return [self.route(t) for t in tickets]


# ── Run it ────────────────────────────────────────────────────────────────────

if __name__ == "__main__":
    router = TicketRouter()

    test_tickets = [
        "I was charged twice for my subscription this month. Please refund the duplicate charge.",
        "L'application se bloque chaque fois que j'essaie d'exporter un fichier PDF.",   # French
        "No puedo iniciar sesión en mi cuenta desde hace dos días.",                     # Spanish
        "Die Rechnung für März fehlt in meinem Abrechnungsbereich.",                     # German
        "Il mio abbonamento non si rinnova automaticamente nonostante il pagamento.",    # Italian
    ]

    print("n" + "=" * 70)
    outcomes = router.route_batch(test_tickets)

    for r in outcomes:
        flag = "🔴 ESCALATE" if r.escalate else "🟢 AUTO"
        print(f"n{flag}")
        print(f"Ticket     : {r.ticket[:70]}...")
        print(f"Class   : {r.class}")
        print(f"Confidence : {r.confidence:.2f}")
        print(f"Reply      : {r.reply[:100]}...")

    escalated = [r for r in results if r.escalate]
    print(f"n{'─'*70}")
    print(f"Whole tickets : {len(outcomes)}")
    print(f"Auto-routed   : {len(outcomes) - len(escalated)}")
    print(f"Escalated     : {len(escalated)}")

# tool_calling.py
# Conditions: transformers>=4.53.0, torch, speed up
# Run: python tool_calling.py

import re
import json
import torch
from transformers import AutoTokenizer, AutoModelForCausalLM

MODEL_ID  = "HuggingFaceTB/SmolLM3-3B"
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
mannequin     = AutoModelForCausalLM.from_pretrained(
    MODEL_ID, torch_dtype=torch.bfloat16, device_map="auto"
)
mannequin.eval()

# ── Device definition ───────────────────────────────────────────────────────────
# SmolLM3 accepts instrument definitions as JSON Schema objects below xml_tools.
# The mannequin makes use of the title and outline to resolve when to name the instrument.
# The parameters schema tells it what arguments to incorporate within the name.

TOOLS = [
    {
        "name": "lookup_order_status",
        "description": (
            "Look up the current status, estimated delivery date, and carrier "
            "for a specific customer order. Call this when the customer mentions "
            "an order number or asks where their order is."
        ),
        "parameters": {
            "type": "object",
            "properties": {
                "order_id": {
                    "type": "string",
                    "description": "The order ID, usually in the format ORD-XXXXXX."
                }
            },
            "required": ["order_id"]
        }
    }
]

# ── Simulated order database ──────────────────────────────────────────────────

def lookup_order_status(order_id: str) -> dict:
    """
    In manufacturing, substitute this with an actual database or API name.
    Returns a dict the mannequin can learn and summarize for the client.
    """
    database = {
        "ORD-4821": {"standing": "shipped",    "eta": "June 18, 2026", "service": "DHL"},
        "ORD-3307": {"standing": "processing", "eta": "June 20, 2026", "service": None},
        "ORD-1190": {"standing": "delivered",  "eta": None,            "service": "FedEx"},
    }
    return database.get(order_id, {"standing": "not_found", "eta": None, "service": None})

# ── Device name parser ──────────────────────────────────────────────────────────

def parse_tool_call(output: str):
    """
    Extract a instrument name from the mannequin's output.
    SmolLM3 emits: {"title": "...", "arguments": {...}}
    Returns (tool_name, arguments) or (None, None) if no instrument name is current.
    """
    match = re.search(r"(.*?)", output, re.DOTALL)
    if not match:
        return None, None
    strive:
        payload = json.hundreds(match.group(1).strip())
        return payload.get("title"), payload.get("arguments", {})
    besides json.JSONDecodeError:
        return None, None

# ── Full tool-call spherical journey ─────────────────────────────────────────────────

def respond_with_tools(user_message: str) -> str:
    """
    Full agentic loop:
    1. Ship consumer message + instrument definitions to the mannequin.
    2. If the mannequin emits a instrument name, execute it and inject the consequence.
    3. Generate the ultimate customer-facing response.
    """
    # Flip 1: give the mannequin the consumer message and out there instruments
    messages = [{"role": "user", "content": user_message}]

    inputs = tokenizer.apply_chat_template(
        messages,
        xml_tools=TOOLS,           # Go instrument definitions right here
        enable_thinking=False,
        add_generation_prompt=True,
        tokenize=True,
        return_tensors="pt",
    ).to(mannequin.machine)

    with torch.no_grad():
        output_ids = mannequin.generate(
            inputs, max_new_tokens=256, temperature=0.3, top_p=0.9, do_sample=True
        )
    turn1 = tokenizer.decode(
        output_ids[0][inputs.shape[-1]:], skip_special_tokens=True
    )

    # Test if the mannequin needs to name a instrument
    tool_name, tool_args = parse_tool_call(turn1)

    if tool_name == "lookup_order_status":
        # Execute the actual perform
        tool_result = lookup_order_status(**tool_args)
        print(f"  [Tool called] {tool_name}({tool_args}) → {tool_result}")

        # Flip 2: inject the instrument consequence and ask for the ultimate response
        messages += [
            {"role": "assistant", "content": turn1},
            {"role": "tool",      "content": json.dumps(tool_result), "name": tool_name},
        ]
        inputs2 = tokenizer.apply_chat_template(
            messages,
            xml_tools=TOOLS,
            enable_thinking=False,
            add_generation_prompt=True,
            tokenize=True,
            return_tensors="pt",
        ).to(mannequin.machine)

        with torch.no_grad():
            output_ids2 = mannequin.generate(
                inputs2, max_new_tokens=256, temperature=0.3, top_p=0.9, do_sample=True
            )
        return tokenizer.decode(
            output_ids2[0][inputs2.shape[-1]:], skip_special_tokens=True
        ).strip()

    # No instrument name -- mannequin answered immediately
    return turn1.strip()


# ── Take a look at it ───────────────────────────────────────────────────────────────────

if __name__ == "__main__":
    queries = [
        "Where is my order ORD-4821? It's been a week.",
        "My order ORD-3307 hasn't shipped yet -- what's the status?",
        "I just want to change my email address.",  # No tool needed
    ]

    for question in queries:
        print(f"nCustomer : {question}")
        response = respond_with_tools(question)
        print(f"Agent    : {response}")

# finetune.py
# Further stipulations: pip set up trl>=0.9.0 peft>=0.11.0 datasets>=2.19.0
# Run: python finetune.py
# Time: ~8-12 minutes on an RTX 3060 for 3 epochs over 50 examples

import json
import torch
from datasets import Dataset
from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
from peft import LoraConfig, get_peft_model
from trl import SFTTrainer, SFTConfig

MODEL_ID   = "HuggingFaceTB/SmolLM3-3B"
OUTPUT_DIR = "./smollm3-ticket-router"

# ── System immediate (identical because the inference router) ──────────────────────────────

SYSTEM_PROMPT = """You're a multilingual buyer assist router for a SaaS firm.
Classify the assist ticket and generate a useful reply in the identical language because the ticket.
Reply ONLY with JSON: {"class": "", "confidence": <0.0-1.0>, "reply": ""}"""

# ── Coaching information ─────────────────────────────────────────────────────────────
# In manufacturing you'll load a whole bunch of actual labelled tickets.
# This minimal set demonstrates the format -- increase along with your actual information.

raw_examples = [
    ("I was charged twice for my subscription.", "billing",
     "We're sorry for the duplicate charge. Our billing team will review and issue a refund within 3-5 business days."),
    ("The app crashes every time I try to export a PDF.", "technical",
     "We apologize for the inconvenience. Our engineering team has been notified and will investigate."),
    ("I can't log into my account since yesterday.", "account",
     "We're sorry you're having trouble. Please try resetting your password. If the issue continues, we'll escalate to our account team."),
    ("Die App stürzt beim Exportieren von PDFs ab.", "technical",
     "Wir entschuldigen uns für die Unannehmlichkeiten. Unser Technikteam wurde benachrichtigt und untersucht das Problem."),
    ("L'application se bloque quand j'exporte un fichier.", "technical",
     "Nous nous excusons pour la gêne occasionnée. Notre équipe technique a été informée et travaille sur ce problème."),
    ("My March invoice is missing from the billing section.", "billing",
     "Thank you for flagging this. Our billing team will locate your March invoice and resend it within 24 hours."),
    ("No puedo iniciar sesión desde ayer por la noche.", "account",
     "Lamentamos el problema de acceso. Por favor, restablezca su contraseña. Si el problema persiste, escalaremos su caso."),
    ("How do I upgrade my plan to the Pro tier?", "general",
     "You can upgrade to Pro directly from Settings → Subscription. The new rate applies from your next billing cycle."),
]

def format_example(ticket: str, class: str, reply: str) -> dict:
    """
    Format a single instance into the SmolLM3 messages format.
    The assistant flip incorporates the goal JSON the mannequin ought to study to supply.
    """
    return {
        "messages": [
            {"role": "system",    "content": SYSTEM_PROMPT},
            {"role": "user",      "content": ticket},
            {"role": "assistant", "content": json.dumps({
                "category": category, "confidence": 0.95, "reply": reply
            })},
        ]
    }

dataset = Dataset.from_list([format_example(*ex) for ex in raw_examples])

# ── Tokenizer ─────────────────────────────────────────────────────────────────

tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
tokenizer.pad_token = tokenizer.eos_token   # SmolLM3 has no separate pad token

# ── Mannequin (4-bit quantized base for QLoRA) ────────────────────────────────────

bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_use_double_quant=True,
    bnb_4bit_quant_type="nf4",
)
mannequin = AutoModelForCausalLM.from_pretrained(
    MODEL_ID,
    quantization_config=bnb_config,
    device_map="auto",
)

# ── LoRA config ───────────────────────────────────────────────────────────────
# We goal the eye and MLP projection layers -- these carry probably the most
# task-specific sign and provides the most effective accuracy/parameter trade-off.

lora_config = LoraConfig(
    r=16,              # Rank of the LoRA replace matrices -- increased = extra expressive, extra reminiscence
    lora_alpha=32,     # Scaling issue; conventionally set to 2*r
    lora_dropout=0.05,
    bias="none",
    task_type="CAUSAL_LM",
    target_modules=[
        "q_proj", "k_proj", "v_proj", "o_proj",   # Attention projections
        "gate_proj", "up_proj", "down_proj",        # MLP projections (SwiGLU)
    ],
)
mannequin = get_peft_model(mannequin, lora_config)
mannequin.print_trainable_parameters()
# Anticipated: trainable params: ~13M (0.4% of 3B complete)

# ── Coaching config ───────────────────────────────────────────────────────────

sft_config = SFTConfig(
    output_dir=OUTPUT_DIR,
    num_train_epochs=3,
    per_device_train_batch_size=2,
    gradient_accumulation_steps=4,   # Efficient batch measurement = 8
    learning_rate=2e-4,
    warmup_ratio=0.1,
    lr_scheduler_type="cosine",
    bf16=True,
    logging_steps=5,
    save_strategy="epoch",
    max_seq_length=512,              # Tickets are quick -- no want for the complete context window
)

# ── Prepare ─────────────────────────────────────────────────────────────────────

coach = SFTTrainer(
    mannequin=mannequin,
    tokenizer=tokenizer,
    train_dataset=dataset,
    args=sft_config,
)
coach.prepare()

# ── Save and merge ────────────────────────────────────────────────────────────
# Save the LoRA adapter -- small file, straightforward to share or model.
coach.save_model(f"{OUTPUT_DIR}/adapter")

# Merge the adapter again into the bottom mannequin weights for standalone deployment.
# The merged mannequin hundreds precisely like the bottom mannequin -- no PEFT dependency at inference.
merged = mannequin.merge_and_unload()
merged.save_pretrained(f"{OUTPUT_DIR}/merged")
tokenizer.save_pretrained(f"{OUTPUT_DIR}/merged")

print(f"nFine-tuned mannequin saved to {OUTPUT_DIR}/merged")
print("Load it with: AutoModelForCausalLM.from_pretrained('./smollm3-ticket-router/merged')")

pip set up trl>=0.9.0 peft>=0.11.0 datasets>=2.19.0
python finetune.py

trainable params: 13,631,488 || all params: 3,085,123,584 || trainable%: 0.4420
{'loss': 1.842, 'learning_rate': 2e-04, 'epoch': 0.5}
{'loss': 0.923, 'learning_rate': 1.4e-04, 'epoch': 1.0}
{'loss': 0.461, 'learning_rate': 6e-05, 'epoch': 2.0}
{'loss': 0.287, 'learning_rate': 0.0, 'epoch': 3.0}