Create main.py

main.py

@@ -0,0 +1,45 @@
+from fastapi import FastAPI, Request
+from llama_cpp import Llama
+from pydantic import BaseModel
+from typing import List
+import uvicorn
+
+app = FastAPI()
+
+# Load small model (e.g., Phi-2 or DeepSeek)
+llm = Llama(model_path="phi-2.Q4_K_M.gguf", n_ctx=2048, n_threads=2)
+
+class Message(BaseModel):
+    role: str
+    content: str
+
+class ChatRequest(BaseModel):
+    model: str
+    messages: List[Message]
+    temperature: float = 0.7
+    max_tokens: int = 256
+    stream: bool = False
+
+@app.post("/v1/chat/completions")
+async def chat_completions(req: ChatRequest):
+    prompt = "\n".join([f"{m.role}: {m.content}" for m in req.messages]) + "\nassistant:"
+    output = llm(
+        prompt,
+        max_tokens=req.max_tokens,
+        temperature=req.temperature,
+        stop=["user:", "assistant:"]
+    )
+    text = output["choices"][0]["text"]
+    return {
+        "id": "chatcmpl-123",
+        "object": "chat.completion",
+        "choices": [{
+            "index": 0,
+            "message": {"role": "assistant", "content": text},
+            "finish_reason": "stop"
+        }],
+        "model": req.model
+    }
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=8000)