|
|
|
|
|
import asyncio |
|
|
import gradio as gr |
|
|
from fastapi import FastAPI |
|
|
|
|
|
from config import settings |
|
|
from rabbit_base import RabbitBase |
|
|
from listener import RabbitListenerBase |
|
|
from rabbit_repo import RabbitRepo |
|
|
from service import LLMService |
|
|
from runners.base import ILLMRunner |
|
|
|
|
|
|
|
|
|
|
|
class EchoRunner(ILLMRunner): |
|
|
Type = "EchoRunner" |
|
|
async def StartProcess(self, llmServiceObj: dict): pass |
|
|
async def RemoveProcess(self, sessionId: str): pass |
|
|
async def StopRequest(self, sessionId: str): pass |
|
|
async def SendInputAndGetResponse(self, llmServiceObj: dict): pass |
|
|
|
|
|
async def runner_factory(llmServiceObj: dict) -> ILLMRunner: |
|
|
return EchoRunner() |
|
|
|
|
|
|
|
|
|
|
|
publisher = RabbitRepo(external_source="https://space.external") |
|
|
service = LLMService(publisher, runner_factory) |
|
|
|
|
|
|
|
|
async def h_start(data): await service.StartProcess(data or {}) |
|
|
async def h_user(data): await service.UserInput(data or {}) |
|
|
async def h_remove(data): await service.RemoveSession(data or {}) |
|
|
async def h_stop(data): await service.StopRequest(data or {}) |
|
|
async def h_qir(data): await service.QueryIndexResult(data or {}) |
|
|
async def h_getreg(_): await service.GetFunctionRegistry(False) |
|
|
async def h_getreg_f(_): await service.GetFunctionRegistry(True) |
|
|
|
|
|
handlers = { |
|
|
"llmStartSession": h_start, |
|
|
"llmUserInput": h_user, |
|
|
"llmRemoveSession": h_remove, |
|
|
"llmStopRequest": h_stop, |
|
|
"queryIndexResult": h_qir, |
|
|
"getFunctionRegistry": h_getreg, |
|
|
"getFunctionRegistryFiltered": h_getreg_f, |
|
|
} |
|
|
|
|
|
|
|
|
base = RabbitBase() |
|
|
listener = RabbitListenerBase( |
|
|
base, |
|
|
instance_name=settings.RABBIT_INSTANCE_NAME, |
|
|
handlers=handlers, |
|
|
) |
|
|
|
|
|
|
|
|
DECLS = [ |
|
|
{"ExchangeName": f"llmStartSession{settings.SERVICE_ID}", "FuncName": "llmStartSession", |
|
|
"MessageTimeout": 600_000, "RoutingKeys": [settings.RABBIT_ROUTING_KEY]}, |
|
|
{"ExchangeName": f"llmUserInput{settings.SERVICE_ID}", "FuncName": "llmUserInput", |
|
|
"MessageTimeout": 600_000, "RoutingKeys": [settings.RABBIT_ROUTING_KEY]}, |
|
|
{"ExchangeName": f"llmRemoveSession{settings.SERVICE_ID}", "FuncName": "llmRemoveSession", |
|
|
"MessageTimeout": 60_000, "RoutingKeys": [settings.RABBIT_ROUTING_KEY]}, |
|
|
{"ExchangeName": f"llmStopRequest{settings.SERVICE_ID}", "FuncName": "llmStopRequest", |
|
|
"MessageTimeout": 60_000, "RoutingKeys": [settings.RABBIT_ROUTING_KEY]}, |
|
|
{"ExchangeName": f"queryIndexResult{settings.SERVICE_ID}", "FuncName": "queryIndexResult", |
|
|
"MessageTimeout": 60_000, "RoutingKeys": [settings.RABBIT_ROUTING_KEY]}, |
|
|
{"ExchangeName": f"getFunctionRegistry{settings.SERVICE_ID}", "FuncName": "getFunctionRegistry", |
|
|
"MessageTimeout": 60_000, "RoutingKeys": [settings.RABBIT_ROUTING_KEY]}, |
|
|
{"ExchangeName": f"getFunctionRegistryFiltered{settings.SERVICE_ID}", "FuncName": "getFunctionRegistryFiltered", |
|
|
"MessageTimeout": 60_000, "RoutingKeys": [settings.RABBIT_ROUTING_KEY]}, |
|
|
] |
|
|
|
|
|
|
|
|
async def ping(): |
|
|
return "ok" |
|
|
|
|
|
with gr.Blocks() as demo: |
|
|
gr.Markdown("### LLM Runner (Python) listening on RabbitMQ") |
|
|
btn = gr.Button("Ping") |
|
|
out = gr.Textbox() |
|
|
btn.click(ping, inputs=None, outputs=out) |
|
|
|
|
|
|
|
|
app = FastAPI() |
|
|
app = gr.mount_gradio_app(app, demo, path="/") |
|
|
|
|
|
@app.get("/health") |
|
|
async def health(): |
|
|
return {"status": "ok"} |
|
|
|
|
|
@app.on_event("startup") |
|
|
async def on_start(): |
|
|
await publisher.connect() |
|
|
await service.init() |
|
|
await listener.start(DECLS) |
|
|
|
|
|
@app.on_event("shutdown") |
|
|
async def on_stop(): |
|
|
|
|
|
pass |
|
|
|
|
|
|
|
|
if __name__ == "__main__": |
|
|
import uvicorn |
|
|
uvicorn.run(app, host="0.0.0.0", port=7860) |
|
|
|