feat: implement AI model API proxy service
This commit is contained in:
8
backend/app/routes/__init__.py
Normal file
8
backend/app/routes/__init__.py
Normal file
@@ -0,0 +1,8 @@
|
||||
"""API routes."""
|
||||
from fastapi import APIRouter
|
||||
from .health import router as health_router
|
||||
from .chat import router as chat_router
|
||||
from .openai import router as openai_router
|
||||
from .anthropic import router as anthropic_router
|
||||
|
||||
__all__ = ["health_router", "chat_router", "openai_router", "anthropic_router"]
|
||||
108
backend/app/routes/anthropic.py
Normal file
108
backend/app/routes/anthropic.py
Normal file
@@ -0,0 +1,108 @@
|
||||
"""Anthropic native endpoint (passthrough)."""
|
||||
import time
|
||||
from typing import Optional, List
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi.responses import StreamingResponse
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ..models import ClientKey
|
||||
from ..middleware.auth import get_current_client
|
||||
from ..services.router import ModelRouter
|
||||
from ..services.key_selector import key_selector
|
||||
from ..services.logger import async_logger, LogEntry
|
||||
from ..providers.litellm_wrapper import LiteLLMWrapper
|
||||
|
||||
router = APIRouter(prefix="/v1/anthropic", tags=["anthropic"])
|
||||
|
||||
|
||||
class Message(BaseModel):
|
||||
role: str
|
||||
content: str
|
||||
|
||||
|
||||
class AnthropicRequest(BaseModel):
|
||||
model: str
|
||||
messages: List[Message]
|
||||
max_tokens: int = 1024
|
||||
stream: bool = False
|
||||
temperature: Optional[float] = None
|
||||
|
||||
|
||||
@router.post("/messages")
|
||||
async def anthropic_messages(
|
||||
request: AnthropicRequest,
|
||||
client: ClientKey = Depends(get_current_client)
|
||||
):
|
||||
"""Anthropic native messages endpoint."""
|
||||
start_time = time.time()
|
||||
|
||||
# Get Anthropic provider
|
||||
provider = await ModelRouter.get_provider("anthropic")
|
||||
if not provider:
|
||||
raise HTTPException(status_code=503, detail={"error": "Anthropic provider not configured"})
|
||||
|
||||
# Get next API key
|
||||
provider_key = await key_selector.get_next_key(provider.id)
|
||||
if not provider_key:
|
||||
raise HTTPException(status_code=503, detail={"error": "No available Anthropic API key"})
|
||||
|
||||
messages = [{"role": m.role, "content": m.content} for m in request.messages]
|
||||
|
||||
kwargs = {"max_tokens": request.max_tokens}
|
||||
if request.temperature is not None:
|
||||
kwargs["temperature"] = request.temperature
|
||||
|
||||
try:
|
||||
response, prompt_tokens, completion_tokens = await LiteLLMWrapper.chat_completion(
|
||||
provider=provider,
|
||||
api_key=provider_key.key,
|
||||
model=request.model,
|
||||
messages=messages,
|
||||
stream=request.stream,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
if request.stream:
|
||||
return StreamingResponse(
|
||||
LiteLLMWrapper.stream_response(
|
||||
response, client.id, provider.id, request.model, start_time
|
||||
),
|
||||
media_type="text/event-stream"
|
||||
)
|
||||
else:
|
||||
latency_ms = int((time.time() - start_time) * 1000)
|
||||
await async_logger.log(LogEntry(
|
||||
client_key_id=client.id,
|
||||
provider_id=provider.id,
|
||||
model=request.model,
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
latency_ms=latency_ms,
|
||||
success=True
|
||||
))
|
||||
return response
|
||||
|
||||
except Exception as e:
|
||||
latency_ms = int((time.time() - start_time) * 1000)
|
||||
await async_logger.log(LogEntry(
|
||||
client_key_id=client.id,
|
||||
provider_id=provider.id,
|
||||
model=request.model,
|
||||
latency_ms=latency_ms,
|
||||
success=False,
|
||||
error_message=str(e)
|
||||
))
|
||||
raise HTTPException(status_code=500, detail={"error": str(e)})
|
||||
|
||||
|
||||
@router.get("/models")
|
||||
async def anthropic_models(client: ClientKey = Depends(get_current_client)):
|
||||
"""List Anthropic models."""
|
||||
return {
|
||||
"object": "list",
|
||||
"data": [
|
||||
{"id": "claude-3-opus-20240229", "object": "model", "owned_by": "anthropic"},
|
||||
{"id": "claude-3-sonnet-20240229", "object": "model", "owned_by": "anthropic"},
|
||||
{"id": "claude-3-haiku-20240307", "object": "model", "owned_by": "anthropic"},
|
||||
]
|
||||
}
|
||||
135
backend/app/routes/chat.py
Normal file
135
backend/app/routes/chat.py
Normal file
@@ -0,0 +1,135 @@
|
||||
"""OpenAI-compatible chat completion endpoint."""
|
||||
import time
|
||||
from typing import Optional, List, Dict, Any
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi.responses import StreamingResponse
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ..models import ClientKey
|
||||
from ..middleware.auth import get_current_client
|
||||
from ..services.router import model_router
|
||||
from ..services.key_selector import key_selector
|
||||
from ..services.logger import async_logger, LogEntry
|
||||
from ..providers.litellm_wrapper import LiteLLMWrapper
|
||||
|
||||
router = APIRouter(prefix="/v1", tags=["chat"])
|
||||
|
||||
|
||||
class ChatMessage(BaseModel):
|
||||
"""Chat message."""
|
||||
role: str
|
||||
content: str
|
||||
|
||||
|
||||
class ChatCompletionRequest(BaseModel):
|
||||
"""Chat completion request."""
|
||||
model: str
|
||||
messages: List[ChatMessage]
|
||||
stream: bool = False
|
||||
temperature: Optional[float] = None
|
||||
max_tokens: Optional[int] = None
|
||||
top_p: Optional[float] = None
|
||||
|
||||
|
||||
@router.post("/chat/completions")
|
||||
async def chat_completions(
|
||||
request: ChatCompletionRequest,
|
||||
client: ClientKey = Depends(get_current_client)
|
||||
):
|
||||
"""OpenAI-compatible chat completions endpoint."""
|
||||
start_time = time.time()
|
||||
|
||||
# Route model to provider
|
||||
provider, error = await model_router.route_model(request.model)
|
||||
if error:
|
||||
raise HTTPException(status_code=400, detail={"error": error})
|
||||
|
||||
# Get next API key
|
||||
provider_key = await key_selector.get_next_key(provider.id)
|
||||
if not provider_key:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail={"error": f"No available API key for provider: {provider.name}"}
|
||||
)
|
||||
|
||||
# Build messages
|
||||
messages = [{"role": m.role, "content": m.content} for m in request.messages]
|
||||
|
||||
# Build kwargs
|
||||
kwargs = {}
|
||||
if request.temperature is not None:
|
||||
kwargs["temperature"] = request.temperature
|
||||
if request.max_tokens is not None:
|
||||
kwargs["max_tokens"] = request.max_tokens
|
||||
if request.top_p is not None:
|
||||
kwargs["top_p"] = request.top_p
|
||||
|
||||
try:
|
||||
response, prompt_tokens, completion_tokens = await LiteLLMWrapper.chat_completion(
|
||||
provider=provider,
|
||||
api_key=provider_key.key,
|
||||
model=request.model,
|
||||
messages=messages,
|
||||
stream=request.stream,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
if request.stream:
|
||||
return StreamingResponse(
|
||||
LiteLLMWrapper.stream_response(
|
||||
response, client.id, provider.id, request.model, start_time
|
||||
),
|
||||
media_type="text/event-stream"
|
||||
)
|
||||
else:
|
||||
# Log successful request
|
||||
latency_ms = int((time.time() - start_time) * 1000)
|
||||
await async_logger.log(LogEntry(
|
||||
client_key_id=client.id,
|
||||
provider_id=provider.id,
|
||||
model=request.model,
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
latency_ms=latency_ms,
|
||||
success=True
|
||||
))
|
||||
|
||||
# Return response
|
||||
return response
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
# Log failed request
|
||||
latency_ms = int((time.time() - start_time) * 1000)
|
||||
await async_logger.log(LogEntry(
|
||||
client_key_id=client.id,
|
||||
provider_id=provider.id,
|
||||
model=request.model,
|
||||
latency_ms=latency_ms,
|
||||
success=False,
|
||||
error_message=str(e)
|
||||
))
|
||||
|
||||
# Handle specific errors
|
||||
error_str = str(e).lower()
|
||||
if "rate" in error_str or "limit" in error_str:
|
||||
raise HTTPException(status_code=429, detail={"error": str(e)})
|
||||
raise HTTPException(status_code=500, detail={"error": str(e)})
|
||||
|
||||
|
||||
@router.get("/models")
|
||||
async def list_models(client: ClientKey = Depends(get_current_client)):
|
||||
"""List available models."""
|
||||
return {
|
||||
"object": "list",
|
||||
"data": [
|
||||
{"id": "gpt-4o", "object": "model", "owned_by": "openai"},
|
||||
{"id": "gpt-4o-mini", "object": "model", "owned_by": "openai"},
|
||||
{"id": "o1", "object": "model", "owned_by": "openai"},
|
||||
{"id": "o1-mini", "object": "model", "owned_by": "openai"},
|
||||
{"id": "claude-3-opus", "object": "model", "owned_by": "anthropic"},
|
||||
{"id": "claude-3-sonnet", "object": "model", "owned_by": "anthropic"},
|
||||
{"id": "claude-3-haiku", "object": "model", "owned_by": "anthropic"},
|
||||
]
|
||||
}
|
||||
10
backend/app/routes/health.py
Normal file
10
backend/app/routes/health.py
Normal file
@@ -0,0 +1,10 @@
|
||||
"""Health check endpoint."""
|
||||
from fastapi import APIRouter
|
||||
|
||||
router = APIRouter(tags=["health"])
|
||||
|
||||
|
||||
@router.get("/health")
|
||||
async def health_check():
|
||||
"""Health check endpoint."""
|
||||
return {"status": "ok"}
|
||||
111
backend/app/routes/openai.py
Normal file
111
backend/app/routes/openai.py
Normal file
@@ -0,0 +1,111 @@
|
||||
"""OpenAI native endpoint (passthrough)."""
|
||||
import time
|
||||
from typing import Optional, List
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi.responses import StreamingResponse
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ..models import ClientKey
|
||||
from ..middleware.auth import get_current_client
|
||||
from ..services.router import ModelRouter
|
||||
from ..services.key_selector import key_selector
|
||||
from ..services.logger import async_logger, LogEntry
|
||||
from ..providers.litellm_wrapper import LiteLLMWrapper
|
||||
|
||||
router = APIRouter(prefix="/v1/openai", tags=["openai"])
|
||||
|
||||
|
||||
class ChatMessage(BaseModel):
|
||||
role: str
|
||||
content: str
|
||||
|
||||
|
||||
class ChatRequest(BaseModel):
|
||||
model: str
|
||||
messages: List[ChatMessage]
|
||||
stream: bool = False
|
||||
temperature: Optional[float] = None
|
||||
max_tokens: Optional[int] = None
|
||||
|
||||
|
||||
@router.post("/chat/completions")
|
||||
async def openai_chat(
|
||||
request: ChatRequest,
|
||||
client: ClientKey = Depends(get_current_client)
|
||||
):
|
||||
"""OpenAI native chat completions (passthrough)."""
|
||||
start_time = time.time()
|
||||
|
||||
# Get OpenAI provider
|
||||
provider = await ModelRouter.get_provider("openai")
|
||||
if not provider:
|
||||
raise HTTPException(status_code=503, detail={"error": "OpenAI provider not configured"})
|
||||
|
||||
# Get next API key
|
||||
provider_key = await key_selector.get_next_key(provider.id)
|
||||
if not provider_key:
|
||||
raise HTTPException(status_code=503, detail={"error": "No available OpenAI API key"})
|
||||
|
||||
messages = [{"role": m.role, "content": m.content} for m in request.messages]
|
||||
|
||||
kwargs = {}
|
||||
if request.temperature is not None:
|
||||
kwargs["temperature"] = request.temperature
|
||||
if request.max_tokens is not None:
|
||||
kwargs["max_tokens"] = request.max_tokens
|
||||
|
||||
try:
|
||||
response, prompt_tokens, completion_tokens = await LiteLLMWrapper.chat_completion(
|
||||
provider=provider,
|
||||
api_key=provider_key.key,
|
||||
model=request.model,
|
||||
messages=messages,
|
||||
stream=request.stream,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
if request.stream:
|
||||
return StreamingResponse(
|
||||
LiteLLMWrapper.stream_response(
|
||||
response, client.id, provider.id, request.model, start_time
|
||||
),
|
||||
media_type="text/event-stream"
|
||||
)
|
||||
else:
|
||||
latency_ms = int((time.time() - start_time) * 1000)
|
||||
await async_logger.log(LogEntry(
|
||||
client_key_id=client.id,
|
||||
provider_id=provider.id,
|
||||
model=request.model,
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
latency_ms=latency_ms,
|
||||
success=True
|
||||
))
|
||||
return response
|
||||
|
||||
except Exception as e:
|
||||
latency_ms = int((time.time() - start_time) * 1000)
|
||||
await async_logger.log(LogEntry(
|
||||
client_key_id=client.id,
|
||||
provider_id=provider.id,
|
||||
model=request.model,
|
||||
latency_ms=latency_ms,
|
||||
success=False,
|
||||
error_message=str(e)
|
||||
))
|
||||
raise HTTPException(status_code=500, detail={"error": str(e)})
|
||||
|
||||
|
||||
@router.get("/models")
|
||||
async def openai_models(client: ClientKey = Depends(get_current_client)):
|
||||
"""List OpenAI models."""
|
||||
return {
|
||||
"object": "list",
|
||||
"data": [
|
||||
{"id": "gpt-4o", "object": "model", "owned_by": "openai"},
|
||||
{"id": "gpt-4o-mini", "object": "model", "owned_by": "openai"},
|
||||
{"id": "o1", "object": "model", "owned_by": "openai"},
|
||||
{"id": "o1-mini", "object": "model", "owned_by": "openai"},
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user