from config.logger import setup_logging from openai import OpenAI import json from core.providers.llm.base import LLMProviderBase TAG = __name__ logger = setup_logging() class LLMProvider(LLMProviderBase): def __init__(self, config): self.model_name = config.get("model_name") self.base_url = config.get("base_url", "http://localhost:9997") # Initialize OpenAI client with Xinference base URL # 如果没有v1,增加v1 if not self.base_url.endswith("/v1"): self.base_url = f"{self.base_url}/v1" logger.bind(tag=TAG).info( f"Initializing Xinference LLM provider with model: {self.model_name}, base_url: {self.base_url}" ) try: self.client = OpenAI( base_url=self.base_url, api_key="xinference", # Xinference has a similar setup to Ollama where it doesn't need an actual key ) logger.bind(tag=TAG).info("Xinference client initialized successfully") except Exception as e: logger.bind(tag=TAG).error(f"Error initializing Xinference client: {e}") raise def response(self, session_id, dialogue, **kwargs): logger.bind(tag=TAG).debug( f"Sending request to Xinference with model: {self.model_name}, dialogue length: {len(dialogue)}" ) responses = self.client.chat.completions.create( model=self.model_name, messages=dialogue, stream=True ) is_active = True for chunk in responses: try: delta = ( chunk.choices[0].delta if getattr(chunk, "choices", None) else None ) content = delta.content if hasattr(delta, "content") else "" if content: if "" in content: is_active = False content = content.split("")[0] if "" in content: is_active = True content = content.split("")[-1] if is_active: yield content except Exception as e: logger.bind(tag=TAG).error(f"Error processing chunk: {e}") def response_with_functions(self, session_id, dialogue, functions=None): logger.bind(tag=TAG).debug( f"Sending function call request to Xinference with model: {self.model_name}, dialogue length: {len(dialogue)}" ) if functions: logger.bind(tag=TAG).debug( f"Function calls enabled with: {[f.get('function', {}).get('name') for f in functions]}" ) stream = self.client.chat.completions.create( model=self.model_name, messages=dialogue, stream=True, tools=functions, ) try: for chunk in stream: delta = chunk.choices[0].delta content = delta.content tool_calls = delta.tool_calls if content: yield content, tool_calls elif tool_calls: yield None, tool_calls finally: stream.close()