File size: 4,710 Bytes
e3a7b6f
 
 
1608585
e3a7b6f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1608585
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e3a7b6f
 
 
 
 
 
 
5e2b717
e3a7b6f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1608585
 
 
 
 
 
 
 
 
 
 
 
 
e3a7b6f
1608585
 
e3a7b6f
1608585
e3a7b6f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5e2b717
 
 
e3a7b6f
 
 
 
 
 
 
1608585
5e2b717
e3a7b6f
 
 
 
1608585
e3a7b6f
 
 
1608585
e3a7b6f
 
 
 
 
1608585
e3a7b6f
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
from fastapi import FastAPI, HTTPException
from pydantic import BaseModel
from llama_cpp import Llama
from concurrent.futures import ProcessPoolExecutor, as_completed
import uvicorn
from dotenv import load_dotenv
from difflib import SequenceMatcher
from tqdm import tqdm

load_dotenv()

app = FastAPI()

# Configuraci贸n de los modelos
models = [
    {"repo_id": "Ffftdtd5dtft/gpt2-xl-Q2_K-GGUF", "filename": "gpt2-xl-q2_k.gguf"},
    {"repo_id": "Ffftdtd5dtft/Meta-Llama-3.1-8B-Instruct-Q2_K-GGUF", "filename": "meta-llama-3.1-8b-instruct-q2_k.gguf"},
    {"repo_id": "Ffftdtd5dtft/gemma-2-9b-it-Q2_K-GGUF", "filename": "gemma-2-9b-it-q2_k.gguf"},
    {"repo_id": "Ffftdtd5dtft/gemma-2-27b-Q2_K-GGUF", "filename": "gemma-2-27b-q2_k.gguf"},
]

# Funci贸n para cargar un modelo
def load_model(model_config):
    return Llama.from_pretrained(repo_id=model_config['repo_id'], filename=model_config['filename'])

# Cargar modelos en paralelo
def load_all_models():
    with ProcessPoolExecutor() as executor:
        future_to_model = {executor.submit(load_model, model): model for model in models}
        loaded_models = {}
        for future in as_completed(future_to_model):
            model = future_to_model[future]
            try:
                loaded_models[model['repo_id']] = future.result()
                print(f"Modelo cargado en RAM: {model['repo_id']}")
            except Exception as exc:
                print(f"Error al cargar modelo {model['repo_id']}: {exc}")
    return loaded_models

# Cargar modelos en memoria
llms = load_all_models()

class ChatRequest(BaseModel):
    message: str
    top_k: int = 50
    top_p: float = 0.95
    temperature: float = 0.7

# Funci贸n global para generar respuestas de chat
def generate_chat_response(request, llm):
    try:
        user_input = normalize_input(request.message)
        response = llm.create_chat_completion(
            messages=[{"role": "user", "content": user_input}],
            top_k=request.top_k,
            top_p=request.top_p,
            temperature=request.temperature
        )
        reply = response['choices'][0]['message']['content']
        return {"response": reply, "literal": user_input}
    except Exception as e:
        return {"response": f"Error: {str(e)}", "literal": user_input}

def normalize_input(input_text):
    return input_text.strip()

def filter_duplicates(responses):
    seen = set()
    unique_responses = []
    for response in responses:
        lines = response.split('\n')
        unique_lines = set()
        for line in lines:
            if line not in seen:
                seen.add(line)
                unique_lines.add(line)
        unique_responses.append('\n'.join(unique_lines))
    return unique_responses

def select_best_response(responses):
    # Eliminar respuestas duplicadas
    unique_responses = filter_duplicates(responses)
    # Deduplicar respuestas
    unique_responses = list(set(unique_responses))
    # Filtrar respuestas coherentes
    coherent_responses = filter_by_coherence(unique_responses)
    # Seleccionar la mejor respuesta
    best_response = filter_by_similarity(coherent_responses)
    return best_response

def filter_by_coherence(responses):
    # Implementa aqu铆 un filtro de coherencia si es necesario
    return responses

def filter_by_similarity(responses):
    responses.sort(key=len, reverse=True)
    best_response = responses[0]
    for i in range(1, len(responses)):
        ratio = SequenceMatcher(None, best_response, responses[i]).ratio()
        if ratio < 0.9:
            best_response = responses[i]
            break
    return best_response

def worker_function(llm, request):
    return generate_chat_response(request, llm)

@app.post("/generate_chat")
async def generate_chat(request: ChatRequest):
    if not request.message.strip():
        raise HTTPException(status_code=400, detail="The message cannot be empty.")
    
    print(f"Procesando solicitud: {request.message}")

    with ProcessPoolExecutor() as executor:
        futures = [executor.submit(worker_function, llm, request) for llm in llms.values()]
        responses = []

        for future in tqdm(as_completed(futures), total=len(futures), desc="Generando respuestas"):
            response = future.result()
            responses.append(response['response'])
            print(f"Modelo procesado: {response['literal'][:30]}...")

    # Seleccionar la mejor respuesta
    best_response = select_best_response(responses)
    
    print(f"Mejor respuesta seleccionada: {best_response}")

    return {
        "best_response": best_response,
        "all_responses": responses
    }

if __name__ == "__main__":
    uvicorn.run(app, host="0.0.0.0", port=7860)