File size: 4,742 Bytes
e3a7b6f
 
 
a17dc9a
 
e3a7b6f
 
 
e6dda1e
e3a7b6f
 
 
 
 
 
a17dc9a
e3a7b6f
 
 
 
 
 
a17dc9a
1608585
 
 
a17dc9a
1608585
a17dc9a
 
 
 
1608585
 
 
e3a7b6f
 
 
 
 
 
 
a17dc9a
e3a7b6f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1608585
 
 
 
 
 
 
 
 
 
 
 
 
e3a7b6f
1608585
 
e3a7b6f
1608585
e3a7b6f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
a17dc9a
 
 
 
5e2b717
e3a7b6f
 
 
 
 
 
 
e6dda1e
a17dc9a
 
 
 
 
 
 
 
 
 
 
 
 
e3a7b6f
 
a17dc9a
e3a7b6f
 
 
 
 
a17dc9a
e3a7b6f
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
from fastapi import FastAPI, HTTPException
from pydantic import BaseModel
from llama_cpp import Llama
from concurrent.futures import ThreadPoolExecutor, as_completed
from tqdm import tqdm
import uvicorn
from dotenv import load_dotenv
from difflib import SequenceMatcher
import threading

load_dotenv()

app = FastAPI()

# Configuraci贸n de los modelos
model_configs = [
    {"repo_id": "Ffftdtd5dtft/gpt2-xl-Q2_K-GGUF", "filename": "gpt2-xl-q2_k.gguf"},
    {"repo_id": "Ffftdtd5dtft/Meta-Llama-3.1-8B-Instruct-Q2_K-GGUF", "filename": "meta-llama-3.1-8b-instruct-q2_k.gguf"},
    {"repo_id": "Ffftdtd5dtft/gemma-2-9b-it-Q2_K-GGUF", "filename": "gemma-2-9b-it-q2_k.gguf"},
    {"repo_id": "Ffftdtd5dtft/gemma-2-27b-Q2_K-GGUF", "filename": "gemma-2-27b-q2_k.gguf"},
]

# Cargar un modelo
def load_model(model_config):
    return Llama.from_pretrained(repo_id=model_config['repo_id'], filename=model_config['filename'])

# Cargar todos los modelos simult谩neamente
def load_all_models():
    with ThreadPoolExecutor(max_workers=len(model_configs)) as executor:
        futures = [executor.submit(load_model, config) for config in model_configs]
        models = [future.result() for future in as_completed(futures)]
    return models

# Cargar modelos en memoria
llms = load_all_models()

class ChatRequest(BaseModel):
    message: str
    top_k: int = 50
    top_p: float = 0.95
    temperature: float = 0.7

# Funci贸n para generar respuestas de chat
def generate_chat_response(request, llm):
    try:
        user_input = normalize_input(request.message)
        response = llm.create_chat_completion(
            messages=[{"role": "user", "content": user_input}],
            top_k=request.top_k,
            top_p=request.top_p,
            temperature=request.temperature
        )
        reply = response['choices'][0]['message']['content']
        return {"response": reply, "literal": user_input}
    except Exception as e:
        return {"response": f"Error: {str(e)}", "literal": user_input}

def normalize_input(input_text):
    return input_text.strip()

def filter_duplicates(responses):
    seen = set()
    unique_responses = []
    for response in responses:
        lines = response.split('\n')
        unique_lines = set()
        for line in lines:
            if line not in seen:
                seen.add(line)
                unique_lines.add(line)
        unique_responses.append('\n'.join(unique_lines))
    return unique_responses

def select_best_response(responses):
    # Eliminar respuestas duplicadas
    unique_responses = filter_duplicates(responses)
    # Deduplicar respuestas
    unique_responses = list(set(unique_responses))
    # Filtrar respuestas coherentes
    coherent_responses = filter_by_coherence(unique_responses)
    # Seleccionar la mejor respuesta
    best_response = filter_by_similarity(coherent_responses)
    return best_response

def filter_by_coherence(responses):
    # Implementa aqu铆 un filtro de coherencia si es necesario
    return responses

def filter_by_similarity(responses):
    responses.sort(key=len, reverse=True)
    best_response = responses[0]
    for i in range(1, len(responses)):
        ratio = SequenceMatcher(None, best_response, responses[i]).ratio()
        if ratio < 0.9:
            best_response = responses[i]
            break
    return best_response

def worker_function(llm, request, progress_bar):
    response = generate_chat_response(request, llm)
    progress_bar.update(1)
    return response

@app.post("/generate_chat")
async def generate_chat(request: ChatRequest):
    if not request.message.strip():
        raise HTTPException(status_code=400, detail="The message cannot be empty.")
    
    print(f"Procesando solicitud: {request.message}")

    responses = []
    num_models = len(llms)

    # Crear barra de progreso
    with tqdm(total=num_models, desc="Generando respuestas", unit="modelo") as progress_bar:
        # Ejecutar modelos en paralelo
        with ThreadPoolExecutor(max_workers=num_models) as executor:
            futures = [executor.submit(worker_function, llm, request, progress_bar) for llm in llms]
            for future in as_completed(futures):
                try:
                    response = future.result()
                    responses.append(response['response'])
                except Exception as exc:
                    print(f"Error en la generaci贸n de respuesta: {exc}")

    # Seleccionar la mejor respuesta
    best_response = select_best_response(responses)
    
    print(f"Mejor respuesta seleccionada: {best_response}")

    return {
        "best_response": best_response,
        "all_responses": responses
    }

if __name__ == "__main__":
    uvicorn.run(app, host="0.0.0.0", port=7860)