Benchmark LLMs 2026: GPT-4 vs Claude vs Gemini como Agentes Autónomos [Resultados]

1┌─────────────────────────────────────────────────────────────────┐
2│                        ORCHESTRATOR                              │
3│  (Coordina el loop de simulación y logging)                     │
4├─────────────────────────────────────────────────────────────────┤
5│                              │                                   │
6│      ┌───────────────┐       │       ┌───────────────┐          │
7│      │   GAME ENGINE │◄──────┼──────►│   LLM AGENT   │          │
8│      │               │       │       │               │          │
9│      │ - Estados     │       │       │ - Prompting   │          │
10│      │ - Acciones    │       │       │ - Parsing     │          │
11│      │ - Reglas      │       │       │ - Decisión    │          │
12│      └───────────────┘       │       └───────────────┘          │
13│                              │                                   │
14├─────────────────────────────────────────────────────────────────┤
15│                      METRICS & LOGGING                           │
16│  (Tasa de éxito, pasos, coherencia, logs JSON)                  │
17└─────────────────────────────────────────────────────────────────┘

1from dataclasses import dataclass
2from typing import List, Dict, Optional, Tuple
3 
4@dataclass
5class GameState:
6    """Estado actual del juego."""
7    current_room: str
8    inventory: List[str]
9    visited_rooms: List[str]
10    objective_completed: bool
11    steps_taken: int
12 
13class TextAdventureGame:
14    """Motor de juego de aventuras de texto para evaluación de agentes."""
15 
16    def __init__(self, world_config: Dict):
17        self.rooms = world_config['rooms']
18        self.items = world_config['items']
19        self.goal = world_config['goal']
20        self.max_steps = world_config.get('max_steps', 50)
21        self.state = self._init_state()
22 
23    def _init_state(self) -> GameState:
24        return GameState(
25            current_room='start',
26            inventory=[],
27            visited_rooms=['start'],
28            objective_completed=False,
29            steps_taken=0
30        )
31 
32    def get_available_actions(self) -> List[str]:
33        """Retorna acciones válidas en el estado actual."""
34        room = self.rooms[self.state.current_room]
35        actions = []
36 
37        # Movimiento
38        for direction in room.get('exits', {}).keys():
39            actions.append(f"ir {direction}")
40 
41        # Objetos en la habitación
42        for item in room.get('items', []):
43            if item not in self.state.inventory:
44                actions.append(f"tomar {item}")
45 
46        # Acciones de inventario
47        for item in self.state.inventory:
48            actions.append(f"usar {item}")
49 
50        actions.append("mirar")
51        actions.append("inventario")
52 
53        return actions
54 
55    def step(self, action: str) -> Tuple[str, float, bool]:
56        """
57        Ejecuta una acción y retorna (feedback, reward, done).
58        """
59        self.state.steps_taken += 1
60        action = action.lower().strip()
61 
62        # Parsear acción
63        if action.startswith("ir "):
64            return self._move(action[3:])
65        elif action.startswith("tomar "):
66            return self._take(action[6:])
67        elif action.startswith("usar "):
68            return self._use(action[5:])
69        elif action == "mirar":
70            return self._look(), 0, False
71        elif action == "inventario":
72            return self._inventory(), 0, False
73        else:
74            return "Acción no reconocida.", -0.1, False
75 
76    def _move(self, direction: str) -> Tuple[str, float, bool]:
77        room = self.rooms[self.state.current_room]
78        exits = room.get('exits', {})
79 
80        if direction in exits:
81            new_room = exits[direction]
82            self.state.current_room = new_room
83            if new_room not in self.state.visited_rooms:
84                self.state.visited_rooms.append(new_room)
85                reward = 0.1  # Bonus por exploración
86            else:
87                reward = 0
88            return f"Te mueves hacia {direction}. {self._look()}", reward, False
89        else:
90            return f"No puedes ir hacia {direction}.", -0.1, False
91 
92    def _take(self, item: str) -> Tuple[str, float, bool]:
93        room = self.rooms[self.state.current_room]
94        if item in room.get('items', []):
95            self.state.inventory.append(item)
96            room['items'].remove(item)
97            return f"Tomas {item}.", 0.2, False
98        return f"No hay {item} aquí.", -0.1, False
99 
100    def _use(self, item: str) -> Tuple[str, float, bool]:
101        if item not in self.state.inventory:
102            return f"No tienes {item}.", -0.1, False
103 
104        # Verificar si cumple el objetivo
105        if self._check_goal(item):
106            self.state.objective_completed = True
107            return "¡Objetivo completado!", 1.0, True
108 
109        return f"Usas {item}, pero no pasa nada especial.", 0, False
110 
111    def _check_goal(self, item: str) -> bool:
112        """Verifica si la acción cumple el objetivo."""
113        goal = self.goal
114        if goal['type'] == 'use_item_in_room':
115            return (item == goal['item'] and
116                    self.state.current_room == goal['room'])
117        return False
118 
119    def _look(self) -> str:
120        room = self.rooms[self.state.current_room]
121        desc = room['description']
122        items = room.get('items', [])
123        exits = list(room.get('exits', {}).keys())
124 
125        text = f"{desc}"
126        if items:
127            text += f" Ves: {', '.join(items)}."
128        if exits:
129            text += f" Salidas: {', '.join(exits)}."
130        return text
131 
132    def _inventory(self) -> str:
133        if self.state.inventory:
134            return f"Inventario: {', '.join(self.state.inventory)}"
135        return "Tu inventario está vacío."
136 
137    def is_done(self) -> bool:
138        return (self.state.objective_completed or
139                self.state.steps_taken >= self.max_steps)

1from openai import OpenAI
2from anthropic import Anthropic
3from typing import List, Optional
4import re
5 
6class LLMAgent:
7    """Agente que usa un LLM para tomar decisiones."""
8 
9    def __init__(self,
10                 provider: str = "openai",
11                 model: str = "gpt-4o",
12                 temperature: float = 0.3):
13        self.provider = provider
14        self.model = model
15        self.temperature = temperature
16 
17        if provider == "openai":
18            self.client = OpenAI()
19        elif provider == "anthropic":
20            self.client = Anthropic()
21 
22        self.action_history: List[str] = []
23        self.total_tokens = 0
24 
25    def build_prompt(self,
26                     state_description: str,
27                     available_actions: List[str],
28                     goal: str,
29                     history: List[str]) -> str:
30        """Construye el prompt para el LLM."""
31 
32        history_text = "\n".join(history[-5:]) if history else "Ninguna"
33        actions_text = ", ".join(available_actions)
34 
35        prompt = f"""Eres un agente jugando un juego de aventuras de texto.
36 
37ESTADO ACTUAL:
38{state_description}
39 
40OBJETIVO:
41{goal}
42 
43ACCIONES DISPONIBLES:
44{actions_text}
45 
46HISTORIAL RECIENTE:
47{history_text}
48 
49INSTRUCCIONES:
50- Analiza el estado actual y el objetivo
51- Elige UNA acción de las disponibles
52- Responde SOLO con la acción exacta, sin explicaciones
53 
54ACCIÓN:"""
55 
56        return prompt
57 
58    def choose_action(self,
59                      state_description: str,
60                      available_actions: List[str],
61                      goal: str) -> str:
62        """Elige una acción usando el LLM."""
63 
64        prompt = self.build_prompt(
65            state_description,
66            available_actions,
67            goal,
68            self.action_history
69        )
70 
71        if self.provider == "openai":
72            response = self.client.chat.completions.create(
73                model=self.model,
74                messages=[{"role": "user", "content": prompt}],
75                temperature=self.temperature,
76                max_tokens=50
77            )
78            action = response.choices[0].message.content.strip()
79            self.total_tokens += response.usage.total_tokens
80 
81        elif self.provider == "anthropic":
82            response = self.client.messages.create(
83                model=self.model,
84                max_tokens=50,
85                messages=[{"role": "user", "content": prompt}]
86            )
87            action = response.content[0].text.strip()
88            self.total_tokens += response.usage.input_tokens + response.usage.output_tokens
89 
90        # Parsear y validar acción
91        action = self._parse_action(action, available_actions)
92        self.action_history.append(action)
93 
94        return action
95 
96    def _parse_action(self,
97                      raw_action: str,
98                      available_actions: List[str]) -> str:
99        """Extrae una acción válida de la respuesta del LLM."""
100 
101        raw_action = raw_action.lower().strip()
102 
103        # Coincidencia exacta
104        if raw_action in [a.lower() for a in available_actions]:
105            return raw_action
106 
107        # Buscar coincidencia parcial
108        for action in available_actions:
109            if action.lower() in raw_action:
110                return action.lower()
111 
112        # Si no hay coincidencia, retornar la más similar
113        # (en producción usaríamos fuzzy matching)
114        return available_actions[0].lower()

1import json
2from datetime import datetime
3from dataclasses import dataclass, asdict
4from typing import List, Dict
5 
6@dataclass
7class EvaluationMetrics:
8    """Métricas de evaluación de un episodio."""
9    model: str
10    success: bool
11    steps_taken: int
12    max_steps: int
13    efficiency: float  # objetivo_alcanzado / pasos
14    exploration_rate: float  # habitaciones_visitadas / total
15    invalid_actions: int
16    total_reward: float
17    total_tokens: int
18    cost_usd: float
19    duration_seconds: float
20 
21class AgentEvaluator:
22    """Orquestador para evaluar agentes LLM."""
23 
24    def __init__(self, game: TextAdventureGame, agent: LLMAgent):
25        self.game = game
26        self.agent = agent
27        self.logs: List[Dict] = []
28 
29    def run_episode(self) -> EvaluationMetrics:
30        """Ejecuta un episodio completo y retorna métricas."""
31 
32        start_time = datetime.now()
33        total_reward = 0
34        invalid_actions = 0
35 
36        while not self.game.is_done():
37            # Obtener estado actual
38            state_desc = self.game._look()
39            available = self.game.get_available_actions()
40            goal_desc = f"Usar {self.game.goal['item']} en {self.game.goal['room']}"
41 
42            # Agente elige acción
43            action = self.agent.choose_action(state_desc, available, goal_desc)
44 
45            # Verificar si es válida
46            if action not in [a.lower() for a in available]:
47                invalid_actions += 1
48 
49            # Ejecutar acción
50            feedback, reward, done = self.game.step(action)
51            total_reward += reward
52 
53            # Log
54            self.logs.append({
55                "step": self.game.state.steps_taken,
56                "state": state_desc,
57                "action": action,
58                "feedback": feedback,
59                "reward": reward
60            })
61 
62            if done:
63                break
64 
65        # Calcular métricas
66        duration = (datetime.now() - start_time).total_seconds()
67        total_rooms = len(self.game.rooms)
68        visited = len(self.game.state.visited_rooms)
69 
70        # Estimar coste (precios aproximados 2026)
71        cost = self._estimate_cost(self.agent.total_tokens, self.agent.model)
72 
73        return EvaluationMetrics(
74            model=self.agent.model,
75            success=self.game.state.objective_completed,
76            steps_taken=self.game.state.steps_taken,
77            max_steps=self.game.max_steps,
78            efficiency=1.0 / self.game.state.steps_taken if self.game.state.objective_completed else 0,
79            exploration_rate=visited / total_rooms,
80            invalid_actions=invalid_actions,
81            total_reward=total_reward,
82            total_tokens=self.agent.total_tokens,
83            cost_usd=cost,
84            duration_seconds=duration
85        )
86 
87    def _estimate_cost(self, tokens: int, model: str) -> float:
88        """Estima el coste en USD basado en tokens y modelo."""
89        # Precios aproximados por 1M tokens (input+output promediado)
90        prices = {
91            "gpt-4o": 7.50,
92            "gpt-4o-mini": 0.30,
93            "gpt-4-turbo": 20.00,
94            "claude-3-opus": 30.00,
95            "claude-3-sonnet": 6.00,
96            "claude-3-haiku": 0.50,
97            "gemini-1.5-pro": 3.50,
98        }
99        price_per_m = prices.get(model, 5.0)
100        return (tokens / 1_000_000) * price_per_m
101 
102    def save_logs(self, filepath: str):
103        """Guarda logs en JSON para análisis."""
104        with open(filepath, 'w') as f:
105            json.dump(self.logs, f, indent=2, ensure_ascii=False)

1def run_benchmark(world_config: Dict,
2                  models: List[Dict],
3                  num_episodes: int = 5) -> List[EvaluationMetrics]:
4    """
5    Ejecuta benchmark comparativo entre modelos.
6 
7    Args:
8        world_config: Configuración del juego
9        models: Lista de {"provider": str, "model": str}
10        num_episodes: Episodios por modelo
11 
12    Returns:
13        Lista de métricas por modelo y episodio
14    """
15    results = []
16 
17    for model_config in models:
18        print(f"\n📊 Evaluando {model_config['model']}...")
19 
20        model_results = []
21        for episode in range(num_episodes):
22            # Crear instancias frescas
23            game = TextAdventureGame(world_config.copy())
24            agent = LLMAgent(**model_config)
25            evaluator = AgentEvaluator(game, agent)
26 
27            # Ejecutar episodio
28            metrics = evaluator.run_episode()
29            model_results.append(metrics)
30 
31            status = "✅" if metrics.success else "❌"
32            print(f"  Episodio {episode + 1}: {status} "
33                  f"({metrics.steps_taken} pasos, ${metrics.cost_usd:.4f})")
34 
35        results.extend(model_results)
36 
37    return results
38 
39# Ejemplo de uso
40if __name__ == "__main__":
41    # Configuración del mundo
42    world = {
43        "rooms": {
44            "start": {
45                "description": "Estás en una habitación oscura.",
46                "exits": {"norte": "pasillo"},
47                "items": ["antorcha"]
48            },
49            "pasillo": {
50                "description": "Un largo pasillo con puertas.",
51                "exits": {"sur": "start", "este": "sala_tesoro"},
52                "items": ["llave"]
53            },
54            "sala_tesoro": {
55                "description": "Una sala con un cofre cerrado.",
56                "exits": {"oeste": "pasillo"},
57                "items": ["cofre"]
58            }
59        },
60        "items": ["antorcha", "llave", "cofre"],
61        "goal": {
62            "type": "use_item_in_room",
63            "item": "llave",
64            "room": "sala_tesoro"
65        },
66        "max_steps": 20
67    }
68 
69    # Modelos a comparar
70    models = [
71        {"provider": "openai", "model": "gpt-4o"},
72        {"provider": "openai", "model": "gpt-4o-mini"},
73        {"provider": "anthropic", "model": "claude-3-sonnet-20240229"},
74    ]
75 
76    # Ejecutar benchmark
77    results = run_benchmark(world, models, num_episodes=3)
78 
79    # Mostrar resumen
80    print("\n" + "="*60)
81    print("RESUMEN DE RESULTADOS")
82    print("="*60)
83 
84    for model in set(r.model for r in results):
85        model_results = [r for r in results if r.model == model]
86        success_rate = sum(1 for r in model_results if r.success) / len(model_results)
87        avg_steps = sum(r.steps_taken for r in model_results) / len(model_results)
88        avg_cost = sum(r.cost_usd for r in model_results) / len(model_results)
89 
90        print(f"\n{model}:")
91        print(f"  Tasa de éxito: {success_rate:.1%}")
92        print(f"  Pasos promedio: {avg_steps:.1f}")
93        print(f"  Coste promedio: ${avg_cost:.4f}")