5. Vector Embeddings y Búsqueda Semántica con .NET

Introducción

Los embeddings vectoriales transforman texto en representaciones numéricas que capturan significado semántico. Esto permite buscar contenido por similitud de significado, no solo por palabras clave. En este tutorial aprenderás a implementar búsqueda semántica con Azure OpenAI y Semantic Kernel.

¿Qué son los Embeddings?

Un embedding es una representación vectorial (array de números) de texto. Textos con significados similares tienen vectores similares:

• “El gato está durmiendo” → [0.23, -0.45, 0.67, ...]
• “El felino descansa” → [0.25, -0.43, 0.69, ...] (similar)
• “Python es un lenguaje” → [0.89, 0.12, -0.34, ...] (diferente)

Configuración de Embeddings

Instalar Paquetes

dotnet add package Microsoft.SemanticKernel
dotnet add package System.Numerics.Tensors

Configurar Servicio de Embeddings

using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.Embeddings;

var builder = Kernel.CreateBuilder();

// Configurar servicio de text embeddings
builder.AddAzureOpenAITextEmbeddingGeneration(
    deploymentName: "text-embedding-ada-002",
    endpoint: "https://tu-recurso.openai.azure.com/",
    apiKey: "tu-api-key");

var kernel = builder.Build();

// Obtener servicio de embeddings
var embeddingService = kernel.GetRequiredService<ITextEmbeddingGenerationService>();

Generar Embeddings

Embedding de un Texto

using Microsoft.SemanticKernel.Embeddings;

public class EmbeddingService
{
    private readonly ITextEmbeddingGenerationService _embeddingService;
    
    public EmbeddingService(ITextEmbeddingGenerationService embeddingService)
    {
        _embeddingService = embeddingService;
    }
    
    public async Task<ReadOnlyMemory<float>> GenerateEmbeddingAsync(
        string text,
        CancellationToken cancellationToken = default)
    {
        if (string.IsNullOrWhiteSpace(text))
        {
            throw new ArgumentException("El texto no puede estar vacío");
        }
        
        var embeddings = await _embeddingService.GenerateEmbeddingsAsync(
            new[] { text },
            kernel: null,
            cancellationToken);
        
        return embeddings.First();
    }
}

Embeddings en Batch

public async Task<IList<ReadOnlyMemory<float>>> GenerateBatchEmbeddingsAsync(
    IEnumerable<string> texts,
    CancellationToken cancellationToken = default)
{
    var textList = texts.ToList();
    
    if (textList.Count == 0)
    {
        return new List<ReadOnlyMemory<float>>();
    }
    
    return await _embeddingService.GenerateEmbeddingsAsync(
        textList,
        kernel: null,
        cancellationToken);
}

Cálculo de Similitud

Similitud Coseno

using System.Numerics.Tensors;

public class SimilarityCalculator
{
    public static double CalculateCosineSimilarity(
        ReadOnlyMemory<float> vector1,
        ReadOnlyMemory<float> vector2)
    {
        var span1 = vector1.Span;
        var span2 = vector2.Span;
        
        if (span1.Length != span2.Length)
        {
            throw new ArgumentException("Los vectores deben tener la misma dimensión");
        }
        
        // Producto punto
        float dotProduct = TensorPrimitives.Dot(span1, span2);
        
        // Magnitudes
        float magnitude1 = TensorPrimitives.Norm(span1);
        float magnitude2 = TensorPrimitives.Norm(span2);
        
        if (magnitude1 == 0 || magnitude2 == 0)
        {
            return 0;
        }
        
        return dotProduct / (magnitude1 * magnitude2);
    }
    
    public static double CalculateEuclideanDistance(
        ReadOnlyMemory<float> vector1,
        ReadOnlyMemory<float> vector2)
    {
        var span1 = vector1.Span;
        var span2 = vector2.Span;
        
        if (span1.Length != span2.Length)
        {
            throw new ArgumentException("Los vectores deben tener la misma dimensión");
        }
        
        float sumSquaredDiff = 0;
        for (int i = 0; i < span1.Length; i++)
        {
            float diff = span1[i] - span2[i];
            sumSquaredDiff += diff * diff;
        }
        
        return Math.Sqrt(sumSquaredDiff);
    }
}

Sistema de Búsqueda Semántica en Memoria

using System.Collections.Concurrent;

public class Document
{
    public required string Id { get; init; }
    public required string Content { get; init; }
    public ReadOnlyMemory<float> Embedding { get; set; }
    public Dictionary<string, string>? Metadata { get; init; }
}

public class SearchResult
{
    public required Document Document { get; init; }
    public double Score { get; init; }
}

public class InMemorySemanticSearch
{
    private readonly ITextEmbeddingGenerationService _embeddingService;
    private readonly ConcurrentDictionary<string, Document> _documents;
    
    public InMemorySemanticSearch(ITextEmbeddingGenerationService embeddingService)
    {
        _embeddingService = embeddingService;
        _documents = new ConcurrentDictionary<string, Document>();
    }
    
    public async Task IndexDocumentAsync(
        Document document,
        CancellationToken cancellationToken = default)
    {
        // Generar embedding para el documento
        var embeddings = await _embeddingService.GenerateEmbeddingsAsync(
            new[] { document.Content },
            kernel: null,
            cancellationToken);
        
        document.Embedding = embeddings.First();
        
        _documents[document.Id] = document;
    }
    
    public async Task IndexDocumentsAsync(
        IEnumerable<Document> documents,
        CancellationToken cancellationToken = default)
    {
        var docList = documents.ToList();
        var texts = docList.Select(d => d.Content).ToList();
        
        // Generar embeddings en batch
        var embeddings = await _embeddingService.GenerateEmbeddingsAsync(
            texts,
            kernel: null,
            cancellationToken);
        
        for (int i = 0; i < docList.Count; i++)
        {
            docList[i].Embedding = embeddings[i];
            _documents[docList[i].Id] = docList[i];
        }
    }
    
    public async Task<List<SearchResult>> SearchAsync(
        string query,
        int topK = 5,
        CancellationToken cancellationToken = default)
    {
        // Generar embedding de la consulta
        var queryEmbeddings = await _embeddingService.GenerateEmbeddingsAsync(
            new[] { query },
            kernel: null,
            cancellationToken);
        
        var queryEmbedding = queryEmbeddings.First();
        
        // Calcular similitud con todos los documentos
        var results = new List<SearchResult>();
        
        foreach (var doc in _documents.Values)
        {
            var similarity = SimilarityCalculator.CalculateCosineSimilarity(
                queryEmbedding,
                doc.Embedding);
            
            results.Add(new SearchResult
            {
                Document = doc,
                Score = similarity
            });
        }
        
        // Ordenar por score descendente y tomar top K
        return results
            .OrderByDescending(r => r.Score)
            .Take(topK)
            .ToList();
    }
    
    public bool RemoveDocument(string documentId)
    {
        return _documents.TryRemove(documentId, out _);
    }
    
    public void Clear()
    {
        _documents.Clear();
    }
    
    public int Count => _documents.Count;
}

Búsqueda con Filtros

public class FilteredSemanticSearch : InMemorySemanticSearch
{
    public FilteredSemanticSearch(ITextEmbeddingGenerationService embeddingService) 
        : base(embeddingService)
    {
    }
    
    public async Task<List<SearchResult>> SearchWithFiltersAsync(
        string query,
        Dictionary<string, string>? filters = null,
        int topK = 5,
        double minScore = 0.0,
        CancellationToken cancellationToken = default)
    {
        var results = await SearchAsync(query, topK * 2, cancellationToken);
        
        // Aplicar filtros de metadata
        if (filters != null && filters.Count > 0)
        {
            results = results
                .Where(r => MatchesFilters(r.Document, filters))
                .ToList();
        }
        
        // Aplicar score mínimo
        results = results
            .Where(r => r.Score >= minScore)
            .Take(topK)
            .ToList();
        
        return results;
    }
    
    private bool MatchesFilters(Document document, Dictionary<string, string> filters)
    {
        if (document.Metadata == null)
            return false;
        
        foreach (var filter in filters)
        {
            if (!document.Metadata.TryGetValue(filter.Key, out var value) ||
                value != filter.Value)
            {
                return false;
            }
        }
        
        return true;
    }
}

Integración con Base de Datos Vectorial

Modelo para Persistencia

public class VectorDocument
{
    public string Id { get; set; } = Guid.NewGuid().ToString();
    public required string Content { get; set; }
    public required float[] Embedding { get; set; }
    public Dictionary<string, string>? Metadata { get; set; }
    public DateTime CreatedAt { get; set; } = DateTime.UtcNow;
}

Servicio con Persistencia

using System.Text.Json;

public class PersistentSemanticSearch
{
    private readonly ITextEmbeddingGenerationService _embeddingService;
    private readonly string _storageDirectory;
    
    public PersistentSemanticSearch(
        ITextEmbeddingGenerationService embeddingService,
        string storageDirectory)
    {
        _embeddingService = embeddingService;
        _storageDirectory = storageDirectory;
        
        Directory.CreateDirectory(storageDirectory);
    }
    
    public async Task IndexAndSaveAsync(
        Document document,
        CancellationToken cancellationToken = default)
    {
        // Generar embedding
        var embeddings = await _embeddingService.GenerateEmbeddingsAsync(
            new[] { document.Content },
            kernel: null,
            cancellationToken);
        
        document.Embedding = embeddings.First();
        
        // Guardar en disco
        var vectorDoc = new VectorDocument
        {
            Id = document.Id,
            Content = document.Content,
            Embedding = document.Embedding.ToArray(),
            Metadata = document.Metadata
        };
        
        var json = JsonSerializer.Serialize(vectorDoc);
        var filePath = Path.Combine(_storageDirectory, $"{document.Id}.json");
        await File.WriteAllTextAsync(filePath, json, cancellationToken);
    }
    
    public async Task<List<SearchResult>> SearchAsync(
        string query,
        int topK = 5,
        CancellationToken cancellationToken = default)
    {
        // Generar embedding de la consulta
        var queryEmbeddings = await _embeddingService.GenerateEmbeddingsAsync(
            new[] { query },
            kernel: null,
            cancellationToken);
        
        var queryEmbedding = queryEmbeddings.First();
        
        // Cargar y buscar en todos los documentos
        var files = Directory.GetFiles(_storageDirectory, "*.json");
        var results = new List<SearchResult>();
        
        foreach (var file in files)
        {
            var json = await File.ReadAllTextAsync(file, cancellationToken);
            var vectorDoc = JsonSerializer.Deserialize<VectorDocument>(json);
            
            if (vectorDoc == null) continue;
            
            var docEmbedding = new ReadOnlyMemory<float>(vectorDoc.Embedding);
            var similarity = SimilarityCalculator.CalculateCosineSimilarity(
                queryEmbedding,
                docEmbedding);
            
            results.Add(new SearchResult
            {
                Document = new Document
                {
                    Id = vectorDoc.Id,
                    Content = vectorDoc.Content,
                    Embedding = docEmbedding,
                    Metadata = vectorDoc.Metadata
                },
                Score = similarity
            });
        }
        
        return results
            .OrderByDescending(r => r.Score)
            .Take(topK)
            .ToList();
    }
}

RAG (Retrieval Augmented Generation)

Combinar búsqueda semántica con generación:

using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.ChatCompletion;

public class RAGService
{
    private readonly InMemorySemanticSearch _search;
    private readonly Kernel _kernel;
    
    public RAGService(
        InMemorySemanticSearch search,
        Kernel kernel)
    {
        _search = search;
        _kernel = kernel;
    }
    
    public async Task<string> AskAsync(
        string question,
        CancellationToken cancellationToken = default)
    {
        // 1. Buscar documentos relevantes
        var searchResults = await _search.SearchAsync(
            question,
            topK: 3,
            cancellationToken);
        
        // 2. Construir contexto con documentos encontrados
        var context = string.Join("\n\n", 
            searchResults.Select(r => $"- {r.Document.Content}"));
        
        // 3. Generar respuesta basada en el contexto
        var chatService = _kernel.GetRequiredService<IChatCompletionService>();
        
        var chatHistory = new ChatHistory();
        chatHistory.AddSystemMessage("""
            Responde la pregunta del usuario basándote ÚNICAMENTE en el contexto proporcionado.
            Si la información no está en el contexto, di que no tienes suficiente información.
            """);
        
        chatHistory.AddUserMessage($"""
            Contexto:
            {context}
            
            Pregunta: {question}
            """);
        
        var response = await chatService.GetChatMessageContentAsync(
            chatHistory,
            kernel: _kernel,
            cancellationToken: cancellationToken);
        
        return response.Content ?? "No pude generar una respuesta";
    }
}

Ejemplo Completo: Sistema de Preguntas y Respuestas

using Microsoft.SemanticKernel;
using Microsoft.Extensions.Logging;

class Program
{
    static async Task Main(string[] args)
    {
        // Configurar kernel
        var builder = Kernel.CreateBuilder();
        
        builder.AddAzureOpenAITextEmbeddingGeneration(
            deploymentName: "text-embedding-ada-002",
            endpoint: Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT")!,
            apiKey: Environment.GetEnvironmentVariable("AZURE_OPENAI_KEY")!);
        
        builder.AddAzureOpenAIChatCompletion(
            deploymentName: "gpt-4",
            endpoint: Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT")!,
            apiKey: Environment.GetEnvironmentVariable("AZURE_OPENAI_KEY")!);
        
        var kernel = builder.Build();
        
        // Configurar búsqueda semántica
        var embeddingService = kernel.GetRequiredService<ITextEmbeddingGenerationService>();
        var search = new InMemorySemanticSearch(embeddingService);
        
        // Indexar documentos de conocimiento
        var documents = new[]
        {
            new Document
            {
                Id = "1",
                Content = "C# es un lenguaje de programación orientado a objetos desarrollado por Microsoft.",
                Metadata = new Dictionary<string, string> { ["category"] = "programming" }
            },
            new Document
            {
                Id = "2",
                Content = "Async/await en C# permite escribir código asíncrono de manera más legible.",
                Metadata = new Dictionary<string, string> { ["category"] = "programming" }
            },
            new Document
            {
                Id = "3",
                Content = "Semantic Kernel es un SDK para integrar LLMs en aplicaciones .NET.",
                Metadata = new Dictionary<string, string> { ["category"] = "ai" }
            }
        };
        
        await search.IndexDocumentsAsync(documents);
        
        // Crear servicio RAG
        var ragService = new RAGService(search, kernel);
        
        // Hacer preguntas
        Console.WriteLine("Sistema de preguntas y respuestas listo.");
        Console.WriteLine("Escribe 'salir' para terminar.\n");
        
        while (true)
        {
            Console.Write("Pregunta: ");
            var question = Console.ReadLine();
            
            if (string.IsNullOrWhiteSpace(question) || question.ToLower() == "salir")
                break;
            
            var answer = await ragService.AskAsync(question);
            Console.WriteLine($"\nRespuesta: {answer}\n");
        }
    }
}

Mejores Prácticas

1. Normalización de Texto

private string NormalizeText(string text)
{
    return text
        .ToLowerInvariant()
        .Trim()
        .Replace("\n", " ")
        .Replace("\r", " ");
}

2. Chunking de Documentos Largos

public List<string> ChunkText(string text, int chunkSize = 500, int overlap = 50)
{
    var words = text.Split(' ', StringSplitOptions.RemoveEmptyEntries);
    var chunks = new List<string>();
    
    for (int i = 0; i < words.Length; i += chunkSize - overlap)
    {
        var chunk = string.Join(" ", words.Skip(i).Take(chunkSize));
        chunks.Add(chunk);
    }
    
    return chunks;
}

3. Caché de Embeddings

using Microsoft.Extensions.Caching.Memory;

public class CachedEmbeddingService
{
    private readonly ITextEmbeddingGenerationService _embeddingService;
    private readonly IMemoryCache _cache;
    
    public async Task<ReadOnlyMemory<float>> GetEmbeddingAsync(string text)
    {
        var cacheKey = $"emb_{text.GetHashCode()}";
        
        if (_cache.TryGetValue<ReadOnlyMemory<float>>(cacheKey, out var cached))
        {
            return cached;
        }
        
        var embeddings = await _embeddingService.GenerateEmbeddingsAsync(
            new[] { text },
            kernel: null);
        
        var embedding = embeddings.First();
        
        _cache.Set(cacheKey, embedding, TimeSpan.FromHours(24));
        
        return embedding;
    }
}

4. Manejo de Errores

public async Task<ReadOnlyMemory<float>> GenerateEmbeddingWithRetryAsync(
    string text,
    int maxRetries = 3)
{
    for (int attempt = 0; attempt < maxRetries; attempt++)
    {
        try
        {
            return await GenerateEmbeddingAsync(text);
        }
        catch (HttpRequestException ex) when (attempt < maxRetries - 1)
        {
            await Task.Delay(TimeSpan.FromSeconds(Math.Pow(2, attempt)));
        }
    }
    
    throw new Exception("No se pudo generar embedding después de varios intentos");
}

Conclusión

Los embeddings vectoriales y la búsqueda semántica son fundamentales para aplicaciones de IA modernas. Permiten buscar por significado, no solo por palabras clave, y son la base de sistemas RAG que combinan recuperación de información con generación de texto. Con Semantic Kernel y Azure OpenAI puedes implementar búsqueda semántica robusta en tus aplicaciones .NET.

Palabras clave: vector embeddings, semantic search, RAG, Azure OpenAI, Semantic Kernel, similarity search, C#