NetCore + 开源大模型，文章标题搜索的全新突破

• 机械图纸标题搜索的挑战：
  • 机械图纸标题通常包含专业术语、缩写和特定格式（如“土豆分拣机 DWG-001 不锈钢”）。
  • 用户查询可能模糊或表述不同（如“马铃薯筛选机”），传统搜索难以匹配语义相似的标题。
  • 标题信息有限，传统搜索容易遗漏相关图纸或返回无关结果。
• 搜索增强的优势：
  • 语义理解：通过大模型生成语义嵌入向量，理解标题和查询的深层含义，支持模糊匹配和语义相关性排序。
    • 示例：用户搜索“土豆分拣机”，传统搜索只能匹配标题中包含“土豆分拣机”的图纸；增强搜索可匹配语义相似的标题，如“马铃薯筛选机”，因为 AI 模型能够理解“土豆”和“马铃薯”是同义词，“分拣”和“筛选”是近义词。
  • 多维度匹配：结合标题中的专业术语和元数据（如材料、尺寸），提升搜索的准确性和全面性。
  • 高效索引：使用向量存储（如 Redis）支持快速的相似度搜索，满足实时性需求。
  • 用户体验提升：返回更相关、更精准的图纸标题结果，减少用户反复调整查询的成本。

from FlagEmbedding import FlagModel 
import pandas as pd 
import numpy as np
from datasets import Dataset
from scipy.spatial import distance 
import datetime
import configparser
import pymysql  
 
model = None 

def getModel():
    global model
    if model is None:
        model = FlagModel("./model",
                    query_instruction_for_retrieval="Represent this sentence for searching relevant passages:",
                    use_fp16=True) 
    return model
#获取向量
def getFlagEmbedding(title):
    global model
    model = getModel()
    embedding = model.encode(title)
    return embedding
  

 /// <summary>
 /// 获取向量
 /// </summary>
 /// <param name="keyword"></param>
 /// <returns></returns>
 public async Task<double[]> GetFlagEmbedding(string keyword)
 {
     var vector = new double[] { };
     vector = null;
     try
     {
         var req = new
         {
             action = "getFlagEmbedding",
             keyword
         };
         var content = new StringContent(
                        JsonSerializer.Serialize(req)
                        , Encoding.UTF8, "application/json");
         var response = await _client.PostAsync(ConfigHelp.FlagSerachUrl, content);
         if (response.IsSuccessStatusCode)
         {
             var result = await response.Content.ReadAsStringAsync();
             var data = JsonSerializer.Deserialize<GetFlagEmbeddingRoot>(result);
             if (data.op)
             {
                 vector = data.msg.Split(',').Select(double.Parse).ToArray();
             }
         }
     }
     catch (Exception ex)
     {
         LogUtils.Error("GetFlagEmbedding ", ex);
     }

     return vector;
 }

   public class RedisVectorHelp
   {
       private readonly IDatabase _db;
       private string _freefix;
       private string _indexName;
       private SearchCommands ft;
       public RedisVectorHelp(string freefix,string redisConnectionString,int dbNum=0)
       {
           var redis = ConnectionMultiplexer.Connect(redisConnectionString);
           _db = redis.GetDatabase(dbNum);
           _freefix = freefix;
           _indexName = _freefix + "_index";
           ft = new SearchCommands(_db, null);
       } 
       /// <summary>
       /// 创建索引
       /// </summary>
       public void CreateFt()
       {
           var list = ft._List();
           var indexList = list.Select(result => result.ToString()).ToArray();
           //判断是否存在索引
           if (indexList.Contains(_indexName))
           {
               Console.WriteLine("Index already exists.");
               return;
           }
           ft.Create(_indexName,
 new FTCreateParams()
     .On(IndexDataType.HASH)
     .Prefix(_freefix + ":"),
 new Schema()
     .AddTextField("id")
     .AddVectorField("vector",
     VectorField.VectorAlgo.FLAT,
         new Dictionary<string, object>
         {
             ["TYPE"] = "FLOAT32",
             ["DIM"] = 1024,
             ["DISTANCE_METRIC"] = "COSINE"
         })
     );
       }
       /// <summary>
       /// 存储向量
       /// </summary>
       /// <param name="id"></param>
       /// <param name="vector"></param>
       public void StoreVectorData(string id, float[] vector)
       {
           // 构造键名
           var key = $"{_freefix}:{id}";
           VectorDom dom = new VectorDom
           {
               id = id,
               vector = vector
           };
           byte[] vectorBinary = vector.SelectMany(f => BitConverter.GetBytes(f)).ToArray();
           _db.HashSet(key, "id", dom.id);
           _db.HashSet(key, "vector", vectorBinary);
       }
       /// <summary>
       /// 向量搜索
       /// </summary>
       /// <param name="queryVector"></param>
       /// <param name="topK"></param>
       public List<string> SearchSimilarVectors(float[] queryVector, int topK = 50)
       {
           byte[] vectorQueryBinary = queryVector.SelectMany(f => BitConverter.GetBytes(f)).ToArray();
           //十六进制字符串
           //string vectorQueryBinaryStr = BitConverter.ToString(vectorQueryBinary).Replace("-", "");
           Query q = new Query($"*=>[KNN {topK} @vector $vec as score]");
           q.SortBy = "score";
           q.AddParam("vec", vectorQueryBinary);
           q.ReturnFields("id", "vector");
           q.Limit(0, topK);
           q.Dialect(2);
           var obj = ft.Search(_indexName, q);
           var docList = obj.Documents;
           var list = new List<string>(); 
           foreach (var doc in docList)
           {
               list.Add(doc["id"]);
           }
           return list;
       } 
   }
   public class VectorDom
   {
       public string id { get; set; }
       public float[] vector { get; set; }
   }

 var searchVector = await GetFlagEmbedding(keyword);
 if (searchVector != null)
 {
     var queryVector = Array.ConvertAll(searchVector, x => (float)x);
     var temp = bykcsjRVHelp.SearchSimilarVectors(queryVector, 30);
     foreach (var id in temp)
     {
         if (!ids.Contains(id))
         {
             ids.Add(id);
         }
     }
 }