C++ 搜索引擎 Searcher 模块：正倒排索引与查询处理详解

2.4 构建 JSON

这一步是序列化，即将内容转变为标准的、线性的、可存储/可传输的格式。代码通过 JsonCpp 非标准库实现序列化，将单个序列化的结果交给总的 root。

Json::StyledWriter writer;
*json_string = writer.write(root);

注：inverted_list_all 的本质是对多个倒排拉链进行'去重、合并、排序'后的候选文档集合，是连接'索引查询'和'结果返回'的中间数据结构。

完整 Search 函数逻辑如下：

void Search(const std::string& query, std::string* json_string) {
    // 1. 分词
    std::vector<std::string> words;
    ns_util::JiebaUtil::CutString(query, &words);
    
    // 2. 触发
    std::vector<InvertedElemPrint> inverted_list_all;
    std::unordered_map<uint64_t, InvertedElemPrint> tokens_map;
    
    for(std::string w : words) {
        boost::to_lower(w);
        ns_index::InvertedList* inverted_list = index->GetInvertedList(w);
        if(inverted_list == nullptr) continue;
        
        for(const auto &elem : *inverted_list){
            auto &item = tokens_map[elem.doc_id];
            item.doc_id = elem.doc_id;
            item.weight += elem.weight;
            item.words.push_back(elem.word);
        }
    }
    
    // 3. 合并
    for(const auto &item : tokens_map)
        inverted_list_all.push_back(std::move(item.second));
    
    std::sort(inverted_list_all.begin(), inverted_list_all.end(), [](const InvertedElemPrint &e1, const InvertedElemPrint &e2){
        return e1.weight > e1.weight;
    });
    
    // 4. 构建
    Json::Value root;
    for(auto& item : inverted_list_all) {
        ns_index::DocInfo* doc = index->GetForwardIndex(item.doc_id);
        if(doc == nullptr) continue;
        Json::Value elem;
        elem["title"] = doc->title;
        elem["desc"] = GetDesc(doc->content, item.words[0]);
        elem["url"] = doc->url;
        root.append(elem);
    }
    
    Json::StyledWriter writer;
    *json_string = writer.write(root);
}

3. GetDesc

该函数用于寻找摘要，即在 html_content 里面查找是否出现了关键字 word。条件判断确保网页信息长度足够，防止越界访问。

std::string GetDesc(const std::string& html_content, const std::string& word) {
    int prev_step = 50;
    int next_step = 100;
    auto iter = std::search(html_content.begin(), html_content.end(), word.begin(), word.end(), [](int x, int y){
        return (std::tolower(x) == std::tolower(y));
    });
    
    if(iter == html_content.end()) return "None1";
    int pos = std::distance(html_content.begin(), iter);
    if(pos == std::string::npos) return "None1";
    
    int start = 0;
    int end = html_content.size() - 1;
    if(pos - prev_step > start) start = pos - prev_step;
    if(pos + next_step < end) end = pos + next_step;
    if(start >= end) return "None2";
    
    std::string desc = html_content.substr(start, end - start);
    desc += "...";
    return desc;
}

4. InvertedElemPrint

该结构体用于解决重复文档的问题。作为'去重与信息聚合'的载体，当同一文档匹配多个关键词时，借助哈希表的键唯一性，避免文档在结果中重复出现。

struct InvertedElemPrint{
    uint64_t doc_id;
    int weight;
    std::vector<std::string> words;
    InvertedElemPrint(): doc_id(0), weight(0) {}
};

5. 完整代码示例

通过'索引构建'提前做好数据准备，通过'查询处理'高效匹配并排序文档，通过'结果格式化'衔接前端展示。

#pragma once
#include "index.hpp"
#include "usuallytool.hpp"
#include <algorithm>
#include <jsoncpp/json/json.h>
#include "log.hpp"

namespace ns_searcher{
    struct InvertedElemPrint{
        uint64_t doc_id;
        int weight;
        std::vector<std::string> words;
        InvertedElemPrint(): doc_id(0), weight(0) {}
    };
    
    class Searcher{
    private:
        ns_index::Index* index;
    public:
        Searcher(){}; ~Searcher(){};
        
        public: void InitSearcher(const std::string& input) {
            index = ns_index::Index::Getinstance();
            index->BuildIndex(input);
            LOG1(NORMAL, "建立索引成功...");
        }
        
        void Search(const std::string& query, std::string* json_string) {
            std::vector<std::string> words;
            ns_util::JiebaUtil::CutString(query, &words);
            std::vector<InvertedElemPrint> inverted_list_all;
            std::unordered_map<uint64_t, InvertedElemPrint> tokens_map;
            
            for(std::string w : words) {
                boost::to_lower(w);
                ns_index::InvertedList* inverted_list = index->GetInvertedList(w);
                if(inverted_list == nullptr) continue;
                
                for(const auto &elem : *inverted_list){
                    auto &item = tokens_map[elem.doc_id];
                    item.doc_id = elem.doc_id;
                    item.weight += elem.weight;
                    item.words.push_back(elem.word);
                }
            }
            
            for(const auto &item : tokens_map)
                inverted_list_all.push_back(std::move(item.second));
            
            std::sort(inverted_list_all.begin(), inverted_list_all.end(), [](const InvertedElemPrint &e1, const InvertedElemPrint &e2){
                return e1.weight > e1.weight;
            });
            
            Json::Value root;
            for(auto& item : inverted_list_all) {
                ns_index::DocInfo* doc = index->GetForwardIndex(item.doc_id);
                if(doc == nullptr) continue;
                Json::Value elem;
                elem["title"] = doc->title;
                elem["desc"] = GetDesc(doc->content, item.words[0]);
                elem["url"] = doc->url;
                root.append(elem);
            }
            
            Json::StyledWriter writer;
            *json_string = writer.write(root);
        }
        
        std::string GetDesc(const std::string& html_content, const std::string& word) {
            int prev_step = 50;
            int next_step = 100;
            auto iter = std::search(html_content.begin(), html_content.end(), word.begin(), word.end(), [](int x, int y){
                return (std::tolower(x) == std::tolower(y));
            });
            
            if(iter == html_content.end()) return "None1";
            int pos = std::distance(html_content.begin(), iter);
            if(pos == std::string::npos) return "None1";
            
            int start = 0;
            int end = html_content.size() - 1;
            if(pos - prev_step > start) start = pos - prev_step;
            if(pos + next_step < end) end = pos + next_step;
            if(start >= end) return "None2";
            
            std::string desc = html_content.substr(start, end - start);
            desc += "...";
            return desc;
        }
    };
}