C++ Boost 搜索引擎：正倒排索引核心实现与详解

3. 索引构建流程

3.1 构建正排索引

通过工具库的分词函数处理原始文本，提取标题、内容和 URL，并按顺序存入向量中。文档 ID 直接由当前向量大小决定。

DocInfo* BuildForwardIndex(const std::string& line) {
    std::vector<std::string> results;
    // 按分隔符切割字符串，获取 title, content, url
    ns_util::StringUtil::Split(line, &results, "\3");
    
    if (results.size() != 3) return nullptr;

    DocInfo doc;
    doc.title = results[0];
    doc.content = results[1];
    doc.url = results[2];
    doc.doc_id = forward_index.size(); // ID 即为当前下标

    forward_index.push_back(doc);
    return &forward_index.back();
}

3.2 构建倒排索引

流程分为分词、词频统计、权重计算和填充索引四个步骤。标题中的词权重要高于正文，因此引入加权系数。

bool BuildInvertedIndex(const DocInfo& doc) {
    // 用于统计词频的结构体
    struct WordCnt {
        int title_cnt = 0;
        int content_cnt = 0;
    };

    std::unordered_map<std::string, WordCnt> word_map;
    std::vector<std::string> title_words, content_words;

    // 分词并统计标题词频
    ns_util::JiebaUtil::CutString(doc.title, &title_words);
    for (auto& tw : title_words) {
        boost::to_lower(tw);
        word_map[tw].title_cnt++;
    }

    // 分词并统计正文词频
    ns_util::JiebaUtil::CutString(doc.content, &content_words);
    for (auto& cw : content_words) {
        boost::to_lower(tw);
        word_map[cw].content_cnt++;
    }

    // 计算权重并填充倒排索引
    #define X 10 // 标题权重系数
    #define Y 1  // 正文权重系数
    for (auto& pair : word_map) {
        InvertedElem item;
        item.doc_id = doc.doc_id;
        item.word = pair.first;
        // 模拟权重计算：标题出现次数 * 10 + 正文出现次数 * 1
        item.weight = X * pair.second.title_cnt + Y * pair.second.content_cnt;
        
        inverted_index[pair.first].push_back(item);
    }
    return true;
}

3.3 批量构建入口

从文件读取去标签后的文档流，逐行调用上述两个接口完成索引构建。过程中包含简单的日志输出以监控进度。

bool BuildIndex(const std::string& input) {
    std::ifstream in(input, std::ios::in | std::ios::binary);
    if (!in.is_open()) {
        std::cout << input << "open error" << std::endl;
        return false;
    }

    int count = 0;
    std::string line;
    while (std::getline(in, line)) {
        DocInfo* doc = BuildForwardIndex(line);
        if (doc == nullptr) {
            std::cout << "BuildIndex error" << std::endl;
            continue;
        }
        BuildInvertedIndex(*doc);
        count++;
        if (count % 50 == 0) LOG1(NORMAL, "索引建立到：" + std::to_string(count));
    }
    return true;
}

4. 索引查询接口

4.1 获取正排索引

通过文档 ID 直接访问向量，需进行边界检查以防越界。

DocInfo* GetForwardIndex(uint64_t doc_id) {
    if (doc_id >= forward_index.size()) {
        std::cout << "doc_id out range, error!" << std::endl;
        return nullptr;
    }
    return &forward_index[doc_id];
}

4.2 获取倒排索引

查找关键词对应的倒排链表地址，若不存在则返回空指针。

InvertedList* GetInvertedList(const std::string& word) {
    auto iter = inverted_index.find(word);
    if (iter == inverted_index.end()) {
        std::cout << word << "get error" << std::endl;
        return nullptr;
    }
    return &(iter->second);
}

5. 完整代码实现

以下是整合后的完整头文件代码，包含了所有结构体定义、类声明及静态成员初始化。

#pragma once
#include <iostream>
#include <string>
#include <vector>
#include <unordered_map>
#include <fstream>
#include <mutex>
#include "usuallytool.hpp"
#include <boost/algorithm/string.hpp>
#include "log.hpp"

namespace ns_index {

    typedef struct DocInfo {
        std::string title;
        std::string content;
        std::string url;
        int doc_id;
    } DocInfo;

    struct InvertedElem {
        int doc_id;
        std::string word;
        int weight;
    };

    typedef std::vector<InvertedElem> InvertedList;

    class Index {
    private:
        std::vector<DocInfo> forward_index;
        std::unordered_map<std::string, InvertedList> inverted_index;

        Index() {};
        Index(const Index&) = delete;
        Index& operator=(const Index&) = delete;

        static Index* instance;
        static std::mutex log;

    public:
        ~Index();

        static Index* GetInstance() {
            if (instance == nullptr) {
                log.lock();
                if (instance == nullptr) {
                    instance = new Index();
                }
                log.unlock();
            }
            return instance;
        }

        DocInfo* GetForwardIndex(uint64_t doc_id) {
            if (doc_id >= forward_index.size()) {
                std::cout << "doc_id out range, error!" << std::endl;
                return nullptr;
            }
            return &forward_index[doc_id];
        }

        InvertedList* GetInvertedList(const std::string& word) {
            auto iter = inverted_index.find(word);
            if (iter == inverted_index.end()) {
                std::cout << word << "get error" << std::endl;
                return nullptr;
            }
            return &(iter->second);
        }

        bool BuildIndex(const std::string& input) {
            std::ifstream in(input, std::ios::in | std::ios::binary);
            if (!in.is_open()) {
                std::cout << input << "open error" << std::endl;
                return false;
            }
            int count = 0;
            std::string line;
            while (std::getline(in, line)) {
                DocInfo* doc = BuildForwardIndex(line);
                if (doc == nullptr) {
                    std::cout << "BuildIndex error" << std::endl;
                    continue;
                }
                BuildInvertedIndex(*doc);
                count++;
                if (count % 50 == 0) LOG1(NORMAL, "索引建立到：" + std::to_string(count));
            }
            return true;
        }

    private:
        DocInfo* BuildForwardIndex(const std::string& line) {
            std::vector<std::string> results;
            ns_util::StringUtil::Split(line, &results, "\3");
            if (results.size() != 3) return nullptr;
            DocInfo doc;
            doc.title = results[0];
            doc.content = results[1];
            doc.url = results[2];
            doc.doc_id = forward_index.size();
            forward_index.push_back(doc);
            return &forward_index.back();
        }

        bool BuildInvertedIndex(const DocInfo& doc) {
            struct WordCnt {
                int title_cnt = 0;
                int content_cnt = 0;
            };
            std::unordered_map<std::string, WordCnt> word_map;
            std::vector<std::string> title_words, content_words;

            ns_util::JiebaUtil::CutString(doc.title, &title_words);
            for (auto& tw : title_words) {
                boost::to_lower(tw);
                word_map[tw].title_cnt++;
            }

            ns_util::JiebaUtil::CutString(doc.content, &content_words);
            for (auto& cw : content_words) {
                boost::to_lower(cw);
                word_map[cw].content_cnt++;
            }

            #define X 10
            #define Y 1
            for (auto& pair : word_map) {
                InvertedElem item;
                item.doc_id = doc.doc_id;
                item.word = pair.first;
                item.weight = X * pair.second.title_cnt + Y * pair.second.content_cnt;
                inverted_index[pair.first].push_back(item);
            }
            return true;
        }
    };

    Index* Index::instance = nullptr;
    std::mutex Index::log;
}