C++ 基于正倒排索引的 Boost 搜索引擎实现

3. 去标签后的文档构建正倒排索引

3.1 创建正排索引

通过 Split 函数进行分词，将切分后的结果存入临时变量 doc，再推入 forward_index。

DocInfo1* BuildForwardIndex(const std::string& line) {
    std::vector<std::string> results;
    ns_util::StringUtil::Split(line, &results, "\3"); // 字符串切割，把 title, content, url 分开
    if (results.size() != 3) return nullptr;
    
    DocInfo1 doc;
    doc.title = results[0];
    doc.content = results[1];
    doc.url = results[2];
    doc.doc_id = forward_index.size(); // ID 为当前下标个数
    
    forward_index.push_back(doc);
    return &forward_index.back();
}

3.2 创建倒排索引

流程：分词 -> 词频统计 -> 权重计算 -> 填充倒排索引。

bool BuildInvertedIndex(const DocInfo1& doc) {
    struct word_cnt {
        int title_cnt;   // title 出现的次数
        int content_cnt; // content 出现的次数
        word_cnt() : title_cnt(0), content_cnt(0) {}
    };
    
    std::unordered_map<std::string, word_cnt> word_map;
    std::vector<std::string> title_words;
    ns_util::JiebaUtil::CutString(doc.title, &title_words);
    for (auto& tw : title_words) {
        boost::to_lower(tw);
        word_map[tw].title_cnt++;
    }
    
    std::vector<std::string> content_words;
    ns_util::JiebaUtil::CutString(doc.content, &content_words);
    for (auto& cw : content_words) {
        boost::to_lower(cw);
        word_map[cw].content_cnt++;
    }
    
    #define X 10
    #define Y 1
    for (auto& word_pair : word_map) {
        InvertedElem item;
        item.doc_id = doc.doc_id;
        item.word = word_pair.first;
        item.weight = X * word_pair.second.title_cnt + Y * word_pair.second.content_cnt;
        inverted_index[word_pair.first].push_back(item);
    }
    return true;
}

3.3 同时调用两个函数创建索引

以二进制方式读取原始文档，逐行处理并构建索引。

bool BuildIndex(const std::string& input) {
    std::ifstream in(input, std::ios::in | std::ios::binary);
    if (!in.is_open()) {
        std::cout << input << "open error" << std::endl;
        return false;
    }
    
    int count = 0;
    std::string line;
    while (std::getline(in, line)) {
        DocInfo1* doc = BuildForwardIndex(line);
        if (doc == nullptr) {
            std::cout << "BuildIndex error" << std::endl;
            continue;
        }
        BuildInvertedIndex(*doc);
        count++;
        if (count % 50 == 0) LOG1(NORMAL, "索引建立到：" + std::to_string(count));
    }
    return true;
}

4. 获取正倒排索引

4.1 获取正排索引

防御性编程，判断 doc_id 范围后返回对应文档内容。

DocInfo1* GetForwardIndex(uint64_t doc_id) {
    if (doc_id >= forward_index.size()) {
        std::cout << "doc_id out range, error!" << std::endl;
        return nullptr;
    }
    return &forward_index[doc_id];
}

4.2 获取倒排索引

查找关键字是否存在，存在则返回对应的倒排链表地址。

InvertedList* GetInvertedList(const std::string& word) {
    auto iter = inverted_index.find(word);
    if (iter == inverted_index.end()) {
        std::cout << word << "get error" << std::endl;
        return nullptr;
    }
    return &(iter->second);
}

5. 完整代码

#pragma once
#include <iostream>
#include <string>
#include <vector>
#include <unordered_map>
#include <fstream>
#include <mutex>
#include "usuallytool.hpp"
#include <boost/algorithm/string.hpp>
#include "log.hpp"

namespace ns_index {
    typedef struct DocInfo {
        std::string title;
        std::string content;
        std::string url;
        int doc_id;
    } DocInfo1;

    struct InvertedElem {
        int doc_id;
        std::string word;
        int weight;
    };
    typedef std::vector<InvertedElem> InvertedList;

    class Index {
    private:
        std::vector<DocInfo1> forward_index;
        std::unordered_map<std::string, InvertedList> inverted_index;

        Index() {};
        Index(const Index&) = delete;
        Index& operator=(const Index&) = delete;
        static Index* instance;
        static std::mutex log;

    public:
        ~Index();
        static Index* Getinstance() {
            if (instance == nullptr) {
                log.lock();
                if (instance == nullptr) {
                    instance = new Index();
                }
                log.unlock();
            }
            return instance;
        }

        DocInfo1* GetForwardIndex(uint64_t doc_id) {
            if (doc_id >= forward_index.size()) {
                std::cout << "doc_id out range, error!" << std::endl;
                return nullptr;
            }
            return &forward_index[doc_id];
        }

        InvertedList* GetInvertedList(const std::string& word) {
            auto iter = inverted_index.find(word);
            if (iter == inverted_index.end()) {
                std::cout << word << "get error" << std::endl;
                return nullptr;
            }
            return &(iter->second);
        }

        bool BuildIndex(const std::string& input) {
            std::ifstream in(input, std::ios::in | std::ios::binary);
            if (!in.is_open()) {
                std::cout << input << "open error" << std::endl;
                return false;
            }
            int count = 0;
            std::string line;
            while (std::getline(in, line)) {
                DocInfo1* doc = BuildForwardIndex(line);
                if (doc == nullptr) {
                    std::cout << "BuildIndex error" << std::endl;
                    continue;
                }
                BuildInvertedIndex(*doc);
                count++;
                if (count % 50 == 0) LOG1(NORMAL, "索引建立到：" + std::to_string(count));
            }
            return true;
        }

    private:
        DocInfo1* BuildForwardIndex(const std::string& line) {
            std::vector<std::string> results;
            ns_util::StringUtil::Split(line, &results, "\3");
            if (results.size() != 3) return nullptr;
            DocInfo1 doc;
            doc.title = results[0];
            doc.content = results[1];
            doc.url = results[2];
            doc.doc_id = forward_index.size();
            forward_index.push_back(doc);
            return &forward_index.back();
        }

        bool BuildInvertedIndex(const DocInfo1& doc) {
            struct word_cnt {
                int title_cnt;
                int content_cnt;
                word_cnt() : title_cnt(0), content_cnt(0) {}
            };
            std::unordered_map<std::string, word_cnt> word_map;
            std::vector<std::string> title_words;
            ns_util::JiebaUtil::CutString(doc.title, &title_words);
            for (auto& tw : title_words) {
                boost::to_lower(tw);
                word_map[tw].title_cnt++;
            }
            std::vector<std::string> content_words;
            ns_util::JiebaUtil::CutString(doc.content, &content_words);
            for (auto& cw : content_words) {
                boost::to_lower(cw);
                word_map[cw].content_cnt++;
            }
            #define X 10
            #define Y 1
            for (auto& word_pair : word_map) {
                InvertedElem item;
                item.doc_id = doc.doc_id;
                item.word = word_pair.first;
                item.weight = X * word_pair.second.title_cnt + Y * word_pair.second.content_cnt;
                inverted_index[word_pair.first].push_back(item);
            }
            return true;
        }
    };
    Index* Index::instance = nullptr;
    std::mutex Index::log;
}