C++ 搜索引擎通用工具类实现：文件读取与分词集成

3. 中文分词工具 (JiebaUsutl)

此处是对 cppjieba 非标准库的封装。通过静态成员变量管理 Jieba 实例，避免重复初始化带来的资源消耗。分词器依赖词典文件（如主词典、HMM 模型、停用词等），初始化成本较高，使用 static 修饰可确保整个程序运行期间仅初始化一次，且所有调用共享同一实例，保证数据一致性。

静态成员函数 CutString 允许直接通过类名调用，无需实例化对象，简化了使用方式。

const char* const DICT_PATH = "test/cppjieba/dict/jieba.dict.utf8";
const char* const HMM_PATH = "test/cppjieba/dict/hmm_model.utf8";
const char* const USER_DICT_PATH = "test/cppjieba/dict/user.dict.utf8";
const char* const IDF_PATH = "test/cppjieba/dict/idf.utf8";
const char* const STOP_WORD_PATH = "test/cppjieba/dict/stop_words.utf8";

class JiebaUsutl {
private:
    static cppjieba::Jieba jieba;

public:
    static void CutString(const std::string& src, std::vector<std::string>* out) {
        jieba.CutForSearch(src, *out);
    }
};

// 静态成员 jieba 的全局初始化，传入各词典路径
// 确保后续调用 CutString 时，jieba 对象已就绪
cppjieba::Jieba JiebaUsutl::jieba(DICT_PATH, HMM_PATH, USER_DICT_PATH, IDF_PATH, STOP_WORD_PATH);

4. 完整源码参考

以下是该模块整合后的完整头文件代码，可直接集成至项目中使用。

#pragma once
#include<iostream>
#include<string>
#include<fstream>
#include<boost/algorithm/string.hpp>
#include"cppjieba/Jieba.hpp"

namespace ns_util {
    class FileUtil {
    public:
        static bool ReadFile(const std::string &file_path, std::string *out) {
            std::ifstream in(file_path, std::ios::in | std::ios::binary);
            if (!in.is_open()) {
                std::cout << "open file " << file_path << ": error" << std::endl;
                return false;
            }
            std::string line;
            while (std::getline(in, line)) {
                *out += line;
            }
            in.close();
            return true;
        }
    };

    class StringUtil {
    public:
        static void Split(const std::string& target, std::vector<std::string>* out, std::string sep) {
            boost::split(*out, target, boost::is_any_of(sep), boost::token_compress_on);
        }
    };

    const char* const DICT_PATH = "test/cppjieba/dict/jieba.dict.utf8";
    const char* const HMM_PATH = "test/cppjieba/dict/hmm_model.utf8";
    const char* const USER_DICT_PATH = "test/cppjieba/dict/user.dict.utf8";
    const char* const IDF_PATH = "test/cppjieba/dict/idf.utf8";
    const char* const STOP_WORD_PATH = "test/cppjieba/dict/stop_words.utf8";

    class JiebaUsutl {
    private:
        static cppjieba::Jieba jieba;
    public:
        static void CutString(const std::string& src, std::vector<std::string>* out) {
            jieba.CutForSearch(src, *out);
        }
    };

    cppjieba::Jieba JiebaUsutl::jieba(DICT_PATH, HMM_PATH, USER_DICT_PATH, IDF_PATH, STOP_WORD_PATH);
}