背景
在 AI 辅助分析问题的场景中，明确指定抓取范围往往依赖模型自身能力。虽然可以通过提示词控制或使用搜索智能体（如 Tavily Search），但引入 Skills 机制可以更丰富地扩展大模型能力。本文旨在分享如何开发一个网页内容抓取 Skill。
Skills 项目结构
skill-name/
├── SKILL.md (唯一必需)
│ ├── YAML 格式 (name, description 必须)
│ └── Markdown instructions
└── Bundled Resources (可选)
├── scripts/ (存放可执行脚本)
├── references/ (文档、API 说明)
├── examples/ (示例文件)
├── evals/ (测试说明)
└── assets/ (模板、图标)
SKILL.md 元数据
| 字段 | 必填 | 说明 |
|---|---|---|
| name | 是 | Skill 显示名称 |
| description | 是 | 技能用途及使用场景 |
| argument-hint | 否 | 参数提示 |
| disable-model-invocation | 否 | 禁止自动触发 |
| user-invocable | 否 | 是否隐藏于菜单 |
| allowed-tools | 否 | 激活时可无授权使用的工具 |
| model | 否 | 使用的模型 |
| context | 否 | 子代理上下文 |
| agent | 否 | 子代理类型 |
| hooks | 否 | 生命周期钩子 |
开发环境
Skills 采用 Prompt + Scripts 架构，Scripts 必须绑定特定运行时环境。本案例使用 Node.js，需配置 node_modules 及 package.json。
开发案例
本项目名为 website-content-fetch，主要实现获取网页文本内容及识别媒体文件 URL。
SKILL.md
---
name: website-content-fetch
description: Fetch and extract content from websites. Use this skill whenever the user mentions fetching website content, extracting text from web pages, or needing to get content from a URL, even if they don't explicitly ask for a 'website content fetch' skill.
---
package.json
{
"name": "website-content-fetch",
"version": "1.0.0",
"description": "fetching website content",
"main": "scripts/fetch-content.js",
"scripts": {
"test": "node scripts/fetch-content.js"
},
"keywords": ["openclaw", "skill", "website", "content", "fetch"],
"license": "MIT",
"dependencies": {
"axios": "^1.6.2",
"cheerio": "^1.0.0-rc.12"
}
}
scripts/fetch-content.js
const axios = require("axios");
const cheerio = require("cheerio");
const path = require("path");
const fs = require("fs");
/**
* Fetch website content
* @param {string} url - The URL to fetch
* @param {object} options - Optional parameters
* @param {string} options.saveDir - Directory to save media files (optional)
* @returns {Promise<object>} - The fetched content and metadata
*/
async function fetchWebsiteContent(url, options = {}) {
try {
const response = await axios.get(url);
const $ = cheerio.load(response.data);
// Extract text content
let content = $("body").text().trim();
content = content.replace(/\s+/g, " ");
// Extract media resources
const media = { images: [], videos: [], audios: [] };
// Extract images
$("img").each((i, elem) => {
const src = $(elem).attr("src");
const alt = $(elem).attr("alt") || "";
if (src) {
const absoluteUrl = new URL(src, url).href;
media.images.push({ url: absoluteUrl, alt: alt });
}
});
// Extract videos
$("video, iframe").each((i, elem) => {
let src = $(elem).attr("src");
if (!src && $(elem).attr("data-src")) {
src = $(elem).attr("data-src");
}
if (src) {
const absoluteUrl = new URL(src, url).href;
media.videos.push({ url: absoluteUrl });
}
});
// Extract audios
$("audio").each((i, elem) => {
const src = $(elem).attr("src");
if (src) {
const absoluteUrl = new URL(src, url).href;
media.audios.push({ url: absoluteUrl });
}
});
// Save media files if saveDir is provided
if (options.saveDir) {
if (!fs.existsSync(options.saveDir)) {
fs.mkdirSync(options.saveDir, { recursive: true });
}
// Save images
for (let i = 0; i < media.images.length; i++) {
const image = media.images[i];
try {
const imageResponse = await axios.get(image.url, { responseType: "stream" });
const imageName = `image_${i}_${path.basename(new URL(image.url).pathname)}`;
const imagePath = path.join(options.saveDir, imageName);
const writer = fs.createWriteStream(imagePath);
imageResponse.data.pipe(writer);
await new Promise((resolve, reject) => {
writer.on("finish", () => resolve());
writer.on("error", reject);
});
image.localPath = imagePath;
} catch (error) {
console.error(`Error saving image ${image.url}:`, error.message);
}
}
// Save videos
for (let i = 0; i < media.videos.length; i++) {
const video = media.videos[i];
try {
const videoResponse = await axios.get(video.url, { responseType: "stream" });
const videoName = `video_${i}_${path.basename(new URL(video.url).pathname)}`;
const videoPath = path.join(options.saveDir, videoName);
const writer = fs.createWriteStream(videoPath);
videoResponse.data.pipe(writer);
await new Promise((resolve, reject) => {
writer.on("finish", () => resolve());
writer.on("error", reject);
});
video.localPath = videoPath;
} catch (error) {
console.error(`Error saving video ${video.url}:`, error.message);
}
}
// Save audios
for (let i = 0; i < media.audios.length; i++) {
const audio = media.audios[i];
try {
const audioResponse = await axios.get(audio.url, { responseType: "stream" });
const audioName = `audio_${i}_${path.basename(new URL(audio.url).pathname)}`;
const audioPath = path.join(options.saveDir, audioName);
const writer = fs.createWriteStream(audioPath);
audioResponse.data.pipe(writer);
await new Promise((resolve, reject) => {
writer.on("finish", () => resolve());
writer.on("error", reject);
});
audio.localPath = audioPath;
} catch (error) {
console.error(`Error saving audio ${audio.url}:`, error.message);
}
}
}
return { content, length: content.length, url, media };
} catch (error) {
console.error("Error fetching website content:", error);
throw new Error(`Failed to fetch content from ${url}: ${error.message}`);
}
}
// If run directly, test the function
if (require.main === module) {
const url = process.argv[2] || "https://example.com";
const saveDir = process.argv[3];
const options = {};
if (saveDir) {
options.saveDir = saveDir;
}
fetchWebsiteContent(url, options).then((result) => {
console.log("Fetched content:");
console.log(`URL: ${result.url}`);
console.log(`Length: ${result.length} characters`);
console.log("Content:");
console.log(result.content);
console.log("\nMedia resources:");
if (result.media.images.length > 0) {
console.log("Images:");
result.media.images.forEach((image, index) => {
console.log(`${index + 1}. ${image.url} (alt: ${image.alt})`);
if (image.localPath) {
console.log(` Saved to: ${image.localPath}`);
}
});
}
if (result.media.videos.length > 0) {
console.log("\nVideos:");
result.media.videos.forEach((video, index) => {
console.log(`${index + 1}. ${video.url}`);
if (video.localPath) {
console.log(` Saved to: ${video.localPath}`);
}
});
}
if (result.media.audios.length > 0) {
console.log("\nAudios:");
result.media.audios.forEach((audio, index) => {
console.log(`${index + 1}. ${audio.url}`);
if (audio.localPath) {
console.log(` Saved to: ${audio.localPath}`);
}
});
}
}).catch((error) => {
console.error("Error:", error.message);
});
}
module.exports = { fetchWebsiteContent };
evals.json
{
"skill_name": "website-content-fetch",
"evals": [
{
"id": 1,
"prompt": "Fetch content from https://example.com",
"expected_output": "Should return the text content of example.com",
"files": []
},
{
"id": 2,
"prompt": "Fetch content from a nonexistent domain",
"expected_output": "Should throw an error about failed to fetch content",
"files": []
},
{
"id": 3,
"prompt": "Fetch content without providing a URL",
"expected_output": "Should throw an error about URL being required",
"files": []
}
]
}
测试与集成
将开发的 skill 目录放入支持 skill 的 AI 助手或 IDE 的对应目录中即可被识别。使用时只需告知 AI 工具使用对应的 skill 抓取目标 URL，系统会自动检测环境并安装依赖后执行任务。开发过程中需反复测试结果，调试脚本功能，优化逻辑以增强 AI 对工具的理解和执行精度。