WebRPA： 分布式网络爬虫

前言

webrpa 是一个分布式的网络爬虫系统，基于 fastapi+fastadmin 开发，通过 web api 接口发起网络爬虫服务，实现流程自动化或数据自动抓取。它包含两部分：

• manager： 用于提供 web 服务，实现了 web server 和 websocket server 。用户通过 web api 发送请求，manager 会将查询请求通过 websocket 转发给对应的 worker ，由 worker 执行查询操作。
• worker： 通过 websocket 和 manager 连接，用于执行网络爬虫任务，接收 manager 的 websocket 请求，并执行查询操作。worker 可以跨网络分布式部署。

graph LR client-->manager-->worker1 manager-->worker2 manager-->workers[worker...] 

主要实现的功能包括：

• 爬虫流程自定义：通过 json 指定访问网页的各种操作，实现流程自动化，并支持通过 markdown flowchart 语法定义流程，实现在不同条件下的跳转
• 数据自动抓取：支持通过 js 对网页数据进行处理，获取结构化的数据以及对页面某个区域进行自动截图
• 跨网络分布式部署：worker 可以跨网络分布式部署，实现负载均衡，并可以根据查询类型不同指定专用的 worker 。
• 代理池支持：支持自定义代理池
• undetected chromedriver：支持无头浏览器以及绕开网站的反扒检测。支持通过 selenium 和 requests 组合查询
• 自定义 captcha 引擎：支持对图形验证码或者其他验证码的自动识别和点击操作等操作。并增加随机扰动，避免 antibot 检测。
• 缓存和自动重试：支持缓存和自动重试，查询失败的请求会自动在闲时重试。
• 自动扩容：支持 k8s 部署，一键扩容。
• 权限和审计：支持数据源权限模型，不同的用户对不同的数据源具备不同的权限，并提供审计数据。

TODO

引入 browser use ，通过 LLM 自动创建数据爬虫服务。

一个示例，爬取某网站

{ "name": "szreorc", "desc": "深圳不动产查询", "driver": "firefox", "url": "", "debug": true, "window_size": "1920x1080", "action_timeout": 5, "wait_redirect": true, "wait_redirect_interval": 2, "identifier": "{username}-{BuildingName}-{UNIT_NO}", "credential": "{username}", "actions": { "1": { "desc": "确认登录", "action": "check_variable", "options": {"script": "return window.location.href;", "target": "^https://pnr.sz.gov.cn/d-ghrer/reroosp/ytcf" } }, "10": { "desc" : "用户名密码登录", "action": "click", "timeout": 2, "target": ["xpath", "//a[contains(@class, 'login-tab') and normalize-space(text())='账号密码']"] }, "11": { "desc" : "输入用户名", "action": "input_text", "target": ["xpath", "//input[@type='text' and @placeholder='请输入账号']"], "param": "username" }, "12": { "desc": "增加计数", "action": "variable", "options": {"variable":"counter1","operator": "+"} }, "13": { "desc": "检测计数", "action": "variable", "stop_on_fail": true, "options": {"variable":"counter1","operator": "<", "target": 2, "sleep": 2000} }, "14": { "desc" : "输入密码", "action": "input_text", "target": ["xpath", "//input[@type='password' and @placeholder='请输入密码']"], "param": "password" }, "15": { "desc": "识别 captcha", "action": "decode_captcha_code", "target": ["xpath","//div[contains(@class, 'captcha-body') and @title='点击刷新']"], "options": {"code_type": 11} }, "16": { "desc": "输入 captcha", "action": "input_text", "target": ["xpath","//div[contains(@class, 'account_verifying')] //input[@type='text']"] }, "17": { "desc": "点击登录", "action": "click", "target": ["xpath", "//button[contains(@class, 'gd-btn-primary') and contains(@class, 'gd-btn') and @type='button']//span[starts-with(text(), '登录 ')]"] }, "18": { "desc": "继续登录", "action": "click", "target": ["xpath", "//button[.//span[contains(text(), '继续登录')]]"] }, "20": { "desc": "确认选择", "action": "click", "timeout": 10, "stop_on_fail": true, "fail_message": "login failed", "options": {"set_credential": true}, "target": ["class name", "jinruxuzhi-checkbox"] }, "21": { "desc": "确认选择下一步", "action": "click", "target": ["class name", "jinruxuzhi-buttonOk"] }, "30": { "desc": "展开查询类型", "action": "click", "options": {"sleep": 2}, "target": ["xpath", "//input[@type='text' and @placeholder='请选择']"] }, "31": { "desc": "等待下拉菜单", "action": "wait_element", "options": {"visible": true}, "target": ["css selector", "div.el-select-dropdown.el-popper"] }, "32": { "desc": "选择查询类型", "action": "click", "target": ["xpath", "//li[contains(@class, 'el-select-dropdown__item') and span[text()='楼名及栋名']]"] }, "33": { "desc" : "输入查询内容", "action": "input_text", "target": ["xpath", "//input[@type='text' and @placeholder='请输入内容']"], "param": "BuildingName" }, "34": { "desc": "点击查询", "action": "click", "target": ["class name", "el-icon-search"] }, "35": { "desc": "点击截图对象", "action": "click", "timeout": 20, "stop_on_fail": true, "fail_message": "search failed", "target": ["xpath", "//div[contains(@class, 'el-dialog__wrapper')]//div[contains(@class, 'el-tabs__item') and normalize-space(text())='楼宇']"] }, "40": { "desc": "获取数据", "action": "get_data", "options": {"script": "var table = document.querySelector(\"#pane-1 table.is-bordered.el-descriptions--mini\");\nvar fields = [\"土地坐落\", \"楼名及栋名\", \"房屋类型\", \"房屋性质\", \"房屋用途\"];\nvar result = {};\nif (table) {\n var rows = table.querySelectorAll(\"tr.el-descriptions-row\");\n rows.forEach(function(row) {\n var label = row.querySelector(\"th.el-descriptions-item__label\").innerText.trim();\n var cOntent= row.querySelector(\"td.el-descriptions-item__content\").innerText.trim();\n if (fields.includes(label)) {\n result[label] = content;\n }\n });\n console.log(JSON.stringify(result));\n} else {\n console.log(\"Table not found.\");\n};\nreturn result;\n"} }, "41": { "desc": "点击截图对象", "action": "click", "target": ["xpath", "//div[contains(@class, 'el-dialog__wrapper')]//div[contains(@class, 'el-tabs__item') and normalize-space(text())='房屋']"] }, "42": { "desc": "下拉房屋查询", "action": "click", "target": ["css selector", "#pane-2 input.el-input__inner"] }, "43": { "desc": "点击房屋查询", "action": "click", "target": ["xpath", "//li[contains(@class, 'el-select-dropdown__item')]//span[text()='{UNIT_NO}']"], "param": "UNIT_NO" }, "44": { "desc": "截图", "action": "screenshot", "target": ["class name", "el-dialog__wrapper"], "options": {"visible": true} }, "45": { "desc": "获取数据", "action": "get_data", "options": {"script": "var table = document.querySelector(\"#pane-2 table.is-bordered.el-descriptions--mini\");\nvar fields = [\"房号\", \"所在楼层\", \"建筑面积\", \"使用年限\", \"存在抵押\", \"存在查封\", \"存在异议\", \"存在居住权\"];\nvar result = {};\nif (table) {\n var rows = table.querySelectorAll(\"tr.el-descriptions-row\");\n rows.forEach(function(row) {\n var label = row.querySelector(\"th.el-descriptions-item__label\").innerText.trim();\n var cOntent= row.querySelector(\"td.el-descriptions-item__content\").innerText.trim();\n if (fields.includes(label)) {\n result[label] = content;\n }\n });\n console.log(JSON.stringify(result));\n} else {\n console.log(\"Table not found.\");\n};\nreturn result;\n"} } }, "processes": "start->1\n1(no)->10->11\n11(no)->12->13\n13(yes)->10\n11(yes)->14->15->16->17->18->20->21->30->31->32->33->34->35->40->41->42->43->44->45->end\n1(yes)->20", "result":["screenshot", "data"] }