chrome extensions cdp(通过debugger

在爬虫开发过程中，踩过无数的坑，与目标方斗智斗勇。总结来看终归是成本的博弈，不管开发成本，时间成本，空间成本，拟或是其他。方案万千，权衡后低成本拿下才是王道，当成本超过预期也就放弃挣扎了。

在某些安全至上的行业，时间、空间成本通常只能往后靠。优先安全的情况下，完全模拟用户行为几乎是最为有效的。想想那些各种加密，接口鉴权，请求策略…，为了最后几根头发妥协吧！

本文是cdp第三种使用方式，目的都是为了嗅探接口数据（为啥嗅探？你去看看强鉴权的网站）。某些变态的网站，selenium cdp、chrome带启动参数cdp都会检测，导致登录或鉴权错误。走投无路的情况下，采用了chrome扩展，通过debugger来实现。

浏览器启动

未避免debugger弹调试弹框，启动项增加--silent-debugger-extension-api配置,如下：

# 跨域、debugger api、指定嗅探扩展目录
"C:\Program Files (x86)\Google\Chrome\Application\chrome.exe" --disable-web-security --user-data-dir="d:\aaa" --silent-debugger-extension-api --load-extension="d:\嗅探"

manifest.json

{
   "background": {
      "persistent": true,
      "scripts": [ "background.js" ]
   },
   "browser_action": {
      "default_icon": "icon_38.png"
   },
   "content_security_policy": "script-src 'self' 'unsafe-eval'; object-src 'self'",
   "description": "网页嗅探",
   "icons": {
      "128": "icon.png",
      "16": "icon.png",
      "48": "icon.png"
   },
   "manifest_version": 2,
   "name": "网页嗅探",
   "permissions": [ "debugger", "storage", "notifications", "tts", "webRequest", "webRequestBlocking", "http://*/*","https://*/*", "tabs", "contextMenus", "webNavigation", "clipboardWrite", "clipboardRead" ],
   "short_name": "网页嗅探",
   "version": "2.7"
}

background.js

本来是准备用Native Message的，但是太复杂了。索性用nodejs简单搭了个web服务用着，脚手架也方便。

console.log("开启嗅探");

//循环定时器，获取规则
var timename = setInterval(initRule, 1500);

// 跨域jsonp
function xhr(url, method, data, callback) {
    url = url || "http://127.0.0.1:8080/";
    method = method || "POST";
    data = data ? (typeof (data) == "object" ? JSON.stringify(data) : data) : "";
    var xhr = new XMLHttpRequest();
    xhr.open(method, url, true);
    xhr.setRequestHeader("Content-Type", "application/json");
    xhr.onload = callback || function () {
        console.log(this.responseText);
    };
    xhr.send(data);
}

// 初始化规则
function initRule() {
    var url = "http://127.0.0.1:8080/rule";
    var callback = function () {
        // 删除定时循环
        clearInterval(timename);
        var rule = this.responseText;
        rule = JSON.parse(rule);
        console.log(rule);
        handleRule(rule);
    };
    xhr(url, "GET", "", callback);
}

// 规则解析
function handleRule(rule) {
    // tab页嗅探规则，指定对哪些域名对应的页面嗅探
    var domain = rule["tabDomain"];
    domain.forEach(function (i, b, c) {
        c[b] = i.replace(/\./ig, "\\.");
    });
    domain = domain.join("|");
    var w_Domain = new RegExp("^https?:\\/\\/(" + domain + ")");
    var b_Domain = new RegExp("^https?:\\/\\/(?!" + domain + ")");
    // 嗅探url过滤
    var fu_Filters = rule["fetchUrlFilters"];
    fu_Filters.forEach(function (i, b, c) {
        c[b] = new RegExp(i);
    });
    initListener(w_Domain, b_Domain, fu_Filters);
}

// 过滤嗅探到的url，
function fetchUrlFilter(fu_Filters, url) {
    for (ft in fu_Filters) {
        if (fu_Filters[ft].test(url)) {
            return true;
        }
    }
    return false;
}

// 创建监听
function initListener(w_Domain, b_Domain, fu_Filters) {
    // 全局tab缓存
    var tab_cache = {};

    // tab页存在更新动作
    chrome.tabs.onUpdated.addListener(function (id, info, tab) {
        console.log(id + ":tab 更新\t" + JSON.stringify(info));
        //var index = global_tab.indexOf(id);
        var index = tab_cache[id + ""];
        // 释放监听
        if (info.status == "loading" && index && b_Domain.test(tab["url"])) {
            chrome.debugger.detach({
                "tabId": id
            }, function () {
                console.log(id + ":tab debugger解绑");
                // 删除监控，这里不安全，异步没有保护。可能其他删除，导致下标不一致。
                delete tab_cache[id + ""]["fetch_urls"];
                delete tab_cache[id + ""]["webSocket_urls"];
                delete tab_cache[id + ""];
            });
        } else if (info.status == "loading" && !index && w_Domain.test(tab["url"])) {
            // 添加监听
            tab_cache[id + ""] = {};
            tab_cache[id + ""]["fetch_urls"] = {};
            tab_cache[id + ""]["webSocket_urls"] = {};

            chrome.debugger.attach({
                "tabId": id
            }, "1.0", function () {
                console.log(id + ":tab debugger绑定");
                chrome.debugger.sendCommand({
                    "tabId": id
                }, "Network.enable", {}, function () {
                    console.log(id + ":tab Network.enable");
                    chrome.debugger.onEvent.addListener(function (source, method, params) {
                        console.log(id + ":tab debugger event fetch");
                        var requestId = params.requestId;
                        var fetch_urls = tab_cache[id + ""]["fetch_urls"];
                        var webSocket_urls = tab_cache[id + ""]["webSocket_urls"];

                        switch (method) {
                            case "Network.requestWillBeSent":
                                var feg = fetchUrlFilter(fu_Filters, params.request.url);
                                if (feg) {
                                    fetch_urls[requestId] = {};
                                    fetch_urls[requestId]["request"] = params["request"];
                                }
                                break;
                            case "Network.responseReceived":
                                if (fetch_urls[requestId]) {
                                    fetch_urls[requestId]["ResponseHeaders"] = params["response"]["headers"];
                                }
                                break;
                            case "Network.loadingFinished":
                                if (fetch_urls[requestId]) {
                                    console.log(method + "\t" + fetch_urls[requestId].request
                                        .url);
                                    chrome.debugger.sendCommand(source,
                                        "Network.getResponseBody", {
                                            "requestId": requestId
                                        },
                                        function (response) {
                                            var body = {};
                                            fetch_urls[requestId]["tabId"] = source.tabId;
                                            fetch_urls[requestId]["response"] = response;
                                            var callback = function () {
                                                console.log(this.responseText);
                                                // 删除requestId，减小缓存
                                                delete fetch_urls[requestId];
                                            };
                                            // 传输嗅探结果
                                            xhr(null, null, fetch_urls[requestId],
                                                callback);
                                        });
                                }
                                break;
                            case "Network.webSocketCreated":
                                var feg = fetchUrlFilter(fu_Filters, params.url);
                                if (feg) {
                                    webSocket_urls[requestId] = params.url;
                                }
                                break;
                            case "Network.webSocketFrameReceived":
                                if (webSocket_urls[requestId]) {
                                    var data = params;
                                    data["url"] = webSocket_urls[params.requestId];
                                    data["tabId"] = source.tabId;
                                    // 传输嗅探结果,websocket是长链接，不能删缓存
                                    xhr(null, null, data);
                                }
                                break;
                            default:
                                break;
                        }
                    });
                });
            });
        }
    });

    // Cleanup the variables when a tab is closed
    chrome.tabs.onRemoved.addListener(function (id) {
        console.log(id + ":tab 关闭");
        var index = tab_cache[id + ""];
        if (index) {
            delete tab_cache[id + ""]["fetch_urls"];
            delete tab_cache[id + ""]["webSocket_urls"];
            delete tab_cache[id + ""];
        }
    });
}

rule.json 嗅探规则

每当浏览器启动时，扩展会每1.5秒请求一次rule规则，直到请求到为止。

{
    "tabDomain": [
        "xxxx.xxxx.com",
        "xxxx.xxxx.com"
    ],
    "fetchUrlFilters": [
        "\\.json",
    ]
}

rule.json说明：
1、浏览器嗅规则，扩展在启动后每隔1.5秒获取一次，请求成功则停止获取。如果想新规则生效，则重启浏览器。
2、浏览器启动参数附带--silent-debugger-extension-api，解决浏览器弹调试框问题。
3、规则：
tabDomain：要嗅探的域名，这里必须是完整的域名。对应浏览器的地址栏的url，进行域名过滤。tab刷新会实时监控，不用担心重复会丢失嗅探。
fetchUrlFilters：要抓取的目标请求地址，正则表达式，用来过滤出要抓取的内容。

最后

嗅探的结构会post到web服务，至于怎么处理嗅探结果，就具体分析了。