// +---------------------------------------------------------------------- declare(strict_types=1); namespace ywxapp\middleware; use think\facade\Db; use ywxapp\model\BaseModel; use ywxapp\library\SpiderDetect; use ywxapp\library\SpiderVerify; /** * 全站搜索蜘蛛统计中间件(全局注册于 app/middleware.php) * * 设计要点: * - 仅命中蜘蛛 UA 才落库,普通用户请求零额外查询; * - 落库在响应生成之后($next 之后),不阻塞蜘蛛抓取响应; * - 明细写 spider_log,按日聚合 upsert 到 spider_stat(报表免全表扫描); * - 任何数据库异常静默吞掉(统计绝不能影响业务),缺表时经 BaseModel 自愈引擎 自愈一次; * - install.sql 为唯一事实源,此处 DDL 仅为老库运行时兜底。 * * 真伪识别(v2 新增): * - UA 命中后调用 SpiderVerify 做 rDNS 双向验证(PTR + 官方后缀 + A/AAAA 正查); * - 结果写入 verified(0待验证 / 1真蜘蛛 / 2假蜘蛛)与 rdns(反查主机名); * - 聚合表额外维护 fake_count,后台可直观看到「多少是伪造 UA 刷的」; * - 校验结果按 IP 缓存 24h,DNS 查询有硬超时与熔断,不会拖慢站点。 * * @author ywxapp */ class SpiderStat { public function handle($request, \Closure $next) { $response = $next($request); try { $ua = (string) $request->header('user-agent', ''); $spider = SpiderDetect::detect($ua); if ($spider !== null) { $this->record($request, $response, $spider, $ua); } } catch (\Throwable $e) { // 统计失败绝不影响正常响应 } return $response; } /** * 落库:明细 + 按日聚合(含真伪判定) */ protected function record($request, $response, string $spider, string $ua): void { $prefix = config('database.connections.mysql.prefix', 'wxapp_'); $logTable = $prefix . 'common_spider_log'; $statTable = $prefix . 'common_spider_stat'; $ip = mb_substr((string) $request->ip(), 0, 45); // 真伪校验:命中缓存则零 DNS 开销;仅对配置了官方 rDNS 的蜘蛛生效 $verdict = SpiderVerify::verify($ip, $spider); $data = [ 'spider' => $spider, 'url' => mb_substr((string) $request->url(), 0, 500), 'ip' => $ip, 'app' => mb_substr((string) (app('http')->getName() ?: ''), 0, 20), 'user_agent' => mb_substr($ua, 0, 500), 'http_code' => method_exists($response, 'getCode') ? (int) $response->getCode() : 200, 'verified' => (int) $verdict['status'], 'rdns' => mb_substr((string) $verdict['rdns'], 0, 255), 'create_at' => time(), ]; try { $this->insert($logTable, $statTable, $data, $spider); } catch (\Throwable $e) { // 表可能不存在或老库缺列:自愈一次后重试 $this->ensureTables($logTable, $statTable); $this->insert($logTable, $statTable, $data, $spider); } } /** * 写明细 + 聚合 upsert */ protected function insert(string $logTable, string $statTable, array $data, string $spider): void { Db::table($logTable)->insert($data); // 按日聚合:主键(stat_date, spider),存在则累加;fake_count 只累加假蜘蛛次数 Db::execute( "INSERT INTO `{$statTable}` (`stat_date`, `spider`, `count`, `fake_count`) VALUES (?, ?, 1, ?) " . 'ON DUPLICATE KEY UPDATE `count` = `count` + 1, `fake_count` = `fake_count` + VALUES(`fake_count`)', [date('Y-m-d'), $spider, (int) ($data['verified'] ?? 0) === 2 ? 1 : 0] ); } /** * 缺表/缺列自愈(DDL 与 public/install/install.sql 保持一致) */ protected function ensureTables(string $logTable, string $statTable): void { BaseModel::ensureTable($logTable, "CREATE TABLE IF NOT EXISTS `{$logTable}` ( `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT '主键ID', `spider` varchar(20) NOT NULL DEFAULT '' COMMENT '蜘蛛标识', `url` varchar(500) NOT NULL DEFAULT '' COMMENT '抓取URL', `ip` varchar(45) NOT NULL DEFAULT '' COMMENT '来源IP', `app` varchar(20) NOT NULL DEFAULT '' COMMENT '应用名', `user_agent` varchar(500) NOT NULL DEFAULT '' COMMENT 'User-Agent', `http_code` smallint unsigned NOT NULL DEFAULT '200' COMMENT '响应状态码', `verified` tinyint(1) NOT NULL DEFAULT '0' COMMENT '真伪:0待验证 1真蜘蛛 2假蜘蛛', `rdns` varchar(255) NOT NULL DEFAULT '' COMMENT 'rDNS反查主机名', `create_at` int NOT NULL DEFAULT '0' COMMENT '抓取时间', PRIMARY KEY (`id`), KEY `idx_spider_time` (`spider`,`create_at`), KEY `idx_create_at` (`create_at`), KEY `idx_verified` (`verified`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='搜索蜘蛛抓取日志'"); BaseModel::ensureTable($statTable, "CREATE TABLE IF NOT EXISTS `{$statTable}` ( `stat_date` date NOT NULL COMMENT '统计日期', `spider` varchar(20) NOT NULL DEFAULT '' COMMENT '蜘蛛标识', `count` int unsigned NOT NULL DEFAULT '0' COMMENT '抓取次数', `fake_count` int unsigned NOT NULL DEFAULT '0' COMMENT '其中假蜘蛛次数', PRIMARY KEY (`stat_date`,`spider`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='搜索蜘蛛按日统计'"); // 老库补列(表已存在但缺新字段) BaseModel::ensureColumn($logTable, 'verified', "tinyint(1) NOT NULL DEFAULT '0' COMMENT '真伪:0待验证 1真蜘蛛 2假蜘蛛'"); BaseModel::ensureColumn($logTable, 'rdns', "varchar(255) NOT NULL DEFAULT '' COMMENT 'rDNS反查主机名'"); BaseModel::ensureColumn($statTable, 'fake_count', "int unsigned NOT NULL DEFAULT '0' COMMENT '其中假蜘蛛次数'"); } }