// +---------------------------------------------------------------------- declare(strict_types=1); namespace ywxapp\library; use think\facade\Cache; /** * 真假蜘蛛识别(Forward-Confirmed rDNS 双向验证) * * 判定流程(业界标准做法,Google / Bing 官方文档推荐): * 1. PTR 反查:来访 IP → 主机名(如 123.125.71.1 → baiduspider-123-125-71-1.crawl.baidu.com) * 2. 后缀匹配:主机名必须落在该蜘蛛的官方域名后缀内(如 *.crawl.baidu.com / baidu.com) * 3. A/AAAA 正查:该主机名必须解析回**同一个来访 IP**(防 DNS 欺骗/自建 PTR) * 三步全过 = 真蜘蛛;任一步明确失败 = 假蜘蛛(伪造 UA);DNS 网络故障 = 待验证(不武断判假) * * 工程要点(决定能不能上生产): * - **自实现 UDP DNS 客户端**:PHP 原生 gethostbyaddr()/dns_get_record() 无超时参数, * 受 /etc/resolv.conf 的 timeout 5s × attempts 2 支配,最坏阻塞 10 秒—— * 绝不可放进请求周期。这里走 UDP 53 手写报文,硬超时 0.5s,完全可控。 * - **结果按 IP 缓存 24h**:蜘蛛 IP 高度集中,同一 IP 段反复抓取只查一次 DNS。 * - **熔断**:DNS 服务器连续失败 3 次 → 降级 5 分钟不查询(避免拖慢整站)。 * - 只对命中蜘蛛 UA 的请求调用(见 middleware/SpiderStat),普通用户零开销。 * * 返回状态(verified 字段): * 0 待验证(DNS 故障 / 该蜘蛛官方不提供 rDNS) * 1 真蜘蛛 * 2 假蜘蛛(伪造 UA) * * @author ywxapp */ class SpiderVerify { /** DNS 记录类型 */ private const T_A = 1; private const T_PTR = 12; private const T_AAAA = 28; /** IP 校验结果缓存时长(秒) */ public const TTL = 86400; /** 单次 DNS 查询硬超时(秒) */ public const DNS_TIMEOUT = 0.5; /** 缓存键前缀 */ private const CACHE_PREFIX = 'spider_verify:'; private const FAIL_KEY = 'spider_dns:fail'; private const DOWN_KEY = 'spider_dns:down'; /** 可配置 DNS 服务器(config/spider.php: dns_server),留空自动探测 */ private static ?string $serverOverride = null; /** * 校验来访 IP 是否为声明的蜘蛛 * * @param string $ip 来访 IP * @param string $spider 蜘蛛标识(SpiderDetect::SPIDERS 的 key) * @return array{status:int, rdns:string} status: 0待验证 1真 2假 */ public static function verify(string $ip, string $spider): array { $ip = trim($ip); if ($ip === '' || filter_var($ip, FILTER_VALIDATE_IP) === false) { return ['status' => 0, 'rdns' => '']; } $suffixes = SpiderDetect::rdnsSuffixes($spider); if ($suffixes === []) { // 该蜘蛛官方不提供 rDNS(如部分国内蜘蛛),无法校验 → 只按 UA 计数 return ['status' => 0, 'rdns' => '']; } $key = self::CACHE_PREFIX . $spider . ':' . $ip; $cached = self::cacheGet($key); if (is_array($cached) && isset($cached['status'])) { return ['status' => (int) $cached['status'], 'rdns' => (string) ($cached['rdns'] ?? '')]; } // DNS 配额:每分钟最多 120 次实查,防止伪造 UA + 海量不同 IP 造成的 DNS 放大 // (超配额时本次记为「待验证」,下分钟继续,不影响统计落库) if (self::rateLimited()) { return ['status' => 0, 'rdns' => '']; } $result = self::check($ip, $suffixes); // 只缓存「确定结论」(真 / 假);DNS 故障导致的待验证不缓存,下次访问重试 if ($result['status'] !== 0) { self::cacheSet($key, $result, self::TTL); } return $result; } /** * 清除某个 IP 的校验缓存(后台「重新校验」用) */ public static function forget(string $ip, string $spider = ''): void { if ($spider !== '') { self::cacheDel(self::CACHE_PREFIX . $spider . ':' . $ip); return; } // 不指定蜘蛛时,遍历已知蜘蛛逐一清除 foreach (array_keys(SpiderDetect::SPIDERS) as $key) { self::cacheDel(self::CACHE_PREFIX . $key . ':' . $ip); } } /** * 当前 DNS 是否处于熔断降级状态(后台可提示) */ public static function dnsDown(): bool { return (bool) self::cacheGet(self::DOWN_KEY); } /** * 手动指定 DNS 服务器(IP 或 IPv6) */ public static function setServer(string $server): void { self::$serverOverride = trim($server); } // ==================== 核心校验 ==================== /** * 三步校验:PTR → 后缀 → A/AAAA 正查 * * @param string $ip 来访 IP * @param string[] $suffixes 官方域名后缀 * @return array{status:int, rdns:string} */ private static function check(string $ip, array $suffixes): array { if (self::dnsDown()) { return ['status' => 0, 'rdns' => '']; } // ---------- 1) PTR 反查 ---------- $hosts = self::query(self::reverseName($ip), self::T_PTR); if ($hosts === null) { self::markDnsFail(); return ['status' => 0, 'rdns' => '']; } if ($hosts === []) { // DNS 正常响应但该 IP 无 PTR 记录:正规蜘蛛必然配置 PTR → 伪造 UA self::markDnsOk(); return ['status' => 2, 'rdns' => '']; } $host = strtolower(rtrim((string) $hosts[0], '.')); // ---------- 2) 官方域名后缀匹配 ---------- if (!self::matchSuffix($host, $suffixes)) { self::markDnsOk(); return ['status' => 2, 'rdns' => $host]; } // ---------- 3) A / AAAA 正查,必须解析回来访 IP ---------- // IPv4 来访只查 A,IPv6 来访只查 AAAA:少一次 DNS 往返(真蜘蛛通常 2 次查询完成) $lookupType = str_contains($ip, ':') ? self::T_AAAA : self::T_A; $records = self::query($host, $lookupType); if ($records === null) { // 主查询失败:再试另一族记录,仍失败则视为 DNS 故障 $fallback = self::query($host, $lookupType === self::T_A ? self::T_AAAA : self::T_A); if ($fallback === null) { self::markDnsFail(); return ['status' => 0, 'rdns' => $host]; } $records = $fallback; } $resolved = array_map([self::class, 'normalizeIp'], $records); if (!in_array(self::normalizeIp($ip), $resolved, true)) { // 域名对但正查不含该 IP:伪造 PTR / DNS 欺骗 self::markDnsOk(); return ['status' => 2, 'rdns' => $host]; } self::markDnsOk(); return ['status' => 1, 'rdns' => $host]; } // ==================== 熔断 & 限流 ==================== /** * 每分钟 DNS 实查配额(超限则本次跳过,不判假) */ private static function rateLimited(): bool { $key = 'spider_dns:rate:' . date('YmdHi'); $n = (int) self::cacheGet($key); if ($n >= 120) { return true; } self::cacheSet($key, $n + 1, 70); return false; } private static function markDnsFail(): void { $fails = (int) self::cacheGet(self::FAIL_KEY) + 1; self::cacheSet(self::FAIL_KEY, $fails, 300); if ($fails >= 3) { self::cacheSet(self::DOWN_KEY, 1, 300); } } private static function markDnsOk(): void { self::cacheDel(self::FAIL_KEY); } // ==================== DNS 客户端(自实现,硬超时) ==================== /** * 执行一次 DNS 查询 * * @param string $name 查询名(域名或 in-addr.arpa / ip6.arpa) * @param int $type 记录类型(T_A / T_PTR / T_AAAA) * @return array|null 记录字符串数组;null 表示网络层失败(超时/无响应/服务器故障) */ private static function query(string $name, int $type): ?array { $server = self::dnsServer(); $addr = str_contains($server, ':') ? "[{$server}]" : $server; $errno = 0; $err = ''; $sock = @stream_socket_client('udp://' . $addr . ':53', $errno, $err, self::DNS_TIMEOUT); if (!is_resource($sock)) { return null; } $sec = (int) floor(self::DNS_TIMEOUT); $usec = (int) round((self::DNS_TIMEOUT - $sec) * 1000000); stream_set_timeout($sock, $sec, $usec); $id = random_int(1, 65535); $packet = pack('n6', $id, 0x0100, 1, 0, 0, 0) . self::encodeName($name) . pack('n2', $type, 1); if (@fwrite($sock, $packet) === false) { fclose($sock); return null; } $resp = @fread($sock, 4096); $meta = stream_get_meta_data($sock); fclose($sock); if (!is_string($resp) || strlen($resp) < 12 || !empty($meta['timed_out'])) { return null; } return self::parseResponse($resp, $type, $id); } /** * 解析 DNS 响应报文 * * @return array|null null=服务器故障(触发熔断) */ private static function parseResponse(string $resp, int $type, int $id): ?array { $header = unpack('nid/nflags/nqd/nan/nns/nar', substr($resp, 0, 12)); if (!is_array($header) || (int) $header['id'] !== $id) { return null; } $rcode = ((int) $header['flags']) & 0x0F; if ($rcode === 3) { return []; // NXDOMAIN:域名/IP 不存在(正常响应) } if ($rcode !== 0) { return null; // SERVFAIL / REFUSED 等 → 视为服务器故障 } $len = strlen($resp); $off = 12; // 跳过 Question 段 for ($i = 0; $i < (int) $header['qd']; $i++) { $off = self::skipName($resp, $off) + 4; if ($off > $len) { return []; } } // 解析 Answer 段 $records = []; for ($i = 0; $i < (int) $header['an']; $i++) { $off = self::skipName($resp, $off); if ($off + 10 > $len) { break; } $meta = unpack('ntype/nclass/Nttl/nrdlen', substr($resp, $off, 10)); $off += 10; if (!is_array($meta)) { break; } $rdlen = (int) $meta['rdlen']; if ($off + $rdlen > $len) { break; } if ((int) $meta['type'] === $type) { $rdata = substr($resp, $off, $rdlen); if ($type === self::T_A && $rdlen === 4) { $records[] = implode('.', array_values(unpack('C4', $rdata))); } elseif ($type === self::T_AAAA && $rdlen === 16) { $ip = @inet_ntop($rdata); if (is_string($ip)) { $records[] = $ip; } } elseif ($type === self::T_PTR) { $records[] = self::decodeName($resp, $off); } } $off += $rdlen; } return $records; } /** * 域名 → DNS 报文字节(label 序列) */ private static function encodeName(string $name): string { $out = ''; foreach (explode('.', trim($name, '.')) as $label) { $len = strlen($label); if ($len === 0 || $len > 63) { continue; } $out .= chr($len) . $label; } return $out . "\x00"; } /** * 跳过报文中的域名(支持压缩指针),返回下一个字段的偏移 */ private static function skipName(string $packet, int $off): int { $len = strlen($packet); while ($off < $len) { $b = ord($packet[$off]); if ($b === 0) { return $off + 1; } if (($b & 0xC0) === 0xC0) { return $off + 2; } $off += 1 + $b; } return $len; } /** * 解码报文中的域名(支持压缩指针) */ private static function decodeName(string $packet, int $off): string { $labels = []; $len = strlen($packet); $jumped = false; $guard = 0; while ($off < $len && $guard++ < 64) { $b = ord($packet[$off]); if ($b === 0) { break; } if (($b & 0xC0) === 0xC0) { if ($jumped || $off + 1 >= $len) { break; // 只允许跳转一次,防解析环 } $jumped = true; $off = (($b & 0x3F) << 8) | ord($packet[$off + 1]); continue; } $labels[] = substr($packet, $off + 1, $b); $off += 1 + $b; } return implode('.', $labels); } /** * IP → PTR 查询名(1.2.3.4 → 4.3.2.1.in-addr.arpa;IPv6 → nibble 反转 + ip6.arpa) */ private static function reverseName(string $ip): string { if (str_contains($ip, ':')) { $packed = @inet_pton($ip); if (!is_string($packed)) { return ''; } $hex = bin2hex($packed); return implode('.', array_reverse(str_split($hex))) . '.ip6.arpa'; } return implode('.', array_reverse(explode('.', $ip))) . '.in-addr.arpa'; } /** * IP 归一化:IPv4-mapped IPv6(::ffff:1.2.3.4)还原为 IPv4,统一小写 */ private static function normalizeIp(string $ip): string { $ip = strtolower(trim($ip)); if (str_starts_with($ip, '::ffff:')) { $tail = substr($ip, 7); if (filter_var($tail, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) !== false) { return $tail; } } return $ip; } /** * 主机名是否落在官方域名后缀内(按 label 边界匹配,防 evil-baidu.com 冒充 baidu.com) * * @param string[] $suffixes */ private static function matchSuffix(string $host, array $suffixes): bool { foreach ($suffixes as $suffix) { $suffix = strtolower(trim((string) $suffix, '. ')); if ($suffix === '') { continue; } if ($host === $suffix || str_ends_with($host, '.' . $suffix)) { return true; } } return false; } /** * DNS 服务器:配置 > /etc/resolv.conf > 公共 DNS */ private static function dnsServer(): string { if (self::$serverOverride !== null && self::$serverOverride !== '') { return self::$serverOverride; } static $detected = null; if ($detected !== null) { return $detected; } $detected = '223.5.5.5'; // 阿里公共 DNS(国内解析快,国外亦可用) try { $cfg = config('spider.dns_server'); if (is_string($cfg) && trim($cfg) !== '') { return $detected = trim($cfg); } } catch (\Throwable $e) { // 配置不可用则走探测 } $conf = @file_get_contents('/etc/resolv.conf'); if (is_string($conf) && preg_match('/^\s*nameserver\s+([0-9a-fA-F:.]+)/m', $conf, $m)) { $detected = $m[1]; } return $detected; } // ==================== 缓存小工具(缓存不可用时不抛异常) ==================== private static function cacheGet(string $key) { try { return Cache::get($key); } catch (\Throwable $e) { return null; } } private static function cacheSet(string $key, $value, int $ttl): void { try { Cache::set($key, $value, $ttl); } catch (\Throwable $e) { // 缓存不可用:退化为每次实时校验 } } private static function cacheDel(string $key): void { try { Cache::delete($key); } catch (\Throwable $e) { // ignore } } }