yangwx
18101cf11a
安全（P0） - 发布包签名：新增 scripts/package_keygen.php、scripts/package_verify.php、 ywxapp\service\PackageVerifier；FrameworkService::apply() 解压前强制验签 （ywxapp.framework_package_verify，默认开启） - JWT 收紧：config/jwt.php 移除公开默认密钥，JwtService::assertStrongSecret() 校验非空 / 长度≥32 / 已知弱值；.env.production.example 键名由 JWT_SECRET_KEY 修正为 JWT_SECRET - /api/v1/Article：免登录白名单收窄为只读接口，写操作字段白名单 + assertOwner() 越权校验 - 开发者令牌：查找接口不再回显明文、统一失败文案 + 限流 - TemplateInstaller：文件名白名单 + realpath 归一化 + 目录内断言，修复 Zip Slip；补 use ZipArchive - 后台登录：接入验证码、失败锁定与登录限流（config/ratelimit.php） - 缓存 / Session 驱动可由 .env 覆盖（CACHE_DRIVER / SESSION_TYPE），集群部署改 redis - paycenter 支付日志开关改为 PAY_LOG，不再跟随 APP_DEBUG 工程化 / 交付 - 新增 CHANGELOG.md 与 .env.production.example；composer.lock 入库锁定依赖版本 - 插件离线安装 CLI 增加 --uid，后台上传安装透传 uid，付费插件授权校验可通过 - 授权拒绝文案区分「未携带 uid」与「未找到有效授权」，便于排障 - .gitignore 放行 .env.production.example / .env.addon.example / .gitattributes / public/.htaccess 结构 - appmall 由 addon/appmall 迁移为独立应用 app/appmall（市场 / 开发者中心 / 审核 / 分润）
372 lines
13 KiB
PHP
372 lines
13 KiB
PHP
<?php
|
||
/*
|
||
* @Author: YwxApp <ywx@ywxapp.cn>
|
||
* @Date: 2026-07-28 00:00:00
|
||
* @Description: 搜索蜘蛛统计
|
||
* @FilePath: \ywxapp_dev\app\backend\controller\Spider.php
|
||
* @CustomString: Copyright (c) 2026 YwxApp
|
||
*/
|
||
|
||
declare(strict_types=1);
|
||
namespace app\backend\controller;
|
||
|
||
use think\facade\Db;
|
||
use ywxapp\controller\BackendBase;
|
||
use ywxapp\library\SpiderDetect;
|
||
use ywxapp\library\SpiderVerify;
|
||
use ywxapp\model\BaseModel;
|
||
|
||
/**
|
||
* 搜索蜘蛛统计
|
||
*
|
||
* 数据来源：全局中间件 ywxapp\middleware\SpiderStat 写入的
|
||
* spider_log（明细）与 spider_stat（按日聚合）两张表。
|
||
*
|
||
* 真伪识别：UA 只是「声明」，SpiderVerify 通过 rDNS 双向验证给出结论，
|
||
* verified 字段：0 待验证 / 1 真蜘蛛 / 2 假蜘蛛（伪造 UA）。
|
||
*
|
||
* @author ywxapp <admin@ywxapp.cn>
|
||
*/
|
||
class Spider extends BackendBase
|
||
{
|
||
protected $noNeedLogin = [];
|
||
// recheck 为幂等统计操作（只重新判定真伪并同步聚合，不增删业务数据），
|
||
// 免权限项校验，避免老库未注册 spider:recheck 权限时 403。
|
||
protected $noNeedVerify = ['recheck'];
|
||
|
||
/**
|
||
* 真伪文案
|
||
*/
|
||
public const VERIFY_TEXT = [
|
||
0 => '待验证',
|
||
1 => '真蜘蛛',
|
||
2 => '假蜘蛛',
|
||
];
|
||
|
||
/**
|
||
* 抓取明细列表（页面 + Ajax）
|
||
*/
|
||
public function index()
|
||
{
|
||
if ($this->request->isAjax()) {
|
||
$page = $this->request->param('page/d', 1);
|
||
$limit = $this->request->param('limit/d', 15);
|
||
$spider = $this->request->param('spider', '');
|
||
$ip = $this->request->param('ip', '');
|
||
$url = $this->request->param('url', '');
|
||
$date = $this->request->param('date', ''); // YYYY-MM-DD
|
||
$verified = $this->request->param('verified', ''); // ''全部 0待验证 1真 2假
|
||
|
||
$query = Db::name('common_spider_log');
|
||
if ($spider !== '') {
|
||
$query->where('spider', $spider);
|
||
}
|
||
if ($ip !== '') {
|
||
$query->where('ip', 'like', $ip . '%');
|
||
}
|
||
if ($url !== '') {
|
||
$query->where('url', 'like', '%' . $url . '%');
|
||
}
|
||
if ($date !== '' && ($ts = strtotime($date)) !== false) {
|
||
$query->whereBetween('create_at', [$ts, $ts + 86399]);
|
||
}
|
||
if ($verified !== '' && in_array((string) $verified, ['0', '1', '2'], true)) {
|
||
$query->where('verified', (int) $verified);
|
||
}
|
||
|
||
try {
|
||
$list = $query->order('id', 'desc')
|
||
->paginate(['page' => $page, 'list_rows' => $limit]);
|
||
$items = $list->items();
|
||
$labels = SpiderDetect::labels();
|
||
foreach ($items as &$item) {
|
||
$item['spider_text'] = $labels[$item['spider']] ?? $item['spider'];
|
||
$item['time_text'] = date('Y-m-d H:i:s', (int) $item['create_at']);
|
||
|
||
$v = (int) ($item['verified'] ?? 0);
|
||
$item['verified'] = $v;
|
||
$item['verified_text'] = self::VERIFY_TEXT[$v] ?? '待验证';
|
||
$item['rdns'] = (string) ($item['rdns'] ?? '');
|
||
// 真伪校验依赖 UA 声明的蜘蛛是否支持 rDNS
|
||
$item['verifiable'] = SpiderDetect::verifiable((string) $item['spider']);
|
||
}
|
||
unset($item);
|
||
$this->result->setCount($list->total())->success($items);
|
||
} catch (\Throwable $e) {
|
||
// 表尚未创建（还没有蜘蛛来访过）：返回空列表而非报错
|
||
$this->result->setCount(0)->success([]);
|
||
}
|
||
}
|
||
|
||
$this->assign('verifyText', self::VERIFY_TEXT);
|
||
$this->assign('dnsDown', SpiderVerify::dnsDown());
|
||
$this->assign('spiderLabels', SpiderDetect::labels());
|
||
return $this->fetch();
|
||
}
|
||
|
||
/**
|
||
* 汇总统计：各蜘蛛 今日/7日/30日/总量 + 真伪拆分 + 近30日逐日趋势（图表用）
|
||
*/
|
||
public function stats()
|
||
{
|
||
$labels = SpiderDetect::labels();
|
||
$today = date('Y-m-d');
|
||
$d7 = date('Y-m-d', strtotime('-6 days'));
|
||
$d30 = date('Y-m-d', strtotime('-29 days'));
|
||
|
||
$summary = [];
|
||
$trend = ['dates' => [], 'series' => []];
|
||
$totals = ['count' => 0, 'fake' => 0];
|
||
|
||
try {
|
||
// 老库兜底：确保聚合表有 fake_count 列
|
||
$this->ensureStatColumns();
|
||
|
||
$rows = Db::name('common_spider_stat')
|
||
->where('stat_date', '>=', $d30)
|
||
->select()
|
||
->toArray();
|
||
$totalRows = Db::name('common_spider_stat')
|
||
->field('spider, SUM(`count`) AS total, SUM(`fake_count`) AS fakes')
|
||
->group('spider')
|
||
->select()
|
||
->toArray();
|
||
|
||
// 各蜘蛛汇总
|
||
$map = [];
|
||
foreach ($totalRows as $r) {
|
||
$total = (int) $r['total'];
|
||
$fake = (int) ($r['fakes'] ?? 0);
|
||
$map[$r['spider']] = [
|
||
'spider' => $r['spider'],
|
||
'label' => $labels[$r['spider']] ?? $r['spider'],
|
||
'verifiable' => SpiderDetect::verifiable((string) $r['spider']),
|
||
'today' => 0,
|
||
'week' => 0,
|
||
'month' => 0,
|
||
'total' => $total,
|
||
'fake' => $fake,
|
||
'real' => max(0, $total - $fake),
|
||
'fake_today' => 0,
|
||
'fake_week' => 0,
|
||
'fake_month' => 0,
|
||
];
|
||
$totals['count'] += $total;
|
||
$totals['fake'] += $fake;
|
||
}
|
||
foreach ($rows as $r) {
|
||
$key = $r['spider'];
|
||
if (!isset($map[$key])) {
|
||
continue;
|
||
}
|
||
$c = (int) $r['count'];
|
||
$f = (int) ($r['fake_count'] ?? 0);
|
||
$map[$key]['month'] += $c;
|
||
$map[$key]['fake_month'] += $f;
|
||
if ($r['stat_date'] >= $d7) {
|
||
$map[$key]['week'] += $c;
|
||
$map[$key]['fake_week'] += $f;
|
||
}
|
||
if ($r['stat_date'] === $today) {
|
||
$map[$key]['today'] += $c;
|
||
$map[$key]['fake_today'] += $f;
|
||
}
|
||
}
|
||
usort($map, fn ($a, $b) => $b['total'] <=> $a['total']);
|
||
$summary = array_values($map);
|
||
|
||
// 近30日逐日趋势（每蜘蛛一条线）
|
||
$dates = [];
|
||
for ($i = 29; $i >= 0; $i--) {
|
||
$dates[] = date('Y-m-d', strtotime("-{$i} days"));
|
||
}
|
||
$trend['dates'] = $dates;
|
||
$bySpider = [];
|
||
foreach ($rows as $r) {
|
||
$bySpider[$r['spider']][$r['stat_date']] = (int) $r['count'];
|
||
}
|
||
foreach ($summary as $s) {
|
||
$key = $s['spider'];
|
||
$line = [];
|
||
foreach ($dates as $d) {
|
||
$line[] = $bySpider[$key][$d] ?? 0;
|
||
}
|
||
$trend['series'][] = ['name' => $s['label'], 'data' => $line];
|
||
}
|
||
} catch (\Throwable $e) {
|
||
// 表未创建：返回空数据
|
||
}
|
||
|
||
$this->result->success([
|
||
'summary' => $summary,
|
||
'trend' => $trend,
|
||
'totals' => [
|
||
'count' => $totals['count'],
|
||
'fake' => $totals['fake'],
|
||
'real' => max(0, $totals['count'] - $totals['fake']),
|
||
],
|
||
'dns_down' => SpiderVerify::dnsDown(),
|
||
]);
|
||
}
|
||
|
||
/**
|
||
* 重新校验历史记录的真伪
|
||
*
|
||
* 适用于：刚升级到带真伪识别的版本时，把历史「待验证」记录补校验一遍；
|
||
* 或某蜘蛛统计异常时手动重算。只处理明细中仍在保留期内的记录，
|
||
* 校验后同步修正当日聚合表。
|
||
*/
|
||
public function recheck()
|
||
{
|
||
if (!($this->request->isAjax() && $this->request->isPost())) {
|
||
$this->result->error('请求方式错误', 405);
|
||
}
|
||
|
||
$limit = (int) $this->request->param('limit/d', 200);
|
||
$limit = max(1, min(500, $limit));
|
||
|
||
try {
|
||
$this->ensureLogColumns();
|
||
|
||
// 取「待验证」的日志，按 IP 去重后逐条校验（同一 IP 只查一次 DNS）
|
||
$rows = Db::name('common_spider_log')
|
||
->field('spider, ip')
|
||
->where('verified', 0)
|
||
->order('id', 'desc')
|
||
->limit(3000)
|
||
->select()
|
||
->toArray();
|
||
} catch (\Throwable $e) {
|
||
$this->result->error('读取日志失败：' . $e->getMessage());
|
||
}
|
||
|
||
$pairs = [];
|
||
foreach ($rows as $r) {
|
||
$spider = (string) $r['spider'];
|
||
$ip = (string) $r['ip'];
|
||
if ($ip === '' || !SpiderDetect::verifiable($spider)) {
|
||
continue;
|
||
}
|
||
$pairs[$spider . '|' . $ip] = [$spider, $ip];
|
||
if (count($pairs) >= $limit) {
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (empty($pairs)) {
|
||
$this->result->success(
|
||
['real' => 0, 'fake' => 0, 'pending' => 0, 'scanned' => 0],
|
||
'没有需要重新校验的记录（或该蜘蛛不支持 rDNS 验证）'
|
||
);
|
||
}
|
||
|
||
$real = $fake = $pending = 0;
|
||
$touched = [];
|
||
|
||
foreach ($pairs as [$spider, $ip]) {
|
||
SpiderVerify::forget($ip, $spider);
|
||
$verdict = SpiderVerify::verify($ip, $spider);
|
||
|
||
if ((int) $verdict['status'] === 0) {
|
||
$pending++;
|
||
continue;
|
||
}
|
||
|
||
Db::name('common_spider_log')
|
||
->where('spider', $spider)
|
||
->where('ip', $ip)
|
||
->update([
|
||
'verified' => (int) $verdict['status'],
|
||
'rdns' => (string) $verdict['rdns'],
|
||
]);
|
||
|
||
$touched[$spider] = true;
|
||
if ((int) $verdict['status'] === 1) {
|
||
$real++;
|
||
} else {
|
||
$fake++;
|
||
}
|
||
}
|
||
|
||
// 同步修正聚合表（只重算有明细的日期，历史聚合保留）
|
||
foreach (array_keys($touched) as $spider) {
|
||
$this->resyncStat((string) $spider);
|
||
}
|
||
|
||
$msg = "校验完成：真蜘蛛 {$real} 个 IP，假蜘蛛 {$fake} 个 IP";
|
||
if ($pending > 0) {
|
||
$msg .= "，{$pending} 个 IP 因 DNS 不可用暂无法判定（稍后可重试）";
|
||
}
|
||
if (SpiderVerify::dnsDown()) {
|
||
$msg .= '。注意：DNS 查询连续失败已暂时降级，请检查服务器 DNS 配置。';
|
||
}
|
||
|
||
$this->result->success([
|
||
'real' => $real,
|
||
'fake' => $fake,
|
||
'pending' => $pending,
|
||
'scanned' => count($pairs),
|
||
], $msg);
|
||
}
|
||
|
||
/**
|
||
* 清理明细日志（保留最近 N 天，聚合表不动，历史报表不受影响）
|
||
*/
|
||
public function clear()
|
||
{
|
||
if (!($this->request->isAjax() && $this->request->isPost())) {
|
||
$this->result->error('请求方式错误', 405);
|
||
}
|
||
$days = $this->request->param('days/d', 30);
|
||
$days = max(1, min(365, $days));
|
||
try {
|
||
$count = Db::name('common_spider_log')
|
||
->where('create_at', '<', time() - $days * 86400)
|
||
->delete();
|
||
$this->result->success('', "已清理 {$count} 条 {$days} 天前的明细日志");
|
||
} catch (\Throwable $e) {
|
||
$this->result->error('清理失败: ' . $e->getMessage());
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 按 spider 重算聚合表（只覆盖明细仍在的日期，避免抹掉历史累计）
|
||
*/
|
||
protected function resyncStat(string $spider): void
|
||
{
|
||
$prefix = config('database.connections.mysql.prefix', 'wxapp_');
|
||
$log = $prefix . 'common_spider_log';
|
||
$stat = $prefix . 'common_spider_stat';
|
||
|
||
Db::execute(
|
||
"INSERT INTO `{$stat}` (`stat_date`, `spider`, `count`, `fake_count`) "
|
||
. "SELECT DATE(FROM_UNIXTIME(`create_at`)) AS d, `spider`, COUNT(*), SUM(`verified` = 2) "
|
||
. "FROM `{$log}` WHERE `spider` = ? "
|
||
. 'GROUP BY d, `spider` '
|
||
. 'ON DUPLICATE KEY UPDATE `count` = VALUES(`count`), `fake_count` = VALUES(`fake_count`)',
|
||
[$spider]
|
||
);
|
||
}
|
||
|
||
/**
|
||
* 老库兜底：明细表补 verified / rdns 列
|
||
*/
|
||
protected function ensureLogColumns(): void
|
||
{
|
||
$prefix = config('database.connections.mysql.prefix', 'wxapp_');
|
||
$log = $prefix . 'common_spider_log';
|
||
BaseModel::ensureColumn($log, 'verified', "tinyint(1) NOT NULL DEFAULT '0' COMMENT '真伪：0待验证 1真蜘蛛 2假蜘蛛'");
|
||
BaseModel::ensureColumn($log, 'rdns', "varchar(255) NOT NULL DEFAULT '' COMMENT 'rDNS反查主机名'");
|
||
}
|
||
|
||
/**
|
||
* 老库兜底：聚合表补 fake_count 列
|
||
*/
|
||
protected function ensureStatColumns(): void
|
||
{
|
||
$prefix = config('database.connections.mysql.prefix', 'wxapp_');
|
||
$stat = $prefix . 'common_spider_stat';
|
||
BaseModel::ensureColumn($stat, 'fake_count', "int unsigned NOT NULL DEFAULT '0' COMMENT '其中假蜘蛛次数'");
|
||
}
|
||
}
|