253 lines
8.4 KiB
PHP
253 lines
8.4 KiB
PHP
<?php
|
|
// +----------------------------------------------------------------------
|
|
// | likeadmin快速开发前后端分离管理后台（PHP版）
|
|
// +----------------------------------------------------------------------
|
|
// | 欢迎阅读学习系统程序代码，建议反馈是我们前进的动力
|
|
// | 开源版本可自由商用，可去除界面版权logo
|
|
// | gitee下载：https://gitee.com/likeshop_gitee/likeadmin
|
|
// | github下载：https://github.com/likeshop-github/likeadmin
|
|
// | 访问官网：https://www.likeadmin.cn
|
|
// | likeadmin团队 版权所有 拥有最终解释权
|
|
// +----------------------------------------------------------------------
|
|
// | author: likeadminTeam
|
|
// +----------------------------------------------------------------------
|
|
|
|
namespace app\common\command;
|
|
|
|
use app\common\enum\CrontabEnum;
|
|
use app\common\model\CrontabLog;
|
|
use think\console\Command;
|
|
use think\console\Input;
|
|
use think\console\Output;
|
|
use Cron\CronExpression;
|
|
use think\facade\Console;
|
|
use app\common\model\Crontab as CrontabModel;
|
|
use app\common\service\crontab\CrontabAlertService;
|
|
|
|
/**
|
|
* 定时任务
|
|
* Class Crontab
|
|
* @package app\command
|
|
*/
|
|
class Crontab extends Command
|
|
{
|
|
protected const CRAWLER_MIGRATED_MESSAGE = 'crawler任务已迁移到Docker容器调度，请查看管理端爬虫任务日志';
|
|
|
|
protected const CRAWLER_MIGRATED_COMMANDS = [
|
|
'crawler',
|
|
'truth_social',
|
|
'fifa_worldcup_news',
|
|
];
|
|
|
|
protected const CRAWLER_DEFAULT_TIMEOUT = 600;
|
|
|
|
protected const CRAWLER_ACTION_TIMEOUTS = [
|
|
'csl_match' => 180,
|
|
'nba_match' => 180,
|
|
'cba_match' => 180,
|
|
'epl_match' => 180,
|
|
'bundesliga_match' => 180,
|
|
'laliga_match' => 180,
|
|
'seriea_match' => 180,
|
|
'ligue1_match' => 180,
|
|
'ucl_match' => 180,
|
|
'uel_match' => 180,
|
|
'tennis_match' => 180,
|
|
'esports_match' => 180,
|
|
'sports_match' => 180,
|
|
'live_detail' => 180,
|
|
'match_finish' => 180,
|
|
'lottery_draw' => 180,
|
|
'lottery_draw_force' => 180,
|
|
'article_content' => 600,
|
|
'article_content_fetch' => 900,
|
|
'league_news' => 900,
|
|
'fifa_worldcup_news' => 600,
|
|
'dqd_worldcup' => 600,
|
|
];
|
|
|
|
protected function configure()
|
|
{
|
|
$this->setName('crontab')
|
|
->setDescription('定时任务');
|
|
}
|
|
|
|
protected function execute(Input $input, Output $output)
|
|
{
|
|
$lists = CrontabModel::where('status', CrontabEnum::START)->select()->toArray();
|
|
if (empty($lists)) {
|
|
return false;
|
|
}
|
|
$time = time();
|
|
foreach ($lists as $item) {
|
|
if (empty($item['last_time'])) {
|
|
$lastTime = (new CronExpression($item['expression']))
|
|
->getNextRunDate()
|
|
->getTimestamp();
|
|
CrontabModel::where('id', $item['id'])->update([
|
|
'last_time' => $lastTime,
|
|
]);
|
|
continue;
|
|
}
|
|
|
|
$nextTime = (new CronExpression($item['expression']))
|
|
->getNextRunDate($item['last_time'])
|
|
->getTimestamp();
|
|
if ($nextTime > $time) {
|
|
// 未到时间，不执行
|
|
continue;
|
|
}
|
|
// 开始执行
|
|
self::start($item);
|
|
}
|
|
}
|
|
|
|
public static function start($item)
|
|
{
|
|
if (in_array($item['command'], self::CRAWLER_MIGRATED_COMMANDS, true)) {
|
|
self::skipMigratedCrawler($item);
|
|
return;
|
|
}
|
|
|
|
$startTime = microtime(true);
|
|
$error = '';
|
|
$outputStr = '';
|
|
try {
|
|
ob_start();
|
|
$params = explode(' ', $item['params']);
|
|
if (is_array($params) && !empty($item['params'])) {
|
|
Console::call($item['command'], $params);
|
|
} else {
|
|
Console::call($item['command']);
|
|
}
|
|
$outputStr = ob_get_clean() ?: '';
|
|
CrontabModel::where('id', $item['id'])->update(['error' => '']);
|
|
} catch (\Exception $e) {
|
|
$outputStr = ob_get_clean() ?: '';
|
|
$error = $e->getMessage();
|
|
CrontabModel::where('id', $item['id'])->update([
|
|
'error' => $error,
|
|
'status' => CrontabEnum::ERROR
|
|
]);
|
|
} finally {
|
|
$endTime = microtime(true);
|
|
$useTime = round(($endTime - $startTime), 2);
|
|
$maxTime = max($useTime, $item['max_time']);
|
|
$logId = 0;
|
|
CrontabModel::where('id', $item['id'])->update([
|
|
'last_time' => time(),
|
|
'time' => $useTime,
|
|
'max_time' => $maxTime
|
|
]);
|
|
|
|
try {
|
|
$log = CrontabLog::create([
|
|
'crontab_id' => $item['id'],
|
|
'name' => $item['name'],
|
|
'command' => $item['command'],
|
|
'params' => $item['params'],
|
|
'status' => $error ? 2 : 1,
|
|
'output' => $error ?: mb_substr($outputStr ?: '执行完成', 0, 5000),
|
|
'elapsed' => $useTime,
|
|
'trigger_type' => 1,
|
|
]);
|
|
$logId = (int) $log->id;
|
|
} catch (\Exception $logEx) {
|
|
}
|
|
|
|
try {
|
|
if ($error) {
|
|
CrontabAlertService::upsertCommandException($item, $error ?: $outputStr, $logId);
|
|
} else {
|
|
CrontabAlertService::resolveByCrontabId((int) $item['id']);
|
|
}
|
|
} catch (\Exception $alertEx) {
|
|
}
|
|
}
|
|
}
|
|
|
|
protected static function skipMigratedCrawler(array $item): void
|
|
{
|
|
CrontabModel::where('id', $item['id'])->update([
|
|
'last_time' => time(),
|
|
'error' => self::CRAWLER_MIGRATED_MESSAGE,
|
|
]);
|
|
}
|
|
|
|
protected static function startCrawler($item)
|
|
{
|
|
$crawlerDir = app()->getRootPath() . 'public/dongqiudi-crawler';
|
|
$python = is_file($crawlerDir . '/venv/bin/python3') ? $crawlerDir . '/venv/bin/python3' : 'python3';
|
|
$action = $item['params'] ?: 'all';
|
|
$timeout = self::getCrawlerActionTimeout($action);
|
|
$now = time();
|
|
|
|
self::markStaleCrawlerLogs($item, $timeout, $now);
|
|
|
|
$running = CrontabLog::where('crontab_id', $item['id'])
|
|
->where('status', 0)
|
|
->where('create_time', '>', $now - $timeout)
|
|
->order('id', 'desc')
|
|
->findOrEmpty();
|
|
if (!$running->isEmpty()) {
|
|
CrontabModel::where('id', $item['id'])->update([
|
|
'last_time' => $now,
|
|
'error' => '上一次任务仍在执行，已跳过本次',
|
|
]);
|
|
return;
|
|
}
|
|
|
|
$log = CrontabLog::create([
|
|
'crontab_id' => $item['id'],
|
|
'name' => $item['name'],
|
|
'command' => $item['command'],
|
|
'params' => $item['params'],
|
|
'status' => 0,
|
|
'output' => '',
|
|
'elapsed' => 0,
|
|
'trigger_type' => 1,
|
|
]);
|
|
|
|
$logId = $log->id;
|
|
$crontabId = $item['id'];
|
|
$cmd = "cd {$crawlerDir} && nohup {$python} main.py {$action} --log-id={$logId} --crontab-id={$crontabId} > /dev/null 2>&1 &";
|
|
\shell_exec($cmd);
|
|
|
|
CrontabModel::where('id', $item['id'])->update([
|
|
'last_time' => time(),
|
|
'error' => '',
|
|
]);
|
|
}
|
|
|
|
protected static function getCrawlerActionTimeout(string $action): int
|
|
{
|
|
return self::CRAWLER_ACTION_TIMEOUTS[$action] ?? self::CRAWLER_DEFAULT_TIMEOUT;
|
|
}
|
|
|
|
protected static function markStaleCrawlerLogs(array $item, int $timeout, int $now): void
|
|
{
|
|
$staleIds = CrontabLog::where('crontab_id', $item['id'])
|
|
->where('status', 0)
|
|
->where('create_time', '<=', $now - $timeout)
|
|
->column('id');
|
|
if (empty($staleIds)) {
|
|
return;
|
|
}
|
|
|
|
$message = sprintf('执行超过 %d 秒未结束，已由调度器标记超时', $timeout);
|
|
CrontabLog::whereIn('id', $staleIds)->update([
|
|
'status' => 2,
|
|
'output' => $message,
|
|
'elapsed' => $timeout,
|
|
]);
|
|
CrontabModel::where('id', $item['id'])->update([
|
|
'error' => $message,
|
|
]);
|
|
|
|
try {
|
|
CrontabAlertService::upsertCommandException($item, $message, (int) end($staleIds));
|
|
} catch (\Exception $e) {
|
|
}
|
|
}
|
|
}
|