暂存
This commit is contained in:
@@ -0,0 +1,133 @@
|
||||
<?php
|
||||
|
||||
namespace app\controller\Api;
|
||||
|
||||
use app\service\ArticleImportService;
|
||||
use JsonException;
|
||||
use support\Request;
|
||||
use support\Response;
|
||||
use Webman\Http\UploadFile;
|
||||
use Throwable;
|
||||
|
||||
class ArticleImportController
|
||||
{
|
||||
public function import(Request $request): Response
|
||||
{
|
||||
try {
|
||||
$payload = $this->payload($request);
|
||||
} catch (JsonException $exception) {
|
||||
return $this->jsonResponse([
|
||||
'code' => 400,
|
||||
'message' => 'Invalid JSON body.',
|
||||
'errors' => ['body' => $exception->getMessage()],
|
||||
], 400);
|
||||
}
|
||||
|
||||
$service = new ArticleImportService();
|
||||
$result = $service->import($payload);
|
||||
|
||||
if (!$result['ok']) {
|
||||
return $this->jsonResponse([
|
||||
'code' => 422,
|
||||
'message' => 'Archive import validation failed.',
|
||||
'errors' => $result['errors'],
|
||||
], 422);
|
||||
}
|
||||
|
||||
try {
|
||||
$service->persistSnapshot($result['data']);
|
||||
(new \app\service\ArchiveRepository())->saveImport($result['data']);
|
||||
if (($result['data']['queue']['needs_ai_metadata'] ?? false) === true) {
|
||||
(new \app\service\AiMetadataQueue())->push($result['data']['archive']['archive_uid']);
|
||||
$result['data']['queue']['ai_metadata_enqueued'] = true;
|
||||
}
|
||||
} catch (Throwable $exception) {
|
||||
return $this->jsonResponse([
|
||||
'code' => 500,
|
||||
'message' => 'Archive import data could not be saved.',
|
||||
'errors' => ['storage' => $exception->getMessage()],
|
||||
], 500);
|
||||
}
|
||||
|
||||
return $this->jsonResponse([
|
||||
'code' => 0,
|
||||
'message' => 'Archive imported.',
|
||||
'data' => $result['data'],
|
||||
], 201);
|
||||
}
|
||||
|
||||
/**
|
||||
* @throws JsonException
|
||||
*/
|
||||
private function payload(Request $request): array
|
||||
{
|
||||
$payload = array_merge($request->get(), $request->post());
|
||||
$file = $request->file('file');
|
||||
|
||||
if ($file instanceof UploadFile) {
|
||||
if (!$file->isValid()) {
|
||||
return $payload + ['file_error' => $file->getUploadErrorCode()];
|
||||
}
|
||||
|
||||
$payload['content'] = file_get_contents($file->getPathname()) ?: '';
|
||||
$payload['source'] = $payload['source'] ?? $file->getUploadName();
|
||||
$payload['title'] = $payload['title'] ?? pathinfo($file->getUploadName() ?? '', PATHINFO_FILENAME);
|
||||
$payload['metadata'] = $this->metadata($payload['metadata'] ?? null);
|
||||
return $payload;
|
||||
}
|
||||
|
||||
$rawBody = trim($request->rawBody());
|
||||
if ($rawBody === '') {
|
||||
$payload['metadata'] = $this->metadata($payload['metadata'] ?? null);
|
||||
return $payload;
|
||||
}
|
||||
|
||||
if ($this->isJsonRequest($request)) {
|
||||
$jsonPayload = json_decode($rawBody, true, 512, JSON_THROW_ON_ERROR);
|
||||
if (!is_array($jsonPayload)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$jsonPayload['metadata'] = $this->metadata($jsonPayload['metadata'] ?? null);
|
||||
return $jsonPayload;
|
||||
}
|
||||
|
||||
$payload['content'] = $rawBody;
|
||||
$payload['source'] = $payload['source'] ?? 'raw-markdown';
|
||||
$payload['metadata'] = $this->metadata($payload['metadata'] ?? null);
|
||||
return $payload;
|
||||
}
|
||||
|
||||
private function isJsonRequest(Request $request): bool
|
||||
{
|
||||
$contentType = strtolower($request->header('content-type', ''));
|
||||
return str_contains($contentType, 'application/json') || str_contains($contentType, '+json');
|
||||
}
|
||||
|
||||
private function metadata(mixed $metadata): array
|
||||
{
|
||||
if (is_array($metadata)) {
|
||||
return $metadata;
|
||||
}
|
||||
|
||||
if (!is_string($metadata) || trim($metadata) === '') {
|
||||
return [];
|
||||
}
|
||||
|
||||
try {
|
||||
$decoded = json_decode($metadata, true, 512, JSON_THROW_ON_ERROR);
|
||||
return is_array($decoded) ? $decoded : [];
|
||||
} catch (JsonException) {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
private function jsonResponse(array $data, int $status): Response
|
||||
{
|
||||
return response(
|
||||
json_encode($data, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR),
|
||||
$status,
|
||||
['Content-Type' => 'application/json']
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
<?php
|
||||
|
||||
namespace app\controller;
|
||||
|
||||
use support\Request;
|
||||
|
||||
class IndexController
|
||||
{
|
||||
public function index(Request $request)
|
||||
{
|
||||
return <<<EOF
|
||||
<style>
|
||||
* {
|
||||
padding: 0;
|
||||
margin: 0;
|
||||
}
|
||||
iframe {
|
||||
border: none;
|
||||
overflow: scroll;
|
||||
}
|
||||
</style>
|
||||
<iframe
|
||||
src="https://www.workerman.net/wellcome"
|
||||
width="100%"
|
||||
height="100%"
|
||||
allow="clipboard-write"
|
||||
sandbox="allow-scripts allow-same-origin allow-popups allow-downloads"
|
||||
></iframe>
|
||||
EOF;
|
||||
}
|
||||
|
||||
public function view(Request $request)
|
||||
{
|
||||
return view('index/view', ['name' => 'webman']);
|
||||
}
|
||||
|
||||
public function json(Request $request)
|
||||
{
|
||||
return json(['code' => 0, 'msg' => 'ok']);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
<?php
|
||||
/**
|
||||
* Here is your custom functions.
|
||||
*/
|
||||
@@ -0,0 +1,42 @@
|
||||
<?php
|
||||
/**
|
||||
* This file is part of webman.
|
||||
*
|
||||
* Licensed under The MIT License
|
||||
* For full copyright and license information, please see the MIT-LICENSE.txt
|
||||
* Redistributions of files must retain the above copyright notice.
|
||||
*
|
||||
* @author walkor<walkor@workerman.net>
|
||||
* @copyright walkor<walkor@workerman.net>
|
||||
* @link http://www.workerman.net/
|
||||
* @license http://www.opensource.org/licenses/mit-license.php MIT License
|
||||
*/
|
||||
|
||||
namespace app\middleware;
|
||||
|
||||
use Webman\MiddlewareInterface;
|
||||
use Webman\Http\Response;
|
||||
use Webman\Http\Request;
|
||||
|
||||
/**
|
||||
* Class StaticFile
|
||||
* @package app\middleware
|
||||
*/
|
||||
class StaticFile implements MiddlewareInterface
|
||||
{
|
||||
public function process(Request $request, callable $handler): Response
|
||||
{
|
||||
// Access to files beginning with. Is prohibited
|
||||
if (strpos($request->path(), '/.') !== false) {
|
||||
return response('<h1>403 forbidden</h1>', 403);
|
||||
}
|
||||
/** @var Response $response */
|
||||
$response = $handler($request);
|
||||
// Add cross domain HTTP header
|
||||
/*$response->withHeaders([
|
||||
'Access-Control-Allow-Origin' => '*',
|
||||
'Access-Control-Allow-Credentials' => 'true',
|
||||
]);*/
|
||||
return $response;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
<?php
|
||||
|
||||
namespace app\model;
|
||||
|
||||
use support\Model;
|
||||
|
||||
class Test extends Model
|
||||
{
|
||||
/**
|
||||
* The table associated with the model.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $table = 'test';
|
||||
|
||||
/**
|
||||
* The primary key associated with the table.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $primaryKey = 'id';
|
||||
|
||||
/**
|
||||
* Indicates if the model should be timestamped.
|
||||
*
|
||||
* @var bool
|
||||
*/
|
||||
public $timestamps = false;
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
<?php
|
||||
|
||||
namespace app\process;
|
||||
|
||||
use app\service\AiMetadataQueue;
|
||||
use app\service\ArchiveMetadataEnrichmentService;
|
||||
use app\service\ArchiveRepository;
|
||||
use Throwable;
|
||||
use Workerman\Timer;
|
||||
|
||||
class AiMetadata
|
||||
{
|
||||
private AiMetadataQueue $queue;
|
||||
private ArchiveRepository $archives;
|
||||
private ArchiveMetadataEnrichmentService $enrichment;
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this->queue = new AiMetadataQueue();
|
||||
$this->archives = new ArchiveRepository();
|
||||
$this->enrichment = new ArchiveMetadataEnrichmentService();
|
||||
}
|
||||
|
||||
public function onWorkerStart(): void
|
||||
{
|
||||
Timer::add(10, fn (): int => $this->queue->releaseDueDelayed());
|
||||
|
||||
while (true) {
|
||||
$this->queue->releaseDueDelayed();
|
||||
$archiveUid = $this->queue->pop($this->queue->blockTimeout());
|
||||
if ($archiveUid === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$this->handle($archiveUid);
|
||||
}
|
||||
}
|
||||
|
||||
private function handle(string $archiveUid): void
|
||||
{
|
||||
try {
|
||||
$archive = $this->archives->findArchive($archiveUid);
|
||||
if ($archive === null) {
|
||||
$this->queue->clearRetry($archiveUid);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!$this->archives->archiveNeedsMetadata($archive)) {
|
||||
$this->queue->clearRetry($archiveUid);
|
||||
return;
|
||||
}
|
||||
|
||||
$payload = $archive;
|
||||
$payload['content'] = $this->archives->findChunksText($archiveUid);
|
||||
|
||||
$enriched = $this->enrichment->enrich($payload);
|
||||
$aiMeta = $enriched['metadata']['ai_enrichment'] ?? [];
|
||||
|
||||
if (($aiMeta['attempted'] ?? false) !== true || ($aiMeta['error'] ?? null)) {
|
||||
$this->queue->retryLater($archiveUid, $aiMeta['error'] ?? 'AI metadata enrichment did not complete.');
|
||||
return;
|
||||
}
|
||||
|
||||
$fields = [];
|
||||
foreach (['title', 'year', 'author', 'tags', 'summary'] as $field) {
|
||||
if (array_key_exists($field, $enriched)) {
|
||||
$fields[$field] = $enriched[$field];
|
||||
}
|
||||
}
|
||||
|
||||
$this->archives->updateMetadata($archiveUid, $fields, $aiMeta);
|
||||
$this->queue->clearRetry($archiveUid);
|
||||
} catch (Throwable $exception) {
|
||||
$this->queue->retryLater($archiveUid, $exception->getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
<?php
|
||||
|
||||
namespace app\process;
|
||||
|
||||
use Webman\App;
|
||||
|
||||
class Http extends App
|
||||
{
|
||||
|
||||
}
|
||||
@@ -0,0 +1,305 @@
|
||||
<?php
|
||||
/**
|
||||
* This file is part of webman.
|
||||
*
|
||||
* Licensed under The MIT License
|
||||
* For full copyright and license information, please see the MIT-LICENSE.txt
|
||||
* Redistributions of files must retain the above copyright notice.
|
||||
*
|
||||
* @author walkor<walkor@workerman.net>
|
||||
* @copyright walkor<walkor@workerman.net>
|
||||
* @link http://www.workerman.net/
|
||||
* @license http://www.opensource.org/licenses/mit-license.php MIT License
|
||||
*/
|
||||
|
||||
namespace app\process;
|
||||
|
||||
use FilesystemIterator;
|
||||
use RecursiveDirectoryIterator;
|
||||
use RecursiveIteratorIterator;
|
||||
use SplFileInfo;
|
||||
use Workerman\Timer;
|
||||
use Workerman\Worker;
|
||||
|
||||
/**
|
||||
* Class FileMonitor
|
||||
* @package process
|
||||
*/
|
||||
class Monitor
|
||||
{
|
||||
/**
|
||||
* @var array
|
||||
*/
|
||||
protected array $paths = [];
|
||||
|
||||
/**
|
||||
* @var array
|
||||
*/
|
||||
protected array $extensions = [];
|
||||
|
||||
/**
|
||||
* @var array
|
||||
*/
|
||||
protected array $loadedFiles = [];
|
||||
|
||||
/**
|
||||
* @var int
|
||||
*/
|
||||
protected int $ppid = 0;
|
||||
|
||||
/**
|
||||
* Pause monitor
|
||||
* @return void
|
||||
*/
|
||||
public static function pause(): void
|
||||
{
|
||||
file_put_contents(static::lockFile(), time());
|
||||
}
|
||||
|
||||
/**
|
||||
* Resume monitor
|
||||
* @return void
|
||||
*/
|
||||
public static function resume(): void
|
||||
{
|
||||
clearstatcache();
|
||||
if (is_file(static::lockFile())) {
|
||||
unlink(static::lockFile());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether monitor is paused
|
||||
* @return bool
|
||||
*/
|
||||
public static function isPaused(): bool
|
||||
{
|
||||
clearstatcache();
|
||||
return file_exists(static::lockFile());
|
||||
}
|
||||
|
||||
/**
|
||||
* Lock file
|
||||
* @return string
|
||||
*/
|
||||
protected static function lockFile(): string
|
||||
{
|
||||
return runtime_path('monitor.lock');
|
||||
}
|
||||
|
||||
/**
|
||||
* FileMonitor constructor.
|
||||
* @param $monitorDir
|
||||
* @param $monitorExtensions
|
||||
* @param array $options
|
||||
*/
|
||||
public function __construct($monitorDir, $monitorExtensions, array $options = [])
|
||||
{
|
||||
$this->ppid = function_exists('posix_getppid') ? posix_getppid() : 0;
|
||||
static::resume();
|
||||
$this->paths = (array)$monitorDir;
|
||||
$this->extensions = $monitorExtensions;
|
||||
foreach (get_included_files() as $index => $file) {
|
||||
$this->loadedFiles[$file] = $index;
|
||||
if (strpos($file, 'webman-framework/src/support/App.php')) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!Worker::getAllWorkers()) {
|
||||
return;
|
||||
}
|
||||
$disableFunctions = explode(',', ini_get('disable_functions'));
|
||||
if (in_array('exec', $disableFunctions, true)) {
|
||||
echo "\nMonitor file change turned off because exec() has been disabled by disable_functions setting in " . PHP_CONFIG_FILE_PATH . "/php.ini\n";
|
||||
} else {
|
||||
if ($options['enable_file_monitor'] ?? true) {
|
||||
Timer::add(1, function () {
|
||||
$this->checkAllFilesChange();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
$memoryLimit = $this->getMemoryLimit($options['memory_limit'] ?? null);
|
||||
if ($memoryLimit && ($options['enable_memory_monitor'] ?? true)) {
|
||||
Timer::add(60, [$this, 'checkMemory'], [$memoryLimit]);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param $monitorDir
|
||||
* @return bool
|
||||
*/
|
||||
public function checkFilesChange($monitorDir): bool
|
||||
{
|
||||
static $lastMtime, $tooManyFilesCheck;
|
||||
if (!$lastMtime) {
|
||||
$lastMtime = time();
|
||||
}
|
||||
clearstatcache();
|
||||
if (!is_dir($monitorDir)) {
|
||||
if (!is_file($monitorDir)) {
|
||||
return false;
|
||||
}
|
||||
$iterator = [new SplFileInfo($monitorDir)];
|
||||
} else {
|
||||
// recursive traversal directory
|
||||
$dirIterator = new RecursiveDirectoryIterator($monitorDir, FilesystemIterator::SKIP_DOTS | FilesystemIterator::FOLLOW_SYMLINKS);
|
||||
$iterator = new RecursiveIteratorIterator($dirIterator);
|
||||
}
|
||||
$count = 0;
|
||||
foreach ($iterator as $file) {
|
||||
$count ++;
|
||||
/** @var SplFileInfo $file */
|
||||
if (is_dir($file->getRealPath())) {
|
||||
continue;
|
||||
}
|
||||
// check mtime
|
||||
if (in_array($file->getExtension(), $this->extensions, true) && $lastMtime < $file->getMTime()) {
|
||||
$lastMtime = $file->getMTime();
|
||||
if (DIRECTORY_SEPARATOR === '/' && isset($this->loadedFiles[$file->getRealPath()])) {
|
||||
echo "$file updated but cannot be reloaded because only auto-loaded files support reload.\n";
|
||||
continue;
|
||||
}
|
||||
$var = 0;
|
||||
exec('"'.PHP_BINARY . '" -l ' . $file, $out, $var);
|
||||
if ($var) {
|
||||
continue;
|
||||
}
|
||||
// send SIGUSR1 signal to master process for reload
|
||||
if (DIRECTORY_SEPARATOR === '/') {
|
||||
if ($masterPid = $this->getMasterPid()) {
|
||||
echo $file . " updated and reload\n";
|
||||
posix_kill($masterPid, SIGUSR1);
|
||||
} else {
|
||||
echo "Master process has gone away and can not reload\n";
|
||||
}
|
||||
return true;
|
||||
}
|
||||
echo $file . " updated and reload\n";
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if (!$tooManyFilesCheck && $count > 1000) {
|
||||
echo "Monitor: There are too many files ($count files) in $monitorDir which makes file monitoring very slow\n";
|
||||
$tooManyFilesCheck = 1;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return int
|
||||
*/
|
||||
public function getMasterPid(): int
|
||||
{
|
||||
if ($this->ppid === 0) {
|
||||
return 0;
|
||||
}
|
||||
if (function_exists('posix_kill') && !posix_kill($this->ppid, 0)) {
|
||||
echo "Master process has gone away\n";
|
||||
return $this->ppid = 0;
|
||||
}
|
||||
if (PHP_OS_FAMILY !== 'Linux') {
|
||||
return $this->ppid;
|
||||
}
|
||||
$cmdline = "/proc/$this->ppid/cmdline";
|
||||
if (!is_readable($cmdline) || !($content = file_get_contents($cmdline)) || (!str_contains($content, 'WorkerMan') && !str_contains($content, 'php'))) {
|
||||
// Process not exist
|
||||
$this->ppid = 0;
|
||||
}
|
||||
return $this->ppid;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return bool
|
||||
*/
|
||||
public function checkAllFilesChange(): bool
|
||||
{
|
||||
if (static::isPaused()) {
|
||||
return false;
|
||||
}
|
||||
foreach ($this->paths as $path) {
|
||||
if ($this->checkFilesChange($path)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param $memoryLimit
|
||||
* @return void
|
||||
*/
|
||||
public function checkMemory($memoryLimit): void
|
||||
{
|
||||
if (static::isPaused() || $memoryLimit <= 0) {
|
||||
return;
|
||||
}
|
||||
$masterPid = $this->getMasterPid();
|
||||
if ($masterPid <= 0) {
|
||||
echo "Master process has gone away\n";
|
||||
return;
|
||||
}
|
||||
|
||||
$childrenFile = "/proc/$masterPid/task/$masterPid/children";
|
||||
if (!is_file($childrenFile) || !($children = file_get_contents($childrenFile))) {
|
||||
return;
|
||||
}
|
||||
foreach (explode(' ', $children) as $pid) {
|
||||
$pid = (int)$pid;
|
||||
$statusFile = "/proc/$pid/status";
|
||||
if (!is_file($statusFile) || !($status = file_get_contents($statusFile))) {
|
||||
continue;
|
||||
}
|
||||
$mem = 0;
|
||||
if (preg_match('/VmRSS\s*?:\s*?(\d+?)\s*?kB/', $status, $match)) {
|
||||
$mem = $match[1];
|
||||
}
|
||||
$mem = (int)($mem / 1024);
|
||||
if ($mem >= $memoryLimit) {
|
||||
posix_kill($pid, SIGINT);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get memory limit
|
||||
* @param $memoryLimit
|
||||
* @return int
|
||||
*/
|
||||
protected function getMemoryLimit($memoryLimit): int
|
||||
{
|
||||
if ($memoryLimit === 0) {
|
||||
return 0;
|
||||
}
|
||||
$usePhpIni = false;
|
||||
if (!$memoryLimit) {
|
||||
$memoryLimit = ini_get('memory_limit');
|
||||
$usePhpIni = true;
|
||||
}
|
||||
|
||||
if ($memoryLimit == -1) {
|
||||
return 0;
|
||||
}
|
||||
$unit = strtolower($memoryLimit[strlen($memoryLimit) - 1]);
|
||||
$memoryLimit = (int)$memoryLimit;
|
||||
if ($unit === 'g') {
|
||||
$memoryLimit = 1024 * $memoryLimit;
|
||||
} else if ($unit === 'k') {
|
||||
$memoryLimit = ($memoryLimit / 1024);
|
||||
} else if ($unit === 'm') {
|
||||
$memoryLimit = (int)($memoryLimit);
|
||||
} else if ($unit === 't') {
|
||||
$memoryLimit = (1024 * 1024 * $memoryLimit);
|
||||
} else {
|
||||
$memoryLimit = ($memoryLimit / (1024 * 1024));
|
||||
}
|
||||
if ($memoryLimit < 50) {
|
||||
$memoryLimit = 50;
|
||||
}
|
||||
if ($usePhpIni) {
|
||||
$memoryLimit = (0.8 * $memoryLimit);
|
||||
}
|
||||
return (int)$memoryLimit;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
<?php
|
||||
|
||||
namespace app\service;
|
||||
|
||||
use support\Redis;
|
||||
|
||||
class AiMetadataQueue
|
||||
{
|
||||
public function push(string $archiveUid): void
|
||||
{
|
||||
Redis::lPush($this->pendingKey(), $archiveUid);
|
||||
}
|
||||
|
||||
public function pop(int $timeout = 5): ?string
|
||||
{
|
||||
$result = Redis::brPop([$this->pendingKey()], $timeout);
|
||||
if (!is_array($result) || count($result) < 2) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (string) $result[1];
|
||||
}
|
||||
|
||||
public function retryLater(string $archiveUid, string $error): void
|
||||
{
|
||||
$retryKey = $this->retryKey($archiveUid);
|
||||
$retryCount = (int) Redis::incr($retryKey);
|
||||
Redis::setEx($this->errorKey($archiveUid), 86400, $error);
|
||||
|
||||
if ($retryCount > $this->maxRetries()) {
|
||||
Redis::lPush($this->failedKey(), $archiveUid);
|
||||
return;
|
||||
}
|
||||
|
||||
$delay = $this->baseDelaySeconds() * (2 ** max(0, $retryCount - 1));
|
||||
Redis::zAdd($this->delayedKey(), time() + $delay, $archiveUid);
|
||||
}
|
||||
|
||||
public function releaseDueDelayed(): int
|
||||
{
|
||||
$now = time();
|
||||
$items = Redis::zRangeByScore($this->delayedKey(), '-inf', (string) $now, ['limit' => [0, 100]]);
|
||||
if (!is_array($items) || $items === []) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
foreach ($items as $archiveUid) {
|
||||
Redis::zRem($this->delayedKey(), $archiveUid);
|
||||
Redis::lPush($this->pendingKey(), $archiveUid);
|
||||
}
|
||||
|
||||
return count($items);
|
||||
}
|
||||
|
||||
public function clearRetry(string $archiveUid): void
|
||||
{
|
||||
Redis::del($this->retryKey($archiveUid), $this->errorKey($archiveUid));
|
||||
}
|
||||
|
||||
public function blockTimeout(): int
|
||||
{
|
||||
return (int) config('queue.ai_metadata.block_timeout', 5);
|
||||
}
|
||||
|
||||
private function pendingKey(): string
|
||||
{
|
||||
return config('queue.ai_metadata.pending', 'proofdb:ai:metadata:pending');
|
||||
}
|
||||
|
||||
private function delayedKey(): string
|
||||
{
|
||||
return config('queue.ai_metadata.delayed', 'proofdb:ai:metadata:delayed');
|
||||
}
|
||||
|
||||
private function failedKey(): string
|
||||
{
|
||||
return config('queue.ai_metadata.failed', 'proofdb:ai:metadata:failed');
|
||||
}
|
||||
|
||||
private function retryKey(string $archiveUid): string
|
||||
{
|
||||
return config('queue.ai_metadata.retry_prefix', 'proofdb:ai:metadata:retry:') . $archiveUid;
|
||||
}
|
||||
|
||||
private function errorKey(string $archiveUid): string
|
||||
{
|
||||
return config('queue.ai_metadata.error_prefix', 'proofdb:ai:metadata:error:') . $archiveUid;
|
||||
}
|
||||
|
||||
private function maxRetries(): int
|
||||
{
|
||||
return (int) config('queue.ai_metadata.max_retries', 5);
|
||||
}
|
||||
|
||||
private function baseDelaySeconds(): int
|
||||
{
|
||||
return (int) config('queue.ai_metadata.base_delay_seconds', 60);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
<?php
|
||||
|
||||
namespace app\service;
|
||||
|
||||
use app\service\LLM\OpenAICompatibleClient;
|
||||
use app\service\LLM\LLMRetryQueue;
|
||||
use Throwable;
|
||||
|
||||
class ArchiveMetadataEnrichmentService
|
||||
{
|
||||
private OpenAICompatibleClient $client;
|
||||
private LLMRetryQueue $queue;
|
||||
|
||||
public function __construct(?OpenAICompatibleClient $client = null, ?LLMRetryQueue $queue = null)
|
||||
{
|
||||
$this->client = $client ?? new OpenAICompatibleClient();
|
||||
$this->queue = $queue ?? new LLMRetryQueue();
|
||||
}
|
||||
|
||||
public function enrich(array $payload): array
|
||||
{
|
||||
$missing = $this->missingFields($payload);
|
||||
if ($missing === [] || !$this->enabled()) {
|
||||
return $this->withAiMeta($payload, [
|
||||
'enabled' => $this->enabled(),
|
||||
'attempted' => false,
|
||||
'filled' => [],
|
||||
'missing' => $missing,
|
||||
]);
|
||||
}
|
||||
|
||||
try {
|
||||
$result = $this->queue->run(
|
||||
fn (): array => $this->client->chatJson($this->messages($payload, $missing), [
|
||||
'model' => config('LLMapi.metadata.model'),
|
||||
'temperature' => config('LLMapi.metadata.temperature', 0.1),
|
||||
'max_tokens' => config('LLMapi.metadata.max_tokens', 1200),
|
||||
'stream' => false,
|
||||
'response_format' => config('LLMapi.metadata.response_format', ['type' => 'json_object']),
|
||||
'thinking' => config('LLMapi.metadata.thinking', ['type' => 'disabled']),
|
||||
'request_id' => $this->requestId($payload, $missing),
|
||||
]),
|
||||
config('LLMapi.metadata.retry', [])
|
||||
);
|
||||
} catch (Throwable $exception) {
|
||||
return $this->withAiMeta($payload, [
|
||||
'enabled' => true,
|
||||
'attempted' => true,
|
||||
'filled' => [],
|
||||
'missing' => $missing,
|
||||
'error' => $exception->getMessage(),
|
||||
]);
|
||||
}
|
||||
|
||||
$filled = [];
|
||||
foreach ($missing as $field) {
|
||||
if (!$this->hasUsefulValue($result, $field)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$payload[$field] = $this->normalizeField($field, $result[$field]);
|
||||
$filled[] = $field;
|
||||
}
|
||||
|
||||
return $this->withAiMeta($payload, [
|
||||
'enabled' => true,
|
||||
'attempted' => true,
|
||||
'filled' => $filled,
|
||||
'missing' => array_values(array_diff($missing, $filled)),
|
||||
'model' => config('LLMapi.metadata.model'),
|
||||
'stream' => false,
|
||||
'response_format' => config('LLMapi.metadata.response_format', ['type' => 'json_object']),
|
||||
'thinking' => config('LLMapi.metadata.thinking', ['type' => 'disabled']),
|
||||
]);
|
||||
}
|
||||
|
||||
private function missingFields(array $payload): array
|
||||
{
|
||||
$fields = ['title', 'year', 'author', 'tags', 'summary'];
|
||||
return array_values(array_filter($fields, fn (string $field): bool => !$this->hasUsefulValue($payload, $field)));
|
||||
}
|
||||
|
||||
private function enabled(): bool
|
||||
{
|
||||
return (bool) config('LLMapi.metadata.enabled', true) && $this->client->isConfigured();
|
||||
}
|
||||
|
||||
private function messages(array $payload, array $missing): array
|
||||
{
|
||||
$text = $this->sampleText($payload);
|
||||
|
||||
return [
|
||||
[
|
||||
'role' => 'system',
|
||||
'content' => implode("\n", [
|
||||
'你是历史档案元数据整理助手。',
|
||||
'你只能根据用户提供的档案文本抽取或推断元数据。',
|
||||
'请只返回 JSON 对象，不要返回 Markdown，不要解释。',
|
||||
'字段：title(string), year(integer|null), author(string|null), tags(array<string>), summary(string)。',
|
||||
'summary 简洁概括档案内容，80-200 字。',
|
||||
'tags 用档案中常见专名和涉及主题，5-10 个。',
|
||||
'无法判断的字段返回 null 或空数组。',
|
||||
'以上请均使用档案中的语言。',
|
||||
]),
|
||||
],
|
||||
[
|
||||
'role' => 'user',
|
||||
'content' => json_encode([
|
||||
'missing_fields' => $missing,
|
||||
'known_fields' => [
|
||||
'title' => $payload['title'] ?? null,
|
||||
'year' => $payload['year'] ?? null,
|
||||
'author' => $payload['author'] ?? null,
|
||||
'source' => $payload['source'] ?? null,
|
||||
'series' => $payload['series'] ?? null,
|
||||
],
|
||||
'archive_text_sample' => $text,
|
||||
], JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
|
||||
],
|
||||
];
|
||||
}
|
||||
|
||||
private function requestId(array $payload, array $missing): string
|
||||
{
|
||||
return 'metadata-' . substr(hash('sha256', implode('|', [
|
||||
(string) ($payload['source'] ?? ''),
|
||||
(string) ($payload['archive_uid'] ?? ''),
|
||||
mb_substr((string) ($payload['content'] ?? ''), 0, 1000),
|
||||
implode(',', $missing ?? []),
|
||||
])), 0, 32);
|
||||
}
|
||||
|
||||
private function sampleText(array $payload): string
|
||||
{
|
||||
$text = '';
|
||||
if (isset($payload['content']) && is_string($payload['content'])) {
|
||||
$text = $payload['content'];
|
||||
} elseif (isset($payload['pages']) && is_array($payload['pages'])) {
|
||||
$parts = [];
|
||||
foreach ($payload['pages'] as $page) {
|
||||
if (isset($page['content']) && is_string($page['content'])) {
|
||||
$parts[] = $page['content'];
|
||||
}
|
||||
}
|
||||
$text = implode("\n\n", $parts);
|
||||
}
|
||||
|
||||
$maxChars = (int) config('LLMapi.metadata.max_input_chars', 12000);
|
||||
return mb_substr($text, 0, $maxChars);
|
||||
}
|
||||
|
||||
private function hasUsefulValue(array $payload, string $field): bool
|
||||
{
|
||||
if (!array_key_exists($field, $payload)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if ($field === 'title' && (($payload['metadata']['title_source'] ?? null) === 'fallback')) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$value = $payload[$field];
|
||||
if (is_array($value)) {
|
||||
return $value !== [];
|
||||
}
|
||||
|
||||
if ($field === 'year') {
|
||||
return is_numeric($value) && (int) $value > 0;
|
||||
}
|
||||
|
||||
return is_string($value) ? trim($value) !== '' : $value !== null;
|
||||
}
|
||||
|
||||
private function normalizeField(string $field, mixed $value): mixed
|
||||
{
|
||||
if ($field === 'year') {
|
||||
return is_numeric($value) ? (int) $value : null;
|
||||
}
|
||||
|
||||
if ($field === 'tags') {
|
||||
if (!is_array($value)) {
|
||||
return [];
|
||||
}
|
||||
return array_values(array_filter(array_map('strval', $value)));
|
||||
}
|
||||
|
||||
return is_string($value) ? trim($value) : $value;
|
||||
}
|
||||
|
||||
private function withAiMeta(array $payload, array $ai): array
|
||||
{
|
||||
$payload['metadata'] = is_array($payload['metadata'] ?? null) ? $payload['metadata'] : [];
|
||||
$payload['metadata']['ai_enrichment'] = $ai;
|
||||
|
||||
return $payload;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
<?php
|
||||
|
||||
namespace app\service;
|
||||
|
||||
use support\Db;
|
||||
|
||||
class ArchiveRepository
|
||||
{
|
||||
public function saveImport(array $import): void
|
||||
{
|
||||
Db::transaction(function () use ($import): void {
|
||||
$archive = $import['archive'];
|
||||
$chunks = $import['chunks'];
|
||||
$chunkUids = array_column($chunks, 'chunk_uid');
|
||||
|
||||
Db::table('archives')->updateOrInsert(
|
||||
['archive_uid' => $archive['archive_uid']],
|
||||
[
|
||||
'title' => $archive['title'] ?? null,
|
||||
'summary' => $archive['summary'] ?? null,
|
||||
'year' => $archive['year'] ?? null,
|
||||
'author' => $archive['author'] ?? null,
|
||||
'source' => $archive['source'] ?? null,
|
||||
'series' => $archive['series'] ?? null,
|
||||
'tags' => json_encode($archive['tags'] ?? [], JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
|
||||
'metadata' => json_encode($archive['metadata'] ?? [], JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
|
||||
'content' => $archive['content'] ?? null,
|
||||
'raw' => $archive['raw'] ?? null,
|
||||
'chunks' => json_encode($chunkUids, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
|
||||
]
|
||||
);
|
||||
|
||||
Db::table('chunks')->where('archive_uid', $archive['archive_uid'])->delete();
|
||||
foreach ($chunks as $chunk) {
|
||||
Db::table('chunks')->insert([
|
||||
'chunk_uid' => $chunk['chunk_uid'],
|
||||
'archive_uid' => $archive['archive_uid'],
|
||||
'chunk_index' => $chunk['chunk_index'],
|
||||
'page_start' => $chunk['page_start'],
|
||||
'page_end' => $chunk['page_end'],
|
||||
'text' => $chunk['text'],
|
||||
'length' => $chunk['length'],
|
||||
'embedding_status' => 0,
|
||||
'embedding_ref' => null,
|
||||
'embedding_model' => null,
|
||||
]);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public function findArchive(string $archiveUid): ?array
|
||||
{
|
||||
$archive = Db::table('archives')->where('archive_uid', $archiveUid)->first();
|
||||
if (!$archive) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return $this->archiveToArray($archive);
|
||||
}
|
||||
|
||||
public function findChunksText(string $archiveUid, int $limit = 20): string
|
||||
{
|
||||
$chunks = Db::table('chunks')
|
||||
->where('archive_uid', $archiveUid)
|
||||
->orderBy('chunk_index')
|
||||
->limit($limit)
|
||||
->get(['text'])
|
||||
->all();
|
||||
|
||||
return implode("\n\n", array_map(fn ($chunk): string => (string) $chunk->text, $chunks));
|
||||
}
|
||||
|
||||
public function updateMetadata(string $archiveUid, array $fields, array $aiMeta): void
|
||||
{
|
||||
$archive = $this->findArchive($archiveUid);
|
||||
$metadata = $archive['metadata'] ?? [];
|
||||
$metadata['ai_enrichment'] = $aiMeta;
|
||||
|
||||
$updates = [
|
||||
'metadata' => json_encode($metadata, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
|
||||
];
|
||||
|
||||
foreach (['title', 'summary', 'year', 'author', 'series', 'tags'] as $field) {
|
||||
if (!array_key_exists($field, $fields)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$updates[$field] = $field === 'tags'
|
||||
? json_encode($fields[$field] ?? [], JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES)
|
||||
: $fields[$field];
|
||||
}
|
||||
|
||||
Db::table('archives')->where('archive_uid', $archiveUid)->update($updates);
|
||||
}
|
||||
|
||||
public function archiveNeedsMetadata(array $archive): bool
|
||||
{
|
||||
foreach (['title', 'year', 'author', 'tags', 'summary'] as $field) {
|
||||
$value = $archive[$field] ?? null;
|
||||
if ($field === 'title' && (($archive['metadata']['title_source'] ?? null) === 'fallback')) {
|
||||
return true;
|
||||
}
|
||||
if (is_array($value) && $value === []) {
|
||||
return true;
|
||||
}
|
||||
if ($field === 'year' && (!$value || (int) $value <= 0)) {
|
||||
return true;
|
||||
}
|
||||
if (!is_array($value) && ($value === null || trim((string) $value) === '')) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private function archiveToArray(object $archive): array
|
||||
{
|
||||
return [
|
||||
'archive_uid' => $archive->archive_uid,
|
||||
'title' => $archive->title,
|
||||
'summary' => $archive->summary,
|
||||
'year' => $archive->year,
|
||||
'author' => $archive->author,
|
||||
'source' => $archive->source,
|
||||
'series' => $archive->series,
|
||||
'tags' => json_decode($archive->tags ?? '[]', true) ?: [],
|
||||
'metadata' => json_decode($archive->metadata ?? '{}', true) ?: [],
|
||||
'content' => $archive->content,
|
||||
'raw' => $archive->raw,
|
||||
'chunks' => json_decode($archive->chunks ?? '[]', true) ?: [],
|
||||
];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,775 @@
|
||||
<?php
|
||||
|
||||
namespace app\service;
|
||||
|
||||
use RuntimeException;
|
||||
use Symfony\Component\Uid\Ulid;
|
||||
|
||||
class ArticleImportService
|
||||
{
|
||||
private const DEFAULT_CHUNK_SIZE = 800;
|
||||
private const DEFAULT_CHUNK_OVERLAP = 120;
|
||||
private const MIN_CHUNK_SIZE = 100;
|
||||
private const MAX_CHUNK_SIZE = 4000;
|
||||
private const SENTENCE_BOUNDARY_PATTERN = '/(?<=[。！？；.!?;])\s*|\R+/u';
|
||||
private const MARKDOWN_PAGE_PATTERN = '/<!--\s*DOCMASTER:PAGE\s+0*([0-9A-Za-z_-]+)\s*-->\s*(?:#+\s*Page\s+\S+\s*)?(.*?)(?=\n---\s*\n\s*<!--\s*DOCMASTER:PAGE|\z)/su';
|
||||
|
||||
public function import(array $payload): array
|
||||
{
|
||||
$payload = $this->normalizePayload($payload);
|
||||
$payload = $this->applyMetadataFallbacks($payload);
|
||||
$errors = $this->validate($payload);
|
||||
if ($errors !== []) {
|
||||
return ['ok' => false, 'errors' => $errors];
|
||||
}
|
||||
|
||||
$archiveUid = $this->archiveUid($payload);
|
||||
$archive = $this->archive($payload, $archiveUid);
|
||||
$pageBlocks = $this->pageBlocks($payload);
|
||||
$chunkSize = $this->intOption($payload, 'chunk_size', self::DEFAULT_CHUNK_SIZE);
|
||||
$chunkOverlap = $this->intOption($payload, 'chunk_overlap', self::DEFAULT_CHUNK_OVERLAP);
|
||||
|
||||
$chunks = $this->chunksFromPages($archiveUid, $pageBlocks, $chunkSize, $chunkOverlap);
|
||||
$pages = $this->pagesSummary($pageBlocks, $chunks);
|
||||
$needsAiMetadata = (new ArchiveRepository())->archiveNeedsMetadata($archive);
|
||||
|
||||
return [
|
||||
'ok' => true,
|
||||
'data' => [
|
||||
'import_uid' => $archiveUid,
|
||||
'archive' => $archive,
|
||||
'chunks' => $chunks,
|
||||
'pages' => $pages,
|
||||
'stats' => [
|
||||
'page_count' => count($pages),
|
||||
'page_block_count' => count($pageBlocks),
|
||||
'chunk_count' => count($chunks),
|
||||
'chunk_size' => $chunkSize,
|
||||
'chunk_overlap' => $chunkOverlap,
|
||||
],
|
||||
'queue' => [
|
||||
'ai_metadata_enqueued' => false,
|
||||
'needs_ai_metadata' => $needsAiMetadata,
|
||||
],
|
||||
],
|
||||
];
|
||||
}
|
||||
|
||||
public function persistSnapshot(array $import): void
|
||||
{
|
||||
$directory = runtime_path('proofdb/imports');
|
||||
if (!is_dir($directory) && !mkdir($directory, 0775, true) && !is_dir($directory)) {
|
||||
throw new RuntimeException("Unable to create directory: {$directory}");
|
||||
}
|
||||
|
||||
$path = $directory . DIRECTORY_SEPARATOR . $import['import_uid'] . '.json';
|
||||
$json = json_encode($import, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR);
|
||||
|
||||
if (file_put_contents($path, $json) === false) {
|
||||
throw new RuntimeException("Unable to write import snapshot: {$path}");
|
||||
}
|
||||
}
|
||||
|
||||
private function validate(array $payload): array
|
||||
{
|
||||
$errors = [];
|
||||
|
||||
if (isset($payload['file_error'])) {
|
||||
$errors['file'][] = 'uploaded file is invalid.';
|
||||
}
|
||||
|
||||
if (!isset($payload['title']) || !is_string($payload['title']) || trim($payload['title']) === '') {
|
||||
$errors['title'][] = 'title is required.';
|
||||
}
|
||||
|
||||
if (!isset($payload['source']) || !is_string($payload['source']) || trim($payload['source']) === '') {
|
||||
$errors['source'][] = 'source is required.';
|
||||
}
|
||||
|
||||
if (!isset($payload['content']) && !isset($payload['paragraphs']) && !isset($payload['pages'])) {
|
||||
$errors['content'][] = 'content, file, pages, or paragraphs is required.';
|
||||
}
|
||||
|
||||
if (isset($payload['paragraphs']) && !is_array($payload['paragraphs'])) {
|
||||
$errors['paragraphs'][] = 'paragraphs must be an array.';
|
||||
}
|
||||
|
||||
if (isset($payload['pages']) && !is_array($payload['pages'])) {
|
||||
$errors['pages'][] = 'pages must be an array.';
|
||||
}
|
||||
|
||||
if (isset($payload['metadata']) && !is_array($payload['metadata'])) {
|
||||
$errors['metadata'][] = 'metadata must be an object.';
|
||||
}
|
||||
|
||||
$chunkSize = $this->intOption($payload, 'chunk_size', self::DEFAULT_CHUNK_SIZE);
|
||||
if ($chunkSize < self::MIN_CHUNK_SIZE || $chunkSize > self::MAX_CHUNK_SIZE) {
|
||||
$errors['chunk_size'][] = 'chunk_size must be between 100 and 4000.';
|
||||
}
|
||||
|
||||
$chunkOverlap = $this->intOption($payload, 'chunk_overlap', self::DEFAULT_CHUNK_OVERLAP);
|
||||
if ($chunkOverlap < 0 || $chunkOverlap >= $chunkSize) {
|
||||
$errors['chunk_overlap'][] = 'chunk_overlap must be greater than or equal to 0 and less than chunk_size.';
|
||||
}
|
||||
|
||||
if (!isset($errors['paragraphs']) && isset($payload['paragraphs'])) {
|
||||
$hasContent = false;
|
||||
foreach ($payload['paragraphs'] as $index => $paragraph) {
|
||||
$content = is_array($paragraph) ? ($paragraph['content'] ?? '') : $paragraph;
|
||||
if (!is_string($content)) {
|
||||
$errors["paragraphs.{$index}.content"][] = 'paragraph content must be a string.';
|
||||
continue;
|
||||
}
|
||||
if (trim($content) !== '') {
|
||||
$hasContent = true;
|
||||
}
|
||||
}
|
||||
if (!$hasContent) {
|
||||
$errors['paragraphs'][] = 'paragraphs must contain at least one non-empty paragraph.';
|
||||
}
|
||||
}
|
||||
|
||||
if (!isset($errors['pages']) && isset($payload['pages'])) {
|
||||
$hasContent = false;
|
||||
foreach ($payload['pages'] as $index => $page) {
|
||||
if (!is_array($page)) {
|
||||
$errors["pages.{$index}"][] = 'page must be an object.';
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!$this->hasPageNumber($page)) {
|
||||
$errors["pages.{$index}.page_number"][] = 'page_number is required.';
|
||||
}
|
||||
|
||||
if (!isset($page['content']) || !is_string($page['content'])) {
|
||||
$errors["pages.{$index}.content"][] = 'page content must be a string.';
|
||||
continue;
|
||||
}
|
||||
|
||||
if (trim($page['content']) !== '') {
|
||||
$hasContent = true;
|
||||
}
|
||||
|
||||
if (isset($page['metadata']) && !is_array($page['metadata'])) {
|
||||
$errors["pages.{$index}.metadata"][] = 'page metadata must be an object.';
|
||||
}
|
||||
}
|
||||
|
||||
if (!$hasContent) {
|
||||
$errors['pages'][] = 'pages must contain at least one non-empty page.';
|
||||
}
|
||||
}
|
||||
|
||||
if (isset($payload['content']) && (!is_string($payload['content']) || trim($payload['content']) === '')) {
|
||||
$errors['content'][] = 'content must be a non-empty string.';
|
||||
}
|
||||
|
||||
return $errors;
|
||||
}
|
||||
|
||||
private function archive(array $payload, string $archiveUid): array
|
||||
{
|
||||
$title = $this->clean($payload['title']);
|
||||
$source = $this->clean($payload['source']);
|
||||
|
||||
return [
|
||||
'archive_uid' => $archiveUid,
|
||||
'title' => $title,
|
||||
'year' => isset($payload['year']) ? (int) $payload['year'] : null,
|
||||
'author' => $this->nullableClean($payload['author'] ?? null),
|
||||
'source' => $source,
|
||||
'series' => $this->nullableClean($payload['series'] ?? null),
|
||||
'tags' => is_array($payload['tags'] ?? null) ? array_values($payload['tags']) : [],
|
||||
'summary' => $this->nullableClean($payload['summary'] ?? null),
|
||||
'metadata' => $payload['metadata'] ?? [],
|
||||
'content' => $this->nullableClean($payload['content_url'] ?? $payload['content_path'] ?? null),
|
||||
'raw' => $this->nullableClean($payload['raw_url'] ?? $payload['raw_path'] ?? null),
|
||||
];
|
||||
}
|
||||
|
||||
private function pageBlocks(array $payload): array
|
||||
{
|
||||
if (isset($payload['pages'])) {
|
||||
return $this->pageBlocksFromPages($payload);
|
||||
}
|
||||
|
||||
if (isset($payload['paragraphs'])) {
|
||||
return $this->pageBlocksFromItems($payload, $payload['paragraphs']);
|
||||
}
|
||||
|
||||
return $this->pageBlocksFromItems($payload, preg_split('/\R{2,}/u', $payload['content']));
|
||||
}
|
||||
|
||||
private function pageBlocksFromPages(array $payload): array
|
||||
{
|
||||
$pageBlocks = [];
|
||||
|
||||
foreach ($payload['pages'] as $pageIndex => $page) {
|
||||
$pageNumber = $this->pageNumber($page);
|
||||
$pageMetadata = $page['metadata'] ?? [];
|
||||
$items = $this->markdownBlocksFromPage($page['content']);
|
||||
|
||||
foreach ($items as $itemIndex => $content) {
|
||||
$pageBlock = $this->pageBlock($payload, $content, count($pageBlocks), $itemIndex, $pageNumber, [
|
||||
'page_index' => $pageIndex,
|
||||
'page_metadata' => $pageMetadata,
|
||||
]);
|
||||
|
||||
if ($pageBlock !== null) {
|
||||
$pageBlocks[] = $pageBlock;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $pageBlocks;
|
||||
}
|
||||
|
||||
private function chunksFromPages(string $archiveUid, array $pageBlocks, int $chunkSize, int $chunkOverlap): array
|
||||
{
|
||||
$chunks = [];
|
||||
$chunkIndex = 1;
|
||||
|
||||
foreach ($this->groupBlocksByPage($pageBlocks) as $pageNumber => $blocks) {
|
||||
$units = [];
|
||||
foreach ($blocks as $block) {
|
||||
$unit = $this->cleanEmbeddingText($block['content']);
|
||||
if ($unit === '' || $this->isNoiseBlock($unit)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$units[] = $unit;
|
||||
}
|
||||
|
||||
foreach ($this->packUnitsForEmbedding($units, $chunkSize, $chunkOverlap) as $text) {
|
||||
$page = $this->restorePageNumber($pageNumber);
|
||||
$chunks[] = $this->chunk($archiveUid, $chunkIndex, $page, $text);
|
||||
$chunkIndex++;
|
||||
}
|
||||
}
|
||||
|
||||
return $chunks;
|
||||
}
|
||||
|
||||
private function groupBlocksByPage(array $pageBlocks): array
|
||||
{
|
||||
$pages = [];
|
||||
foreach ($pageBlocks as $block) {
|
||||
$key = $block['page_number'] === null ? '' : (string) $block['page_number'];
|
||||
$pages[$key][] = $block;
|
||||
}
|
||||
|
||||
return $pages;
|
||||
}
|
||||
|
||||
private function pagesSummary(array $pageBlocks, array $chunks): array
|
||||
{
|
||||
$pages = [];
|
||||
|
||||
foreach ($this->groupBlocksByPage($pageBlocks) as $pageNumber => $blocks) {
|
||||
$page = $this->restorePageNumber($pageNumber);
|
||||
$pageChunks = array_values(array_filter($chunks, fn (array $chunk): bool => $chunk['page_start'] === $page));
|
||||
$contentLength = array_sum(array_map(fn (array $block): int => mb_strlen($block['content']), $blocks));
|
||||
|
||||
$pages[] = [
|
||||
'page_number' => $page,
|
||||
'block_count' => count($blocks),
|
||||
'chunk_count' => count($pageChunks),
|
||||
'content_length' => $contentLength,
|
||||
'chunk_uids' => array_column($pageChunks, 'chunk_uid'),
|
||||
];
|
||||
}
|
||||
|
||||
return $pages;
|
||||
}
|
||||
|
||||
private function packUnitsForEmbedding(array $units, int $chunkSize, int $chunkOverlap): array
|
||||
{
|
||||
$chunks = [];
|
||||
$current = '';
|
||||
|
||||
foreach ($units as $unit) {
|
||||
$unit = $this->clean($unit);
|
||||
if ($unit === '') {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (mb_strlen($unit) > $chunkSize) {
|
||||
if ($current !== '') {
|
||||
$chunks[] = $current;
|
||||
$current = '';
|
||||
}
|
||||
|
||||
array_push($chunks, ...$this->chunkLongUnit($unit, $chunkSize, $chunkOverlap));
|
||||
continue;
|
||||
}
|
||||
|
||||
$candidate = $current === '' ? $unit : $current . "\n\n" . $unit;
|
||||
if (mb_strlen($candidate) <= $chunkSize) {
|
||||
$current = $candidate;
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($current !== '') {
|
||||
$chunks[] = $current;
|
||||
}
|
||||
$current = $unit;
|
||||
}
|
||||
|
||||
if ($current !== '') {
|
||||
$chunks[] = $current;
|
||||
}
|
||||
|
||||
return $chunks;
|
||||
}
|
||||
|
||||
private function chunk(string $archiveUid, int $chunkIndex, int|string|null $pageNumber, string $text): array
|
||||
{
|
||||
$chunkUid = $this->chunkUid($archiveUid, $chunkIndex, implode('|', [
|
||||
$chunkIndex,
|
||||
(string) ($pageNumber ?? ''),
|
||||
$text,
|
||||
]));
|
||||
|
||||
return [
|
||||
'chunk_uid' => $chunkUid,
|
||||
'chunk_index' => $chunkIndex,
|
||||
'page_start' => $pageNumber,
|
||||
'page_end' => $pageNumber,
|
||||
'pages' => $pageNumber === null ? [] : [$pageNumber],
|
||||
'text' => $text,
|
||||
'length' => mb_strlen($text),
|
||||
'embedding_ref' => null,
|
||||
];
|
||||
}
|
||||
|
||||
private function normalizePayload(array $payload): array
|
||||
{
|
||||
if (isset($payload['content']) && is_string($payload['content']) && !isset($payload['pages']) && !isset($payload['paragraphs'])) {
|
||||
$payload['pages'] = $this->pagesFromMarkdown($payload['content']);
|
||||
}
|
||||
|
||||
if (!isset($payload['source']) || trim((string) $payload['source']) === '') {
|
||||
$payload['source'] = 'raw-markdown';
|
||||
}
|
||||
|
||||
return $payload;
|
||||
}
|
||||
|
||||
private function applyMetadataFallbacks(array $payload): array
|
||||
{
|
||||
if ((!isset($payload['title']) || trim((string) $payload['title']) === '') && isset($payload['content']) && is_string($payload['content'])) {
|
||||
$payload['title'] = $this->inferTitle($payload['content'], (string) ($payload['source'] ?? ''));
|
||||
$payload['metadata'] = is_array($payload['metadata'] ?? null) ? $payload['metadata'] : [];
|
||||
$payload['metadata']['title_source'] = 'fallback';
|
||||
}
|
||||
|
||||
if (isset($payload['tags']) && is_string($payload['tags'])) {
|
||||
$payload['tags'] = $this->tagsFromString($payload['tags']);
|
||||
}
|
||||
$payload['tags'] = is_array($payload['tags'] ?? null) ? $payload['tags'] : [];
|
||||
|
||||
if (isset($payload['year']) && is_numeric($payload['year'])) {
|
||||
$payload['year'] = (int) $payload['year'];
|
||||
}
|
||||
|
||||
$payload['metadata'] = is_array($payload['metadata'] ?? null) ? $payload['metadata'] : [];
|
||||
|
||||
return $payload;
|
||||
}
|
||||
|
||||
private function pagesFromMarkdown(string $markdown): array
|
||||
{
|
||||
preg_match_all(self::MARKDOWN_PAGE_PATTERN, $markdown, $matches, PREG_SET_ORDER);
|
||||
if ($matches === []) {
|
||||
return [[
|
||||
'page_number' => 1,
|
||||
'content' => $this->cleanMarkdownPage($markdown),
|
||||
'metadata' => ['parser' => 'markdown_single_page'],
|
||||
]];
|
||||
}
|
||||
|
||||
$pages = [];
|
||||
foreach ($matches as $index => $match) {
|
||||
$pageNumber = ctype_digit($match[1]) ? (int) $match[1] : $match[1];
|
||||
$pages[] = [
|
||||
'page_number' => $pageNumber,
|
||||
'content' => $this->cleanMarkdownPage($match[2]),
|
||||
'metadata' => [
|
||||
'parser' => 'docmaster_markdown',
|
||||
'page_index' => $index,
|
||||
],
|
||||
];
|
||||
}
|
||||
|
||||
return $pages;
|
||||
}
|
||||
|
||||
private function markdownBlocksFromPage(string $content): array
|
||||
{
|
||||
$content = $this->cleanMarkdownPage($content);
|
||||
$blocks = preg_split('/\R{2,}/u', $content, -1, PREG_SPLIT_NO_EMPTY);
|
||||
if ($blocks === false) {
|
||||
return [$content];
|
||||
}
|
||||
|
||||
$records = [];
|
||||
foreach ($blocks as $block) {
|
||||
$block = $this->cleanMarkdownBlock($block);
|
||||
if ($block === '') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$lastIndex = count($records) - 1;
|
||||
if ($lastIndex >= 0 && $this->isCommentBlock($block) && $this->isPolicyRecordBlock($records[$lastIndex])) {
|
||||
$records[$lastIndex] .= "\n" . $block;
|
||||
continue;
|
||||
}
|
||||
|
||||
$records[] = $block;
|
||||
}
|
||||
|
||||
return $records;
|
||||
}
|
||||
|
||||
private function cleanMarkdownPage(string $content): string
|
||||
{
|
||||
$content = preg_replace('/<!--\s*DOCMASTER:PAGE\s+[^>]+-->/iu', '', $content) ?? $content;
|
||||
$content = preg_replace('/^\s*#+\s*Page\s+\S+\s*$/imu', '', $content) ?? $content;
|
||||
$content = preg_replace('/^\s*---+\s*$/mu', '', $content) ?? $content;
|
||||
$content = html_entity_decode($content, ENT_QUOTES | ENT_HTML5, 'UTF-8');
|
||||
|
||||
return trim($content);
|
||||
}
|
||||
|
||||
private function cleanMarkdownBlock(string $block): string
|
||||
{
|
||||
$block = preg_replace('/[ \t]+/u', ' ', $block) ?? $block;
|
||||
$block = preg_replace('/\R[ \t]+/u', "\n", $block) ?? $block;
|
||||
return trim($block);
|
||||
}
|
||||
|
||||
private function isCommentBlock(string $block): bool
|
||||
{
|
||||
return (bool) preg_match('/^\s*(?:[*#\s_~`>-])*COMMENT\b/iu', $this->plainBlock($block));
|
||||
}
|
||||
|
||||
private function isPolicyRecordBlock(string $block): bool
|
||||
{
|
||||
return (bool) preg_match('/^\s*(?:NSAM|NSDM|NSDD|NSD|PD)\s+\d+/iu', $this->plainBlock($block));
|
||||
}
|
||||
|
||||
private function isNoiseBlock(string $block): bool
|
||||
{
|
||||
$plain = strtoupper($this->plainBlock($block));
|
||||
$plain = preg_replace('/\s+/u', ' ', $plain) ?? $plain;
|
||||
|
||||
if ($plain === '') {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (preg_match('/^\d{1,6}$/', $plain)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
$patterns = [
|
||||
'/^(?:# )?UNCLASSIFIED$/',
|
||||
'/^TOP SECRET$/',
|
||||
'/^UNCLASSIFIED WITH TOP SECRET ATTACHMENTS$/',
|
||||
'/^DECLASSIFY ON: OADR$/',
|
||||
'/^\*? ?UNCLASSIFIED\*?$/',
|
||||
];
|
||||
|
||||
foreach ($patterns as $pattern) {
|
||||
if (preg_match($pattern, $plain)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private function cleanEmbeddingText(string $text): string
|
||||
{
|
||||
$lines = preg_split('/\R/u', $text);
|
||||
if ($lines === false) {
|
||||
return $this->cleanMarkdownBlock($text);
|
||||
}
|
||||
|
||||
$kept = [];
|
||||
foreach ($lines as $line) {
|
||||
$line = trim($line);
|
||||
if ($line === '' || $this->isNoiseLine($line)) {
|
||||
continue;
|
||||
}
|
||||
$kept[] = $line;
|
||||
}
|
||||
|
||||
return $this->cleanMarkdownBlock(implode("\n", $kept));
|
||||
}
|
||||
|
||||
private function isNoiseLine(string $line): bool
|
||||
{
|
||||
$plain = strtoupper($this->plainBlock($line));
|
||||
$plain = preg_replace('/\s+/u', ' ', $plain) ?? $plain;
|
||||
|
||||
if ($plain === '' || preg_match('/^\d{1,6}$/', $plain)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
$patterns = [
|
||||
'/^(?:# )?UNCLASSIFIED$/',
|
||||
'/^TOP SECRET$/',
|
||||
'/^UNCLASSIFIED WITH TOP SECRET ATTACHMENTS$/',
|
||||
'/^DECLASSIFY ON: OADR$/',
|
||||
'/^PARTIALLY DECLASSIFIED\/RELEASED ON .+$/',
|
||||
'/^UNDER PROVISIONS OF .+$/',
|
||||
'/^BY .+ NATIONAL SECURITY COUNCIL$/',
|
||||
'/^F \d{2}-\d+$/',
|
||||
];
|
||||
|
||||
foreach ($patterns as $pattern) {
|
||||
if (preg_match($pattern, $plain)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private function plainBlock(string $block): string
|
||||
{
|
||||
$block = str_replace(['*', '_', '`', '~'], '', $block);
|
||||
$block = preg_replace('/\s+/u', ' ', $block) ?? $block;
|
||||
return trim($block);
|
||||
}
|
||||
|
||||
private function inferTitle(string $markdown, string $source): string
|
||||
{
|
||||
if (preg_match_all('/^\s*#+\s*(.+?)\s*$/imu', $markdown, $matches)) {
|
||||
foreach ($matches[1] as $heading) {
|
||||
$heading = $this->clean($heading);
|
||||
if (!preg_match('/^Page\s+\S+$/iu', $heading) && !$this->isNoiseLine($heading)) {
|
||||
return $heading;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($source !== '') {
|
||||
return pathinfo($source, PATHINFO_FILENAME) ?: $source;
|
||||
}
|
||||
|
||||
return 'Untitled Markdown Import';
|
||||
}
|
||||
|
||||
private function pageBlocksFromItems(array $payload, array $items): array
|
||||
{
|
||||
$pageBlocks = [];
|
||||
|
||||
foreach ($items as $index => $item) {
|
||||
$content = is_array($item) ? ($item['content'] ?? '') : $item;
|
||||
$pageNumber = is_array($item) && $this->hasPageNumber($item) ? $this->pageNumber($item) : null;
|
||||
$metadata = is_array($item) ? ($item['metadata'] ?? []) : [];
|
||||
$pageBlock = $this->pageBlock($payload, $content, count($pageBlocks), $index, $pageNumber, $metadata);
|
||||
|
||||
if ($pageBlock !== null) {
|
||||
$pageBlocks[] = $pageBlock;
|
||||
}
|
||||
}
|
||||
|
||||
return $pageBlocks;
|
||||
}
|
||||
|
||||
private function pageBlock(
|
||||
array $payload,
|
||||
mixed $content,
|
||||
int $blockIndex,
|
||||
int $sourceIndex,
|
||||
int|string|null $pageNumber,
|
||||
array $metadata
|
||||
): ?array {
|
||||
$content = $this->clean((string) $content);
|
||||
if ($content === '') {
|
||||
return null;
|
||||
}
|
||||
|
||||
return [
|
||||
'block_uid' => $this->uid('block', implode('|', [
|
||||
$payload['source'],
|
||||
$payload['title'],
|
||||
(string) $pageNumber,
|
||||
$sourceIndex,
|
||||
$content,
|
||||
])),
|
||||
'index' => $blockIndex,
|
||||
'page_number' => $pageNumber,
|
||||
'content' => $content,
|
||||
'metadata' => $metadata,
|
||||
];
|
||||
}
|
||||
|
||||
private function chunkLongUnit(string $text, int $chunkSize, int $chunkOverlap): array
|
||||
{
|
||||
if (mb_strlen($text) <= $chunkSize) {
|
||||
return [$text];
|
||||
}
|
||||
|
||||
$chunks = [];
|
||||
$current = '';
|
||||
|
||||
foreach ($this->semanticUnits($text) as $unit) {
|
||||
if (mb_strlen($unit) > $chunkSize) {
|
||||
if ($current !== '') {
|
||||
$chunks[] = $current;
|
||||
$current = '';
|
||||
}
|
||||
|
||||
array_push($chunks, ...$this->hardChunk($unit, $chunkSize, $chunkOverlap));
|
||||
continue;
|
||||
}
|
||||
|
||||
$candidate = $current === '' ? $unit : $current . ' ' . $unit;
|
||||
if (mb_strlen($candidate) <= $chunkSize) {
|
||||
$current = $candidate;
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($current !== '') {
|
||||
$chunks[] = $current;
|
||||
}
|
||||
$current = $unit;
|
||||
}
|
||||
|
||||
if ($current !== '') {
|
||||
$chunks[] = $current;
|
||||
}
|
||||
|
||||
return $chunks === [] ? [$text] : $chunks;
|
||||
}
|
||||
|
||||
private function semanticUnits(string $text): array
|
||||
{
|
||||
$units = preg_split(self::SENTENCE_BOUNDARY_PATTERN, $text, -1, PREG_SPLIT_NO_EMPTY);
|
||||
if ($units === false || $units === []) {
|
||||
return [$text];
|
||||
}
|
||||
|
||||
return array_values(array_filter(array_map(fn (string $unit): string => $this->clean($unit), $units)));
|
||||
}
|
||||
|
||||
private function hardChunk(string $text, int $chunkSize, int $chunkOverlap): array
|
||||
{
|
||||
$length = mb_strlen($text);
|
||||
if ($length <= $chunkSize) {
|
||||
return [$text];
|
||||
}
|
||||
|
||||
$chunks = [];
|
||||
$start = 0;
|
||||
while ($start < $length) {
|
||||
$chunk = mb_substr($text, $start, $chunkSize);
|
||||
if ($chunk === '') {
|
||||
break;
|
||||
}
|
||||
|
||||
$chunks[] = $chunk;
|
||||
if ($start + $chunkSize >= $length) {
|
||||
break;
|
||||
}
|
||||
|
||||
$start += $chunkSize - $chunkOverlap;
|
||||
}
|
||||
|
||||
return $chunks;
|
||||
}
|
||||
|
||||
private function hasPageNumber(array $item): bool
|
||||
{
|
||||
return array_key_exists('page_number', $item)
|
||||
|| array_key_exists('page', $item)
|
||||
|| array_key_exists('number', $item);
|
||||
}
|
||||
|
||||
private function pageNumber(array $item): int|string
|
||||
{
|
||||
$pageNumber = $item['page_number'] ?? $item['page'] ?? $item['number'];
|
||||
return is_int($pageNumber) ? $pageNumber : $this->clean((string) $pageNumber);
|
||||
}
|
||||
|
||||
private function restorePageNumber(string $pageNumber): int|string|null
|
||||
{
|
||||
if ($pageNumber === '') {
|
||||
return null;
|
||||
}
|
||||
|
||||
return ctype_digit($pageNumber) ? (int) $pageNumber : $pageNumber;
|
||||
}
|
||||
|
||||
private function intOption(array $payload, string $key, int $default): int
|
||||
{
|
||||
if (!isset($payload[$key]) || $payload[$key] === '') {
|
||||
return $default;
|
||||
}
|
||||
|
||||
return (int) $payload[$key];
|
||||
}
|
||||
|
||||
private function clean(string $value): string
|
||||
{
|
||||
return trim(preg_replace('/[ \t]+/u', ' ', $value) ?? $value);
|
||||
}
|
||||
|
||||
private function nullableClean(mixed $value): ?string
|
||||
{
|
||||
if (!is_string($value)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$value = $this->clean($value);
|
||||
return $value === '' ? null : $value;
|
||||
}
|
||||
|
||||
private function tagsFromString(string $value): array
|
||||
{
|
||||
$value = trim($value);
|
||||
if ($value === '') {
|
||||
return [];
|
||||
}
|
||||
|
||||
$decoded = json_decode($value, true);
|
||||
if (is_array($decoded)) {
|
||||
return array_values(array_filter(array_map('strval', $decoded)));
|
||||
}
|
||||
|
||||
return array_values(array_filter(array_map('trim', preg_split('/[,，]/u', $value) ?: [])));
|
||||
}
|
||||
|
||||
private function archiveUid(array $payload): string
|
||||
{
|
||||
if (isset($payload['archive_uid']) && is_string($payload['archive_uid']) && $this->isUlid($payload['archive_uid'])) {
|
||||
return strtoupper($payload['archive_uid']);
|
||||
}
|
||||
|
||||
return (string) new Ulid();
|
||||
}
|
||||
|
||||
private function chunkUid(string $archiveUid, int $chunkIndex, string $value): string
|
||||
{
|
||||
return $archiveUid . '_' . $chunkIndex . '_' . $this->shortUid($value);
|
||||
}
|
||||
|
||||
private function shortUid(string $value): string
|
||||
{
|
||||
$number = hexdec(substr(hash('crc32b', $value), 0, 8)) % 100000;
|
||||
return str_pad((string) $number, 5, '0', STR_PAD_LEFT);
|
||||
}
|
||||
|
||||
private function isUlid(string $value): bool
|
||||
{
|
||||
return (bool) preg_match('/^[0-9A-HJKMNP-TV-Z]{26}$/', strtoupper($value));
|
||||
}
|
||||
|
||||
private function uid(string $prefix, string $value): string
|
||||
{
|
||||
return $prefix . '_' . substr(hash('sha256', $value), 0, 24);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
<?php
|
||||
|
||||
namespace app\service\LLM;
|
||||
|
||||
use RuntimeException;
|
||||
|
||||
class LLMRequestException extends RuntimeException
|
||||
{
|
||||
public function __construct(
|
||||
string $message,
|
||||
private readonly ?int $statusCode = null,
|
||||
private readonly ?string $providerCode = null,
|
||||
private readonly ?array $payload = null
|
||||
) {
|
||||
parent::__construct($message, $statusCode ?? 0);
|
||||
}
|
||||
|
||||
public function statusCode(): ?int
|
||||
{
|
||||
return $this->statusCode;
|
||||
}
|
||||
|
||||
public function providerCode(): ?string
|
||||
{
|
||||
return $this->providerCode;
|
||||
}
|
||||
|
||||
public function payload(): ?array
|
||||
{
|
||||
return $this->payload;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
<?php
|
||||
|
||||
namespace app\service\LLM;
|
||||
|
||||
use Throwable;
|
||||
|
||||
class LLMRetryQueue
|
||||
{
|
||||
public function run(callable $job, array $config = []): mixed
|
||||
{
|
||||
$enabled = (bool) ($config['enabled'] ?? true);
|
||||
$maxAttempts = max(1, (int) ($config['max_attempts'] ?? 3));
|
||||
$baseDelayMs = max(0, (int) ($config['base_delay_ms'] ?? 1500));
|
||||
$maxDelayMs = max($baseDelayMs, (int) ($config['max_delay_ms'] ?? 10000));
|
||||
$retryStatuses = array_map('intval', $config['retry_statuses'] ?? [429]);
|
||||
$retryErrorCodes = array_map('strval', $config['retry_error_codes'] ?? ['1302', '1303', '1304', '1305', '1306', '1307', '1308']);
|
||||
|
||||
$attempt = 0;
|
||||
while (true) {
|
||||
$attempt++;
|
||||
|
||||
try {
|
||||
return $job($attempt);
|
||||
} catch (Throwable $exception) {
|
||||
if (!$enabled || $attempt >= $maxAttempts || !$this->shouldRetry($exception, $retryStatuses, $retryErrorCodes)) {
|
||||
throw $exception;
|
||||
}
|
||||
|
||||
usleep($this->delayMs($attempt, $baseDelayMs, $maxDelayMs) * 1000);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private function shouldRetry(Throwable $exception, array $retryStatuses, array $retryErrorCodes): bool
|
||||
{
|
||||
if (!$exception instanceof LLMRequestException) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if ($exception->statusCode() !== null && in_array($exception->statusCode(), $retryStatuses, true)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return $exception->providerCode() !== null && in_array($exception->providerCode(), $retryErrorCodes, true);
|
||||
}
|
||||
|
||||
private function delayMs(int $attempt, int $baseDelayMs, int $maxDelayMs): int
|
||||
{
|
||||
$delay = min($maxDelayMs, $baseDelayMs * (2 ** max(0, $attempt - 1)));
|
||||
$jitter = $delay > 0 ? random_int(0, min(500, $delay)) : 0;
|
||||
|
||||
return $delay + $jitter;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
<?php
|
||||
|
||||
namespace app\service\LLM;
|
||||
|
||||
use GuzzleHttp\Client;
|
||||
use GuzzleHttp\Exception\RequestException;
|
||||
use RuntimeException;
|
||||
use Throwable;
|
||||
|
||||
class OpenAICompatibleClient
|
||||
{
|
||||
private Client $client;
|
||||
private array $config;
|
||||
|
||||
public function __construct(?array $config = null)
|
||||
{
|
||||
$this->config = $config ?? config('LLMapi.default', []);
|
||||
$baseUrl = rtrim((string) ($this->config['base_url'] ?? ''), '/');
|
||||
|
||||
$this->client = new Client([
|
||||
'base_uri' => $baseUrl . '/',
|
||||
'timeout' => (int) ($this->config['timeout'] ?? 60),
|
||||
'connect_timeout' => (int) ($this->config['connect_timeout'] ?? 10),
|
||||
]);
|
||||
}
|
||||
|
||||
public function isConfigured(): bool
|
||||
{
|
||||
return trim((string) ($this->config['api_key'] ?? '')) !== ''
|
||||
&& trim((string) ($this->config['base_url'] ?? '')) !== '';
|
||||
}
|
||||
|
||||
public function chatJson(array $messages, array $options = []): array
|
||||
{
|
||||
$content = $this->chat($messages, $options);
|
||||
$decoded = json_decode($this->extractJson($content), true);
|
||||
|
||||
if (!is_array($decoded)) {
|
||||
throw new RuntimeException('LLM response is not valid JSON.');
|
||||
}
|
||||
|
||||
return $decoded;
|
||||
}
|
||||
|
||||
public function chat(array $messages, array $options = []): string
|
||||
{
|
||||
if (!$this->isConfigured()) {
|
||||
throw new RuntimeException('LLM API is not configured.');
|
||||
}
|
||||
|
||||
$headers = [
|
||||
'Authorization' => 'Bearer ' . $this->config['api_key'],
|
||||
'Content-Type' => 'application/json',
|
||||
];
|
||||
|
||||
if (!empty($this->config['organization'])) {
|
||||
$headers['OpenAI-Organization'] = $this->config['organization'];
|
||||
}
|
||||
|
||||
if (!empty($this->config['project'])) {
|
||||
$headers['OpenAI-Project'] = $this->config['project'];
|
||||
}
|
||||
|
||||
$body = [
|
||||
'model' => $options['model'] ?? config('LLMapi.chat.model'),
|
||||
'messages' => $messages,
|
||||
'temperature' => $options['temperature'] ?? config('LLMapi.chat.temperature', 0.2),
|
||||
'max_tokens' => $options['max_tokens'] ?? config('LLMapi.chat.max_tokens', 1200),
|
||||
'stream' => (bool) ($options['stream'] ?? config('LLMapi.chat.stream', false)),
|
||||
];
|
||||
|
||||
if (array_key_exists('response_format', $options) && is_array($options['response_format'])) {
|
||||
$body['response_format'] = $options['response_format'];
|
||||
}
|
||||
|
||||
if (array_key_exists('thinking', $options) && is_array($options['thinking'])) {
|
||||
$body['thinking'] = $options['thinking'];
|
||||
}
|
||||
|
||||
if (array_key_exists('request_id', $options) && is_string($options['request_id'])) {
|
||||
$body['request_id'] = $options['request_id'];
|
||||
}
|
||||
|
||||
if (array_key_exists('user_id', $options) && is_string($options['user_id'])) {
|
||||
$body['user_id'] = $options['user_id'];
|
||||
}
|
||||
|
||||
try {
|
||||
$response = $this->client->post('chat/completions', [
|
||||
'headers' => $headers,
|
||||
'json' => $body,
|
||||
]);
|
||||
} catch (RequestException $exception) {
|
||||
throw $this->requestException($exception);
|
||||
} catch (Throwable $exception) {
|
||||
throw new RuntimeException('LLM chat request failed: ' . $exception->getMessage(), 0, $exception);
|
||||
}
|
||||
|
||||
$payload = json_decode((string) $response->getBody(), true);
|
||||
$content = $payload['choices'][0]['message']['content'] ?? null;
|
||||
|
||||
if (!is_string($content) || trim($content) === '') {
|
||||
throw new RuntimeException('LLM chat response is empty.');
|
||||
}
|
||||
|
||||
return $content;
|
||||
}
|
||||
|
||||
private function extractJson(string $content): string
|
||||
{
|
||||
$content = trim($content);
|
||||
$content = preg_replace('/^```(?:json)?\s*/i', '', $content) ?? $content;
|
||||
$content = preg_replace('/\s*```$/', '', $content) ?? $content;
|
||||
|
||||
$start = strpos($content, '{');
|
||||
$end = strrpos($content, '}');
|
||||
if ($start !== false && $end !== false && $end > $start) {
|
||||
return substr($content, $start, $end - $start + 1);
|
||||
}
|
||||
|
||||
return $content;
|
||||
}
|
||||
|
||||
private function requestException(RequestException $exception): LLMRequestException
|
||||
{
|
||||
$statusCode = $exception->getResponse()?->getStatusCode();
|
||||
$body = $exception->getResponse() ? (string) $exception->getResponse()->getBody() : '';
|
||||
$payload = json_decode($body, true);
|
||||
$providerCode = null;
|
||||
$providerMessage = null;
|
||||
|
||||
if (is_array($payload)) {
|
||||
$providerCode = isset($payload['error']['code']) ? (string) $payload['error']['code'] : null;
|
||||
$providerMessage = isset($payload['error']['message']) ? (string) $payload['error']['message'] : null;
|
||||
}
|
||||
|
||||
$message = 'LLM chat request failed';
|
||||
if ($statusCode !== null) {
|
||||
$message .= " with HTTP {$statusCode}";
|
||||
}
|
||||
if ($providerCode !== null) {
|
||||
$message .= " and provider code {$providerCode}";
|
||||
}
|
||||
if ($providerMessage !== null) {
|
||||
$message .= ": {$providerMessage}";
|
||||
} else {
|
||||
$message .= ': ' . $exception->getMessage();
|
||||
}
|
||||
|
||||
return new LLMRequestException($message, $statusCode, $providerCode, is_array($payload) ? $payload : null);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<link rel="shortcut icon" href="/favicon.ico"/>
|
||||
<title>webman</title>
|
||||
|
||||
</head>
|
||||
<body>
|
||||
hello <?=htmlspecialchars($name)?>
|
||||
</body>
|
||||
</html>
|
||||
Reference in New Issue
Block a user