暂存
This commit is contained in:
@@ -0,0 +1,99 @@
|
||||
<?php
|
||||
|
||||
namespace app\service;
|
||||
|
||||
use support\Redis;
|
||||
|
||||
class AiMetadataQueue
|
||||
{
|
||||
public function push(string $archiveUid): void
|
||||
{
|
||||
Redis::lPush($this->pendingKey(), $archiveUid);
|
||||
}
|
||||
|
||||
public function pop(int $timeout = 5): ?string
|
||||
{
|
||||
$result = Redis::brPop([$this->pendingKey()], $timeout);
|
||||
if (!is_array($result) || count($result) < 2) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (string) $result[1];
|
||||
}
|
||||
|
||||
public function retryLater(string $archiveUid, string $error): void
|
||||
{
|
||||
$retryKey = $this->retryKey($archiveUid);
|
||||
$retryCount = (int) Redis::incr($retryKey);
|
||||
Redis::setEx($this->errorKey($archiveUid), 86400, $error);
|
||||
|
||||
if ($retryCount > $this->maxRetries()) {
|
||||
Redis::lPush($this->failedKey(), $archiveUid);
|
||||
return;
|
||||
}
|
||||
|
||||
$delay = $this->baseDelaySeconds() * (2 ** max(0, $retryCount - 1));
|
||||
Redis::zAdd($this->delayedKey(), time() + $delay, $archiveUid);
|
||||
}
|
||||
|
||||
public function releaseDueDelayed(): int
|
||||
{
|
||||
$now = time();
|
||||
$items = Redis::zRangeByScore($this->delayedKey(), '-inf', (string) $now, ['limit' => [0, 100]]);
|
||||
if (!is_array($items) || $items === []) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
foreach ($items as $archiveUid) {
|
||||
Redis::zRem($this->delayedKey(), $archiveUid);
|
||||
Redis::lPush($this->pendingKey(), $archiveUid);
|
||||
}
|
||||
|
||||
return count($items);
|
||||
}
|
||||
|
||||
public function clearRetry(string $archiveUid): void
|
||||
{
|
||||
Redis::del($this->retryKey($archiveUid), $this->errorKey($archiveUid));
|
||||
}
|
||||
|
||||
public function blockTimeout(): int
|
||||
{
|
||||
return (int) config('queue.ai_metadata.block_timeout', 5);
|
||||
}
|
||||
|
||||
private function pendingKey(): string
|
||||
{
|
||||
return config('queue.ai_metadata.pending', 'proofdb:ai:metadata:pending');
|
||||
}
|
||||
|
||||
private function delayedKey(): string
|
||||
{
|
||||
return config('queue.ai_metadata.delayed', 'proofdb:ai:metadata:delayed');
|
||||
}
|
||||
|
||||
private function failedKey(): string
|
||||
{
|
||||
return config('queue.ai_metadata.failed', 'proofdb:ai:metadata:failed');
|
||||
}
|
||||
|
||||
private function retryKey(string $archiveUid): string
|
||||
{
|
||||
return config('queue.ai_metadata.retry_prefix', 'proofdb:ai:metadata:retry:') . $archiveUid;
|
||||
}
|
||||
|
||||
private function errorKey(string $archiveUid): string
|
||||
{
|
||||
return config('queue.ai_metadata.error_prefix', 'proofdb:ai:metadata:error:') . $archiveUid;
|
||||
}
|
||||
|
||||
private function maxRetries(): int
|
||||
{
|
||||
return (int) config('queue.ai_metadata.max_retries', 5);
|
||||
}
|
||||
|
||||
private function baseDelaySeconds(): int
|
||||
{
|
||||
return (int) config('queue.ai_metadata.base_delay_seconds', 60);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
<?php
|
||||
|
||||
namespace app\service;
|
||||
|
||||
use app\service\LLM\OpenAICompatibleClient;
|
||||
use app\service\LLM\LLMRetryQueue;
|
||||
use Throwable;
|
||||
|
||||
class ArchiveMetadataEnrichmentService
|
||||
{
|
||||
private OpenAICompatibleClient $client;
|
||||
private LLMRetryQueue $queue;
|
||||
|
||||
public function __construct(?OpenAICompatibleClient $client = null, ?LLMRetryQueue $queue = null)
|
||||
{
|
||||
$this->client = $client ?? new OpenAICompatibleClient();
|
||||
$this->queue = $queue ?? new LLMRetryQueue();
|
||||
}
|
||||
|
||||
public function enrich(array $payload): array
|
||||
{
|
||||
$missing = $this->missingFields($payload);
|
||||
if ($missing === [] || !$this->enabled()) {
|
||||
return $this->withAiMeta($payload, [
|
||||
'enabled' => $this->enabled(),
|
||||
'attempted' => false,
|
||||
'filled' => [],
|
||||
'missing' => $missing,
|
||||
]);
|
||||
}
|
||||
|
||||
try {
|
||||
$result = $this->queue->run(
|
||||
fn (): array => $this->client->chatJson($this->messages($payload, $missing), [
|
||||
'model' => config('LLMapi.metadata.model'),
|
||||
'temperature' => config('LLMapi.metadata.temperature', 0.1),
|
||||
'max_tokens' => config('LLMapi.metadata.max_tokens', 1200),
|
||||
'stream' => false,
|
||||
'response_format' => config('LLMapi.metadata.response_format', ['type' => 'json_object']),
|
||||
'thinking' => config('LLMapi.metadata.thinking', ['type' => 'disabled']),
|
||||
'request_id' => $this->requestId($payload, $missing),
|
||||
]),
|
||||
config('LLMapi.metadata.retry', [])
|
||||
);
|
||||
} catch (Throwable $exception) {
|
||||
return $this->withAiMeta($payload, [
|
||||
'enabled' => true,
|
||||
'attempted' => true,
|
||||
'filled' => [],
|
||||
'missing' => $missing,
|
||||
'error' => $exception->getMessage(),
|
||||
]);
|
||||
}
|
||||
|
||||
$filled = [];
|
||||
foreach ($missing as $field) {
|
||||
if (!$this->hasUsefulValue($result, $field)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$payload[$field] = $this->normalizeField($field, $result[$field]);
|
||||
$filled[] = $field;
|
||||
}
|
||||
|
||||
return $this->withAiMeta($payload, [
|
||||
'enabled' => true,
|
||||
'attempted' => true,
|
||||
'filled' => $filled,
|
||||
'missing' => array_values(array_diff($missing, $filled)),
|
||||
'model' => config('LLMapi.metadata.model'),
|
||||
'stream' => false,
|
||||
'response_format' => config('LLMapi.metadata.response_format', ['type' => 'json_object']),
|
||||
'thinking' => config('LLMapi.metadata.thinking', ['type' => 'disabled']),
|
||||
]);
|
||||
}
|
||||
|
||||
private function missingFields(array $payload): array
|
||||
{
|
||||
$fields = ['title', 'year', 'author', 'tags', 'summary'];
|
||||
return array_values(array_filter($fields, fn (string $field): bool => !$this->hasUsefulValue($payload, $field)));
|
||||
}
|
||||
|
||||
private function enabled(): bool
|
||||
{
|
||||
return (bool) config('LLMapi.metadata.enabled', true) && $this->client->isConfigured();
|
||||
}
|
||||
|
||||
private function messages(array $payload, array $missing): array
|
||||
{
|
||||
$text = $this->sampleText($payload);
|
||||
|
||||
return [
|
||||
[
|
||||
'role' => 'system',
|
||||
'content' => implode("\n", [
|
||||
'你是历史档案元数据整理助手。',
|
||||
'你只能根据用户提供的档案文本抽取或推断元数据。',
|
||||
'请只返回 JSON 对象，不要返回 Markdown，不要解释。',
|
||||
'字段：title(string), year(integer|null), author(string|null), tags(array<string>), summary(string)。',
|
||||
'summary 简洁概括档案内容，80-200 字。',
|
||||
'tags 用档案中常见专名和涉及主题，5-10 个。',
|
||||
'无法判断的字段返回 null 或空数组。',
|
||||
'以上请均使用档案中的语言。',
|
||||
]),
|
||||
],
|
||||
[
|
||||
'role' => 'user',
|
||||
'content' => json_encode([
|
||||
'missing_fields' => $missing,
|
||||
'known_fields' => [
|
||||
'title' => $payload['title'] ?? null,
|
||||
'year' => $payload['year'] ?? null,
|
||||
'author' => $payload['author'] ?? null,
|
||||
'source' => $payload['source'] ?? null,
|
||||
'series' => $payload['series'] ?? null,
|
||||
],
|
||||
'archive_text_sample' => $text,
|
||||
], JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
|
||||
],
|
||||
];
|
||||
}
|
||||
|
||||
private function requestId(array $payload, array $missing): string
|
||||
{
|
||||
return 'metadata-' . substr(hash('sha256', implode('|', [
|
||||
(string) ($payload['source'] ?? ''),
|
||||
(string) ($payload['archive_uid'] ?? ''),
|
||||
mb_substr((string) ($payload['content'] ?? ''), 0, 1000),
|
||||
implode(',', $missing ?? []),
|
||||
])), 0, 32);
|
||||
}
|
||||
|
||||
private function sampleText(array $payload): string
|
||||
{
|
||||
$text = '';
|
||||
if (isset($payload['content']) && is_string($payload['content'])) {
|
||||
$text = $payload['content'];
|
||||
} elseif (isset($payload['pages']) && is_array($payload['pages'])) {
|
||||
$parts = [];
|
||||
foreach ($payload['pages'] as $page) {
|
||||
if (isset($page['content']) && is_string($page['content'])) {
|
||||
$parts[] = $page['content'];
|
||||
}
|
||||
}
|
||||
$text = implode("\n\n", $parts);
|
||||
}
|
||||
|
||||
$maxChars = (int) config('LLMapi.metadata.max_input_chars', 12000);
|
||||
return mb_substr($text, 0, $maxChars);
|
||||
}
|
||||
|
||||
private function hasUsefulValue(array $payload, string $field): bool
|
||||
{
|
||||
if (!array_key_exists($field, $payload)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if ($field === 'title' && (($payload['metadata']['title_source'] ?? null) === 'fallback')) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$value = $payload[$field];
|
||||
if (is_array($value)) {
|
||||
return $value !== [];
|
||||
}
|
||||
|
||||
if ($field === 'year') {
|
||||
return is_numeric($value) && (int) $value > 0;
|
||||
}
|
||||
|
||||
return is_string($value) ? trim($value) !== '' : $value !== null;
|
||||
}
|
||||
|
||||
private function normalizeField(string $field, mixed $value): mixed
|
||||
{
|
||||
if ($field === 'year') {
|
||||
return is_numeric($value) ? (int) $value : null;
|
||||
}
|
||||
|
||||
if ($field === 'tags') {
|
||||
if (!is_array($value)) {
|
||||
return [];
|
||||
}
|
||||
return array_values(array_filter(array_map('strval', $value)));
|
||||
}
|
||||
|
||||
return is_string($value) ? trim($value) : $value;
|
||||
}
|
||||
|
||||
private function withAiMeta(array $payload, array $ai): array
|
||||
{
|
||||
$payload['metadata'] = is_array($payload['metadata'] ?? null) ? $payload['metadata'] : [];
|
||||
$payload['metadata']['ai_enrichment'] = $ai;
|
||||
|
||||
return $payload;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
<?php
|
||||
|
||||
namespace app\service;
|
||||
|
||||
use support\Db;
|
||||
|
||||
class ArchiveRepository
|
||||
{
|
||||
public function saveImport(array $import): void
|
||||
{
|
||||
Db::transaction(function () use ($import): void {
|
||||
$archive = $import['archive'];
|
||||
$chunks = $import['chunks'];
|
||||
$chunkUids = array_column($chunks, 'chunk_uid');
|
||||
|
||||
Db::table('archives')->updateOrInsert(
|
||||
['archive_uid' => $archive['archive_uid']],
|
||||
[
|
||||
'title' => $archive['title'] ?? null,
|
||||
'summary' => $archive['summary'] ?? null,
|
||||
'year' => $archive['year'] ?? null,
|
||||
'author' => $archive['author'] ?? null,
|
||||
'source' => $archive['source'] ?? null,
|
||||
'series' => $archive['series'] ?? null,
|
||||
'tags' => json_encode($archive['tags'] ?? [], JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
|
||||
'metadata' => json_encode($archive['metadata'] ?? [], JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
|
||||
'content' => $archive['content'] ?? null,
|
||||
'raw' => $archive['raw'] ?? null,
|
||||
'chunks' => json_encode($chunkUids, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
|
||||
]
|
||||
);
|
||||
|
||||
Db::table('chunks')->where('archive_uid', $archive['archive_uid'])->delete();
|
||||
foreach ($chunks as $chunk) {
|
||||
Db::table('chunks')->insert([
|
||||
'chunk_uid' => $chunk['chunk_uid'],
|
||||
'archive_uid' => $archive['archive_uid'],
|
||||
'chunk_index' => $chunk['chunk_index'],
|
||||
'page_start' => $chunk['page_start'],
|
||||
'page_end' => $chunk['page_end'],
|
||||
'text' => $chunk['text'],
|
||||
'length' => $chunk['length'],
|
||||
'embedding_status' => 0,
|
||||
'embedding_ref' => null,
|
||||
'embedding_model' => null,
|
||||
]);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public function findArchive(string $archiveUid): ?array
|
||||
{
|
||||
$archive = Db::table('archives')->where('archive_uid', $archiveUid)->first();
|
||||
if (!$archive) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return $this->archiveToArray($archive);
|
||||
}
|
||||
|
||||
public function findChunksText(string $archiveUid, int $limit = 20): string
|
||||
{
|
||||
$chunks = Db::table('chunks')
|
||||
->where('archive_uid', $archiveUid)
|
||||
->orderBy('chunk_index')
|
||||
->limit($limit)
|
||||
->get(['text'])
|
||||
->all();
|
||||
|
||||
return implode("\n\n", array_map(fn ($chunk): string => (string) $chunk->text, $chunks));
|
||||
}
|
||||
|
||||
public function updateMetadata(string $archiveUid, array $fields, array $aiMeta): void
|
||||
{
|
||||
$archive = $this->findArchive($archiveUid);
|
||||
$metadata = $archive['metadata'] ?? [];
|
||||
$metadata['ai_enrichment'] = $aiMeta;
|
||||
|
||||
$updates = [
|
||||
'metadata' => json_encode($metadata, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
|
||||
];
|
||||
|
||||
foreach (['title', 'summary', 'year', 'author', 'series', 'tags'] as $field) {
|
||||
if (!array_key_exists($field, $fields)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$updates[$field] = $field === 'tags'
|
||||
? json_encode($fields[$field] ?? [], JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES)
|
||||
: $fields[$field];
|
||||
}
|
||||
|
||||
Db::table('archives')->where('archive_uid', $archiveUid)->update($updates);
|
||||
}
|
||||
|
||||
public function archiveNeedsMetadata(array $archive): bool
|
||||
{
|
||||
foreach (['title', 'year', 'author', 'tags', 'summary'] as $field) {
|
||||
$value = $archive[$field] ?? null;
|
||||
if ($field === 'title' && (($archive['metadata']['title_source'] ?? null) === 'fallback')) {
|
||||
return true;
|
||||
}
|
||||
if (is_array($value) && $value === []) {
|
||||
return true;
|
||||
}
|
||||
if ($field === 'year' && (!$value || (int) $value <= 0)) {
|
||||
return true;
|
||||
}
|
||||
if (!is_array($value) && ($value === null || trim((string) $value) === '')) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private function archiveToArray(object $archive): array
|
||||
{
|
||||
return [
|
||||
'archive_uid' => $archive->archive_uid,
|
||||
'title' => $archive->title,
|
||||
'summary' => $archive->summary,
|
||||
'year' => $archive->year,
|
||||
'author' => $archive->author,
|
||||
'source' => $archive->source,
|
||||
'series' => $archive->series,
|
||||
'tags' => json_decode($archive->tags ?? '[]', true) ?: [],
|
||||
'metadata' => json_decode($archive->metadata ?? '{}', true) ?: [],
|
||||
'content' => $archive->content,
|
||||
'raw' => $archive->raw,
|
||||
'chunks' => json_decode($archive->chunks ?? '[]', true) ?: [],
|
||||
];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,775 @@
|
||||
<?php
|
||||
|
||||
namespace app\service;
|
||||
|
||||
use RuntimeException;
|
||||
use Symfony\Component\Uid\Ulid;
|
||||
|
||||
class ArticleImportService
|
||||
{
|
||||
private const DEFAULT_CHUNK_SIZE = 800;
|
||||
private const DEFAULT_CHUNK_OVERLAP = 120;
|
||||
private const MIN_CHUNK_SIZE = 100;
|
||||
private const MAX_CHUNK_SIZE = 4000;
|
||||
private const SENTENCE_BOUNDARY_PATTERN = '/(?<=[。！？；.!?;])\s*|\R+/u';
|
||||
private const MARKDOWN_PAGE_PATTERN = '/<!--\s*DOCMASTER:PAGE\s+0*([0-9A-Za-z_-]+)\s*-->\s*(?:#+\s*Page\s+\S+\s*)?(.*?)(?=\n---\s*\n\s*<!--\s*DOCMASTER:PAGE|\z)/su';
|
||||
|
||||
public function import(array $payload): array
|
||||
{
|
||||
$payload = $this->normalizePayload($payload);
|
||||
$payload = $this->applyMetadataFallbacks($payload);
|
||||
$errors = $this->validate($payload);
|
||||
if ($errors !== []) {
|
||||
return ['ok' => false, 'errors' => $errors];
|
||||
}
|
||||
|
||||
$archiveUid = $this->archiveUid($payload);
|
||||
$archive = $this->archive($payload, $archiveUid);
|
||||
$pageBlocks = $this->pageBlocks($payload);
|
||||
$chunkSize = $this->intOption($payload, 'chunk_size', self::DEFAULT_CHUNK_SIZE);
|
||||
$chunkOverlap = $this->intOption($payload, 'chunk_overlap', self::DEFAULT_CHUNK_OVERLAP);
|
||||
|
||||
$chunks = $this->chunksFromPages($archiveUid, $pageBlocks, $chunkSize, $chunkOverlap);
|
||||
$pages = $this->pagesSummary($pageBlocks, $chunks);
|
||||
$needsAiMetadata = (new ArchiveRepository())->archiveNeedsMetadata($archive);
|
||||
|
||||
return [
|
||||
'ok' => true,
|
||||
'data' => [
|
||||
'import_uid' => $archiveUid,
|
||||
'archive' => $archive,
|
||||
'chunks' => $chunks,
|
||||
'pages' => $pages,
|
||||
'stats' => [
|
||||
'page_count' => count($pages),
|
||||
'page_block_count' => count($pageBlocks),
|
||||
'chunk_count' => count($chunks),
|
||||
'chunk_size' => $chunkSize,
|
||||
'chunk_overlap' => $chunkOverlap,
|
||||
],
|
||||
'queue' => [
|
||||
'ai_metadata_enqueued' => false,
|
||||
'needs_ai_metadata' => $needsAiMetadata,
|
||||
],
|
||||
],
|
||||
];
|
||||
}
|
||||
|
||||
public function persistSnapshot(array $import): void
|
||||
{
|
||||
$directory = runtime_path('proofdb/imports');
|
||||
if (!is_dir($directory) && !mkdir($directory, 0775, true) && !is_dir($directory)) {
|
||||
throw new RuntimeException("Unable to create directory: {$directory}");
|
||||
}
|
||||
|
||||
$path = $directory . DIRECTORY_SEPARATOR . $import['import_uid'] . '.json';
|
||||
$json = json_encode($import, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR);
|
||||
|
||||
if (file_put_contents($path, $json) === false) {
|
||||
throw new RuntimeException("Unable to write import snapshot: {$path}");
|
||||
}
|
||||
}
|
||||
|
||||
private function validate(array $payload): array
|
||||
{
|
||||
$errors = [];
|
||||
|
||||
if (isset($payload['file_error'])) {
|
||||
$errors['file'][] = 'uploaded file is invalid.';
|
||||
}
|
||||
|
||||
if (!isset($payload['title']) || !is_string($payload['title']) || trim($payload['title']) === '') {
|
||||
$errors['title'][] = 'title is required.';
|
||||
}
|
||||
|
||||
if (!isset($payload['source']) || !is_string($payload['source']) || trim($payload['source']) === '') {
|
||||
$errors['source'][] = 'source is required.';
|
||||
}
|
||||
|
||||
if (!isset($payload['content']) && !isset($payload['paragraphs']) && !isset($payload['pages'])) {
|
||||
$errors['content'][] = 'content, file, pages, or paragraphs is required.';
|
||||
}
|
||||
|
||||
if (isset($payload['paragraphs']) && !is_array($payload['paragraphs'])) {
|
||||
$errors['paragraphs'][] = 'paragraphs must be an array.';
|
||||
}
|
||||
|
||||
if (isset($payload['pages']) && !is_array($payload['pages'])) {
|
||||
$errors['pages'][] = 'pages must be an array.';
|
||||
}
|
||||
|
||||
if (isset($payload['metadata']) && !is_array($payload['metadata'])) {
|
||||
$errors['metadata'][] = 'metadata must be an object.';
|
||||
}
|
||||
|
||||
$chunkSize = $this->intOption($payload, 'chunk_size', self::DEFAULT_CHUNK_SIZE);
|
||||
if ($chunkSize < self::MIN_CHUNK_SIZE || $chunkSize > self::MAX_CHUNK_SIZE) {
|
||||
$errors['chunk_size'][] = 'chunk_size must be between 100 and 4000.';
|
||||
}
|
||||
|
||||
$chunkOverlap = $this->intOption($payload, 'chunk_overlap', self::DEFAULT_CHUNK_OVERLAP);
|
||||
if ($chunkOverlap < 0 || $chunkOverlap >= $chunkSize) {
|
||||
$errors['chunk_overlap'][] = 'chunk_overlap must be greater than or equal to 0 and less than chunk_size.';
|
||||
}
|
||||
|
||||
if (!isset($errors['paragraphs']) && isset($payload['paragraphs'])) {
|
||||
$hasContent = false;
|
||||
foreach ($payload['paragraphs'] as $index => $paragraph) {
|
||||
$content = is_array($paragraph) ? ($paragraph['content'] ?? '') : $paragraph;
|
||||
if (!is_string($content)) {
|
||||
$errors["paragraphs.{$index}.content"][] = 'paragraph content must be a string.';
|
||||
continue;
|
||||
}
|
||||
if (trim($content) !== '') {
|
||||
$hasContent = true;
|
||||
}
|
||||
}
|
||||
if (!$hasContent) {
|
||||
$errors['paragraphs'][] = 'paragraphs must contain at least one non-empty paragraph.';
|
||||
}
|
||||
}
|
||||
|
||||
if (!isset($errors['pages']) && isset($payload['pages'])) {
|
||||
$hasContent = false;
|
||||
foreach ($payload['pages'] as $index => $page) {
|
||||
if (!is_array($page)) {
|
||||
$errors["pages.{$index}"][] = 'page must be an object.';
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!$this->hasPageNumber($page)) {
|
||||
$errors["pages.{$index}.page_number"][] = 'page_number is required.';
|
||||
}
|
||||
|
||||
if (!isset($page['content']) || !is_string($page['content'])) {
|
||||
$errors["pages.{$index}.content"][] = 'page content must be a string.';
|
||||
continue;
|
||||
}
|
||||
|
||||
if (trim($page['content']) !== '') {
|
||||
$hasContent = true;
|
||||
}
|
||||
|
||||
if (isset($page['metadata']) && !is_array($page['metadata'])) {
|
||||
$errors["pages.{$index}.metadata"][] = 'page metadata must be an object.';
|
||||
}
|
||||
}
|
||||
|
||||
if (!$hasContent) {
|
||||
$errors['pages'][] = 'pages must contain at least one non-empty page.';
|
||||
}
|
||||
}
|
||||
|
||||
if (isset($payload['content']) && (!is_string($payload['content']) || trim($payload['content']) === '')) {
|
||||
$errors['content'][] = 'content must be a non-empty string.';
|
||||
}
|
||||
|
||||
return $errors;
|
||||
}
|
||||
|
||||
private function archive(array $payload, string $archiveUid): array
|
||||
{
|
||||
$title = $this->clean($payload['title']);
|
||||
$source = $this->clean($payload['source']);
|
||||
|
||||
return [
|
||||
'archive_uid' => $archiveUid,
|
||||
'title' => $title,
|
||||
'year' => isset($payload['year']) ? (int) $payload['year'] : null,
|
||||
'author' => $this->nullableClean($payload['author'] ?? null),
|
||||
'source' => $source,
|
||||
'series' => $this->nullableClean($payload['series'] ?? null),
|
||||
'tags' => is_array($payload['tags'] ?? null) ? array_values($payload['tags']) : [],
|
||||
'summary' => $this->nullableClean($payload['summary'] ?? null),
|
||||
'metadata' => $payload['metadata'] ?? [],
|
||||
'content' => $this->nullableClean($payload['content_url'] ?? $payload['content_path'] ?? null),
|
||||
'raw' => $this->nullableClean($payload['raw_url'] ?? $payload['raw_path'] ?? null),
|
||||
];
|
||||
}
|
||||
|
||||
private function pageBlocks(array $payload): array
|
||||
{
|
||||
if (isset($payload['pages'])) {
|
||||
return $this->pageBlocksFromPages($payload);
|
||||
}
|
||||
|
||||
if (isset($payload['paragraphs'])) {
|
||||
return $this->pageBlocksFromItems($payload, $payload['paragraphs']);
|
||||
}
|
||||
|
||||
return $this->pageBlocksFromItems($payload, preg_split('/\R{2,}/u', $payload['content']));
|
||||
}
|
||||
|
||||
private function pageBlocksFromPages(array $payload): array
|
||||
{
|
||||
$pageBlocks = [];
|
||||
|
||||
foreach ($payload['pages'] as $pageIndex => $page) {
|
||||
$pageNumber = $this->pageNumber($page);
|
||||
$pageMetadata = $page['metadata'] ?? [];
|
||||
$items = $this->markdownBlocksFromPage($page['content']);
|
||||
|
||||
foreach ($items as $itemIndex => $content) {
|
||||
$pageBlock = $this->pageBlock($payload, $content, count($pageBlocks), $itemIndex, $pageNumber, [
|
||||
'page_index' => $pageIndex,
|
||||
'page_metadata' => $pageMetadata,
|
||||
]);
|
||||
|
||||
if ($pageBlock !== null) {
|
||||
$pageBlocks[] = $pageBlock;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $pageBlocks;
|
||||
}
|
||||
|
||||
private function chunksFromPages(string $archiveUid, array $pageBlocks, int $chunkSize, int $chunkOverlap): array
|
||||
{
|
||||
$chunks = [];
|
||||
$chunkIndex = 1;
|
||||
|
||||
foreach ($this->groupBlocksByPage($pageBlocks) as $pageNumber => $blocks) {
|
||||
$units = [];
|
||||
foreach ($blocks as $block) {
|
||||
$unit = $this->cleanEmbeddingText($block['content']);
|
||||
if ($unit === '' || $this->isNoiseBlock($unit)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$units[] = $unit;
|
||||
}
|
||||
|
||||
foreach ($this->packUnitsForEmbedding($units, $chunkSize, $chunkOverlap) as $text) {
|
||||
$page = $this->restorePageNumber($pageNumber);
|
||||
$chunks[] = $this->chunk($archiveUid, $chunkIndex, $page, $text);
|
||||
$chunkIndex++;
|
||||
}
|
||||
}
|
||||
|
||||
return $chunks;
|
||||
}
|
||||
|
||||
private function groupBlocksByPage(array $pageBlocks): array
|
||||
{
|
||||
$pages = [];
|
||||
foreach ($pageBlocks as $block) {
|
||||
$key = $block['page_number'] === null ? '' : (string) $block['page_number'];
|
||||
$pages[$key][] = $block;
|
||||
}
|
||||
|
||||
return $pages;
|
||||
}
|
||||
|
||||
private function pagesSummary(array $pageBlocks, array $chunks): array
|
||||
{
|
||||
$pages = [];
|
||||
|
||||
foreach ($this->groupBlocksByPage($pageBlocks) as $pageNumber => $blocks) {
|
||||
$page = $this->restorePageNumber($pageNumber);
|
||||
$pageChunks = array_values(array_filter($chunks, fn (array $chunk): bool => $chunk['page_start'] === $page));
|
||||
$contentLength = array_sum(array_map(fn (array $block): int => mb_strlen($block['content']), $blocks));
|
||||
|
||||
$pages[] = [
|
||||
'page_number' => $page,
|
||||
'block_count' => count($blocks),
|
||||
'chunk_count' => count($pageChunks),
|
||||
'content_length' => $contentLength,
|
||||
'chunk_uids' => array_column($pageChunks, 'chunk_uid'),
|
||||
];
|
||||
}
|
||||
|
||||
return $pages;
|
||||
}
|
||||
|
||||
private function packUnitsForEmbedding(array $units, int $chunkSize, int $chunkOverlap): array
|
||||
{
|
||||
$chunks = [];
|
||||
$current = '';
|
||||
|
||||
foreach ($units as $unit) {
|
||||
$unit = $this->clean($unit);
|
||||
if ($unit === '') {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (mb_strlen($unit) > $chunkSize) {
|
||||
if ($current !== '') {
|
||||
$chunks[] = $current;
|
||||
$current = '';
|
||||
}
|
||||
|
||||
array_push($chunks, ...$this->chunkLongUnit($unit, $chunkSize, $chunkOverlap));
|
||||
continue;
|
||||
}
|
||||
|
||||
$candidate = $current === '' ? $unit : $current . "\n\n" . $unit;
|
||||
if (mb_strlen($candidate) <= $chunkSize) {
|
||||
$current = $candidate;
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($current !== '') {
|
||||
$chunks[] = $current;
|
||||
}
|
||||
$current = $unit;
|
||||
}
|
||||
|
||||
if ($current !== '') {
|
||||
$chunks[] = $current;
|
||||
}
|
||||
|
||||
return $chunks;
|
||||
}
|
||||
|
||||
private function chunk(string $archiveUid, int $chunkIndex, int|string|null $pageNumber, string $text): array
|
||||
{
|
||||
$chunkUid = $this->chunkUid($archiveUid, $chunkIndex, implode('|', [
|
||||
$chunkIndex,
|
||||
(string) ($pageNumber ?? ''),
|
||||
$text,
|
||||
]));
|
||||
|
||||
return [
|
||||
'chunk_uid' => $chunkUid,
|
||||
'chunk_index' => $chunkIndex,
|
||||
'page_start' => $pageNumber,
|
||||
'page_end' => $pageNumber,
|
||||
'pages' => $pageNumber === null ? [] : [$pageNumber],
|
||||
'text' => $text,
|
||||
'length' => mb_strlen($text),
|
||||
'embedding_ref' => null,
|
||||
];
|
||||
}
|
||||
|
||||
private function normalizePayload(array $payload): array
|
||||
{
|
||||
if (isset($payload['content']) && is_string($payload['content']) && !isset($payload['pages']) && !isset($payload['paragraphs'])) {
|
||||
$payload['pages'] = $this->pagesFromMarkdown($payload['content']);
|
||||
}
|
||||
|
||||
if (!isset($payload['source']) || trim((string) $payload['source']) === '') {
|
||||
$payload['source'] = 'raw-markdown';
|
||||
}
|
||||
|
||||
return $payload;
|
||||
}
|
||||
|
||||
private function applyMetadataFallbacks(array $payload): array
|
||||
{
|
||||
if ((!isset($payload['title']) || trim((string) $payload['title']) === '') && isset($payload['content']) && is_string($payload['content'])) {
|
||||
$payload['title'] = $this->inferTitle($payload['content'], (string) ($payload['source'] ?? ''));
|
||||
$payload['metadata'] = is_array($payload['metadata'] ?? null) ? $payload['metadata'] : [];
|
||||
$payload['metadata']['title_source'] = 'fallback';
|
||||
}
|
||||
|
||||
if (isset($payload['tags']) && is_string($payload['tags'])) {
|
||||
$payload['tags'] = $this->tagsFromString($payload['tags']);
|
||||
}
|
||||
$payload['tags'] = is_array($payload['tags'] ?? null) ? $payload['tags'] : [];
|
||||
|
||||
if (isset($payload['year']) && is_numeric($payload['year'])) {
|
||||
$payload['year'] = (int) $payload['year'];
|
||||
}
|
||||
|
||||
$payload['metadata'] = is_array($payload['metadata'] ?? null) ? $payload['metadata'] : [];
|
||||
|
||||
return $payload;
|
||||
}
|
||||
|
||||
private function pagesFromMarkdown(string $markdown): array
|
||||
{
|
||||
preg_match_all(self::MARKDOWN_PAGE_PATTERN, $markdown, $matches, PREG_SET_ORDER);
|
||||
if ($matches === []) {
|
||||
return [[
|
||||
'page_number' => 1,
|
||||
'content' => $this->cleanMarkdownPage($markdown),
|
||||
'metadata' => ['parser' => 'markdown_single_page'],
|
||||
]];
|
||||
}
|
||||
|
||||
$pages = [];
|
||||
foreach ($matches as $index => $match) {
|
||||
$pageNumber = ctype_digit($match[1]) ? (int) $match[1] : $match[1];
|
||||
$pages[] = [
|
||||
'page_number' => $pageNumber,
|
||||
'content' => $this->cleanMarkdownPage($match[2]),
|
||||
'metadata' => [
|
||||
'parser' => 'docmaster_markdown',
|
||||
'page_index' => $index,
|
||||
],
|
||||
];
|
||||
}
|
||||
|
||||
return $pages;
|
||||
}
|
||||
|
||||
private function markdownBlocksFromPage(string $content): array
|
||||
{
|
||||
$content = $this->cleanMarkdownPage($content);
|
||||
$blocks = preg_split('/\R{2,}/u', $content, -1, PREG_SPLIT_NO_EMPTY);
|
||||
if ($blocks === false) {
|
||||
return [$content];
|
||||
}
|
||||
|
||||
$records = [];
|
||||
foreach ($blocks as $block) {
|
||||
$block = $this->cleanMarkdownBlock($block);
|
||||
if ($block === '') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$lastIndex = count($records) - 1;
|
||||
if ($lastIndex >= 0 && $this->isCommentBlock($block) && $this->isPolicyRecordBlock($records[$lastIndex])) {
|
||||
$records[$lastIndex] .= "\n" . $block;
|
||||
continue;
|
||||
}
|
||||
|
||||
$records[] = $block;
|
||||
}
|
||||
|
||||
return $records;
|
||||
}
|
||||
|
||||
private function cleanMarkdownPage(string $content): string
|
||||
{
|
||||
$content = preg_replace('/<!--\s*DOCMASTER:PAGE\s+[^>]+-->/iu', '', $content) ?? $content;
|
||||
$content = preg_replace('/^\s*#+\s*Page\s+\S+\s*$/imu', '', $content) ?? $content;
|
||||
$content = preg_replace('/^\s*---+\s*$/mu', '', $content) ?? $content;
|
||||
$content = html_entity_decode($content, ENT_QUOTES | ENT_HTML5, 'UTF-8');
|
||||
|
||||
return trim($content);
|
||||
}
|
||||
|
||||
private function cleanMarkdownBlock(string $block): string
|
||||
{
|
||||
$block = preg_replace('/[ \t]+/u', ' ', $block) ?? $block;
|
||||
$block = preg_replace('/\R[ \t]+/u', "\n", $block) ?? $block;
|
||||
return trim($block);
|
||||
}
|
||||
|
||||
private function isCommentBlock(string $block): bool
|
||||
{
|
||||
return (bool) preg_match('/^\s*(?:[*#\s_~`>-])*COMMENT\b/iu', $this->plainBlock($block));
|
||||
}
|
||||
|
||||
private function isPolicyRecordBlock(string $block): bool
|
||||
{
|
||||
return (bool) preg_match('/^\s*(?:NSAM|NSDM|NSDD|NSD|PD)\s+\d+/iu', $this->plainBlock($block));
|
||||
}
|
||||
|
||||
private function isNoiseBlock(string $block): bool
|
||||
{
|
||||
$plain = strtoupper($this->plainBlock($block));
|
||||
$plain = preg_replace('/\s+/u', ' ', $plain) ?? $plain;
|
||||
|
||||
if ($plain === '') {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (preg_match('/^\d{1,6}$/', $plain)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
$patterns = [
|
||||
'/^(?:# )?UNCLASSIFIED$/',
|
||||
'/^TOP SECRET$/',
|
||||
'/^UNCLASSIFIED WITH TOP SECRET ATTACHMENTS$/',
|
||||
'/^DECLASSIFY ON: OADR$/',
|
||||
'/^\*? ?UNCLASSIFIED\*?$/',
|
||||
];
|
||||
|
||||
foreach ($patterns as $pattern) {
|
||||
if (preg_match($pattern, $plain)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private function cleanEmbeddingText(string $text): string
|
||||
{
|
||||
$lines = preg_split('/\R/u', $text);
|
||||
if ($lines === false) {
|
||||
return $this->cleanMarkdownBlock($text);
|
||||
}
|
||||
|
||||
$kept = [];
|
||||
foreach ($lines as $line) {
|
||||
$line = trim($line);
|
||||
if ($line === '' || $this->isNoiseLine($line)) {
|
||||
continue;
|
||||
}
|
||||
$kept[] = $line;
|
||||
}
|
||||
|
||||
return $this->cleanMarkdownBlock(implode("\n", $kept));
|
||||
}
|
||||
|
||||
private function isNoiseLine(string $line): bool
|
||||
{
|
||||
$plain = strtoupper($this->plainBlock($line));
|
||||
$plain = preg_replace('/\s+/u', ' ', $plain) ?? $plain;
|
||||
|
||||
if ($plain === '' || preg_match('/^\d{1,6}$/', $plain)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
$patterns = [
|
||||
'/^(?:# )?UNCLASSIFIED$/',
|
||||
'/^TOP SECRET$/',
|
||||
'/^UNCLASSIFIED WITH TOP SECRET ATTACHMENTS$/',
|
||||
'/^DECLASSIFY ON: OADR$/',
|
||||
'/^PARTIALLY DECLASSIFIED\/RELEASED ON .+$/',
|
||||
'/^UNDER PROVISIONS OF .+$/',
|
||||
'/^BY .+ NATIONAL SECURITY COUNCIL$/',
|
||||
'/^F \d{2}-\d+$/',
|
||||
];
|
||||
|
||||
foreach ($patterns as $pattern) {
|
||||
if (preg_match($pattern, $plain)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private function plainBlock(string $block): string
|
||||
{
|
||||
$block = str_replace(['*', '_', '`', '~'], '', $block);
|
||||
$block = preg_replace('/\s+/u', ' ', $block) ?? $block;
|
||||
return trim($block);
|
||||
}
|
||||
|
||||
private function inferTitle(string $markdown, string $source): string
|
||||
{
|
||||
if (preg_match_all('/^\s*#+\s*(.+?)\s*$/imu', $markdown, $matches)) {
|
||||
foreach ($matches[1] as $heading) {
|
||||
$heading = $this->clean($heading);
|
||||
if (!preg_match('/^Page\s+\S+$/iu', $heading) && !$this->isNoiseLine($heading)) {
|
||||
return $heading;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($source !== '') {
|
||||
return pathinfo($source, PATHINFO_FILENAME) ?: $source;
|
||||
}
|
||||
|
||||
return 'Untitled Markdown Import';
|
||||
}
|
||||
|
||||
private function pageBlocksFromItems(array $payload, array $items): array
|
||||
{
|
||||
$pageBlocks = [];
|
||||
|
||||
foreach ($items as $index => $item) {
|
||||
$content = is_array($item) ? ($item['content'] ?? '') : $item;
|
||||
$pageNumber = is_array($item) && $this->hasPageNumber($item) ? $this->pageNumber($item) : null;
|
||||
$metadata = is_array($item) ? ($item['metadata'] ?? []) : [];
|
||||
$pageBlock = $this->pageBlock($payload, $content, count($pageBlocks), $index, $pageNumber, $metadata);
|
||||
|
||||
if ($pageBlock !== null) {
|
||||
$pageBlocks[] = $pageBlock;
|
||||
}
|
||||
}
|
||||
|
||||
return $pageBlocks;
|
||||
}
|
||||
|
||||
private function pageBlock(
|
||||
array $payload,
|
||||
mixed $content,
|
||||
int $blockIndex,
|
||||
int $sourceIndex,
|
||||
int|string|null $pageNumber,
|
||||
array $metadata
|
||||
): ?array {
|
||||
$content = $this->clean((string) $content);
|
||||
if ($content === '') {
|
||||
return null;
|
||||
}
|
||||
|
||||
return [
|
||||
'block_uid' => $this->uid('block', implode('|', [
|
||||
$payload['source'],
|
||||
$payload['title'],
|
||||
(string) $pageNumber,
|
||||
$sourceIndex,
|
||||
$content,
|
||||
])),
|
||||
'index' => $blockIndex,
|
||||
'page_number' => $pageNumber,
|
||||
'content' => $content,
|
||||
'metadata' => $metadata,
|
||||
];
|
||||
}
|
||||
|
||||
private function chunkLongUnit(string $text, int $chunkSize, int $chunkOverlap): array
|
||||
{
|
||||
if (mb_strlen($text) <= $chunkSize) {
|
||||
return [$text];
|
||||
}
|
||||
|
||||
$chunks = [];
|
||||
$current = '';
|
||||
|
||||
foreach ($this->semanticUnits($text) as $unit) {
|
||||
if (mb_strlen($unit) > $chunkSize) {
|
||||
if ($current !== '') {
|
||||
$chunks[] = $current;
|
||||
$current = '';
|
||||
}
|
||||
|
||||
array_push($chunks, ...$this->hardChunk($unit, $chunkSize, $chunkOverlap));
|
||||
continue;
|
||||
}
|
||||
|
||||
$candidate = $current === '' ? $unit : $current . ' ' . $unit;
|
||||
if (mb_strlen($candidate) <= $chunkSize) {
|
||||
$current = $candidate;
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($current !== '') {
|
||||
$chunks[] = $current;
|
||||
}
|
||||
$current = $unit;
|
||||
}
|
||||
|
||||
if ($current !== '') {
|
||||
$chunks[] = $current;
|
||||
}
|
||||
|
||||
return $chunks === [] ? [$text] : $chunks;
|
||||
}
|
||||
|
||||
private function semanticUnits(string $text): array
|
||||
{
|
||||
$units = preg_split(self::SENTENCE_BOUNDARY_PATTERN, $text, -1, PREG_SPLIT_NO_EMPTY);
|
||||
if ($units === false || $units === []) {
|
||||
return [$text];
|
||||
}
|
||||
|
||||
return array_values(array_filter(array_map(fn (string $unit): string => $this->clean($unit), $units)));
|
||||
}
|
||||
|
||||
private function hardChunk(string $text, int $chunkSize, int $chunkOverlap): array
|
||||
{
|
||||
$length = mb_strlen($text);
|
||||
if ($length <= $chunkSize) {
|
||||
return [$text];
|
||||
}
|
||||
|
||||
$chunks = [];
|
||||
$start = 0;
|
||||
while ($start < $length) {
|
||||
$chunk = mb_substr($text, $start, $chunkSize);
|
||||
if ($chunk === '') {
|
||||
break;
|
||||
}
|
||||
|
||||
$chunks[] = $chunk;
|
||||
if ($start + $chunkSize >= $length) {
|
||||
break;
|
||||
}
|
||||
|
||||
$start += $chunkSize - $chunkOverlap;
|
||||
}
|
||||
|
||||
return $chunks;
|
||||
}
|
||||
|
||||
private function hasPageNumber(array $item): bool
|
||||
{
|
||||
return array_key_exists('page_number', $item)
|
||||
|| array_key_exists('page', $item)
|
||||
|| array_key_exists('number', $item);
|
||||
}
|
||||
|
||||
private function pageNumber(array $item): int|string
|
||||
{
|
||||
$pageNumber = $item['page_number'] ?? $item['page'] ?? $item['number'];
|
||||
return is_int($pageNumber) ? $pageNumber : $this->clean((string) $pageNumber);
|
||||
}
|
||||
|
||||
private function restorePageNumber(string $pageNumber): int|string|null
|
||||
{
|
||||
if ($pageNumber === '') {
|
||||
return null;
|
||||
}
|
||||
|
||||
return ctype_digit($pageNumber) ? (int) $pageNumber : $pageNumber;
|
||||
}
|
||||
|
||||
private function intOption(array $payload, string $key, int $default): int
|
||||
{
|
||||
if (!isset($payload[$key]) || $payload[$key] === '') {
|
||||
return $default;
|
||||
}
|
||||
|
||||
return (int) $payload[$key];
|
||||
}
|
||||
|
||||
private function clean(string $value): string
|
||||
{
|
||||
return trim(preg_replace('/[ \t]+/u', ' ', $value) ?? $value);
|
||||
}
|
||||
|
||||
private function nullableClean(mixed $value): ?string
|
||||
{
|
||||
if (!is_string($value)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$value = $this->clean($value);
|
||||
return $value === '' ? null : $value;
|
||||
}
|
||||
|
||||
private function tagsFromString(string $value): array
|
||||
{
|
||||
$value = trim($value);
|
||||
if ($value === '') {
|
||||
return [];
|
||||
}
|
||||
|
||||
$decoded = json_decode($value, true);
|
||||
if (is_array($decoded)) {
|
||||
return array_values(array_filter(array_map('strval', $decoded)));
|
||||
}
|
||||
|
||||
return array_values(array_filter(array_map('trim', preg_split('/[,，]/u', $value) ?: [])));
|
||||
}
|
||||
|
||||
private function archiveUid(array $payload): string
|
||||
{
|
||||
if (isset($payload['archive_uid']) && is_string($payload['archive_uid']) && $this->isUlid($payload['archive_uid'])) {
|
||||
return strtoupper($payload['archive_uid']);
|
||||
}
|
||||
|
||||
return (string) new Ulid();
|
||||
}
|
||||
|
||||
private function chunkUid(string $archiveUid, int $chunkIndex, string $value): string
|
||||
{
|
||||
return $archiveUid . '_' . $chunkIndex . '_' . $this->shortUid($value);
|
||||
}
|
||||
|
||||
private function shortUid(string $value): string
|
||||
{
|
||||
$number = hexdec(substr(hash('crc32b', $value), 0, 8)) % 100000;
|
||||
return str_pad((string) $number, 5, '0', STR_PAD_LEFT);
|
||||
}
|
||||
|
||||
private function isUlid(string $value): bool
|
||||
{
|
||||
return (bool) preg_match('/^[0-9A-HJKMNP-TV-Z]{26}$/', strtoupper($value));
|
||||
}
|
||||
|
||||
private function uid(string $prefix, string $value): string
|
||||
{
|
||||
return $prefix . '_' . substr(hash('sha256', $value), 0, 24);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
<?php
|
||||
|
||||
namespace app\service\LLM;
|
||||
|
||||
use RuntimeException;
|
||||
|
||||
class LLMRequestException extends RuntimeException
|
||||
{
|
||||
public function __construct(
|
||||
string $message,
|
||||
private readonly ?int $statusCode = null,
|
||||
private readonly ?string $providerCode = null,
|
||||
private readonly ?array $payload = null
|
||||
) {
|
||||
parent::__construct($message, $statusCode ?? 0);
|
||||
}
|
||||
|
||||
public function statusCode(): ?int
|
||||
{
|
||||
return $this->statusCode;
|
||||
}
|
||||
|
||||
public function providerCode(): ?string
|
||||
{
|
||||
return $this->providerCode;
|
||||
}
|
||||
|
||||
public function payload(): ?array
|
||||
{
|
||||
return $this->payload;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
<?php
|
||||
|
||||
namespace app\service\LLM;
|
||||
|
||||
use Throwable;
|
||||
|
||||
class LLMRetryQueue
|
||||
{
|
||||
public function run(callable $job, array $config = []): mixed
|
||||
{
|
||||
$enabled = (bool) ($config['enabled'] ?? true);
|
||||
$maxAttempts = max(1, (int) ($config['max_attempts'] ?? 3));
|
||||
$baseDelayMs = max(0, (int) ($config['base_delay_ms'] ?? 1500));
|
||||
$maxDelayMs = max($baseDelayMs, (int) ($config['max_delay_ms'] ?? 10000));
|
||||
$retryStatuses = array_map('intval', $config['retry_statuses'] ?? [429]);
|
||||
$retryErrorCodes = array_map('strval', $config['retry_error_codes'] ?? ['1302', '1303', '1304', '1305', '1306', '1307', '1308']);
|
||||
|
||||
$attempt = 0;
|
||||
while (true) {
|
||||
$attempt++;
|
||||
|
||||
try {
|
||||
return $job($attempt);
|
||||
} catch (Throwable $exception) {
|
||||
if (!$enabled || $attempt >= $maxAttempts || !$this->shouldRetry($exception, $retryStatuses, $retryErrorCodes)) {
|
||||
throw $exception;
|
||||
}
|
||||
|
||||
usleep($this->delayMs($attempt, $baseDelayMs, $maxDelayMs) * 1000);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private function shouldRetry(Throwable $exception, array $retryStatuses, array $retryErrorCodes): bool
|
||||
{
|
||||
if (!$exception instanceof LLMRequestException) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if ($exception->statusCode() !== null && in_array($exception->statusCode(), $retryStatuses, true)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return $exception->providerCode() !== null && in_array($exception->providerCode(), $retryErrorCodes, true);
|
||||
}
|
||||
|
||||
private function delayMs(int $attempt, int $baseDelayMs, int $maxDelayMs): int
|
||||
{
|
||||
$delay = min($maxDelayMs, $baseDelayMs * (2 ** max(0, $attempt - 1)));
|
||||
$jitter = $delay > 0 ? random_int(0, min(500, $delay)) : 0;
|
||||
|
||||
return $delay + $jitter;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
<?php
|
||||
|
||||
namespace app\service\LLM;
|
||||
|
||||
use GuzzleHttp\Client;
|
||||
use GuzzleHttp\Exception\RequestException;
|
||||
use RuntimeException;
|
||||
use Throwable;
|
||||
|
||||
class OpenAICompatibleClient
|
||||
{
|
||||
private Client $client;
|
||||
private array $config;
|
||||
|
||||
public function __construct(?array $config = null)
|
||||
{
|
||||
$this->config = $config ?? config('LLMapi.default', []);
|
||||
$baseUrl = rtrim((string) ($this->config['base_url'] ?? ''), '/');
|
||||
|
||||
$this->client = new Client([
|
||||
'base_uri' => $baseUrl . '/',
|
||||
'timeout' => (int) ($this->config['timeout'] ?? 60),
|
||||
'connect_timeout' => (int) ($this->config['connect_timeout'] ?? 10),
|
||||
]);
|
||||
}
|
||||
|
||||
public function isConfigured(): bool
|
||||
{
|
||||
return trim((string) ($this->config['api_key'] ?? '')) !== ''
|
||||
&& trim((string) ($this->config['base_url'] ?? '')) !== '';
|
||||
}
|
||||
|
||||
public function chatJson(array $messages, array $options = []): array
|
||||
{
|
||||
$content = $this->chat($messages, $options);
|
||||
$decoded = json_decode($this->extractJson($content), true);
|
||||
|
||||
if (!is_array($decoded)) {
|
||||
throw new RuntimeException('LLM response is not valid JSON.');
|
||||
}
|
||||
|
||||
return $decoded;
|
||||
}
|
||||
|
||||
public function chat(array $messages, array $options = []): string
|
||||
{
|
||||
if (!$this->isConfigured()) {
|
||||
throw new RuntimeException('LLM API is not configured.');
|
||||
}
|
||||
|
||||
$headers = [
|
||||
'Authorization' => 'Bearer ' . $this->config['api_key'],
|
||||
'Content-Type' => 'application/json',
|
||||
];
|
||||
|
||||
if (!empty($this->config['organization'])) {
|
||||
$headers['OpenAI-Organization'] = $this->config['organization'];
|
||||
}
|
||||
|
||||
if (!empty($this->config['project'])) {
|
||||
$headers['OpenAI-Project'] = $this->config['project'];
|
||||
}
|
||||
|
||||
$body = [
|
||||
'model' => $options['model'] ?? config('LLMapi.chat.model'),
|
||||
'messages' => $messages,
|
||||
'temperature' => $options['temperature'] ?? config('LLMapi.chat.temperature', 0.2),
|
||||
'max_tokens' => $options['max_tokens'] ?? config('LLMapi.chat.max_tokens', 1200),
|
||||
'stream' => (bool) ($options['stream'] ?? config('LLMapi.chat.stream', false)),
|
||||
];
|
||||
|
||||
if (array_key_exists('response_format', $options) && is_array($options['response_format'])) {
|
||||
$body['response_format'] = $options['response_format'];
|
||||
}
|
||||
|
||||
if (array_key_exists('thinking', $options) && is_array($options['thinking'])) {
|
||||
$body['thinking'] = $options['thinking'];
|
||||
}
|
||||
|
||||
if (array_key_exists('request_id', $options) && is_string($options['request_id'])) {
|
||||
$body['request_id'] = $options['request_id'];
|
||||
}
|
||||
|
||||
if (array_key_exists('user_id', $options) && is_string($options['user_id'])) {
|
||||
$body['user_id'] = $options['user_id'];
|
||||
}
|
||||
|
||||
try {
|
||||
$response = $this->client->post('chat/completions', [
|
||||
'headers' => $headers,
|
||||
'json' => $body,
|
||||
]);
|
||||
} catch (RequestException $exception) {
|
||||
throw $this->requestException($exception);
|
||||
} catch (Throwable $exception) {
|
||||
throw new RuntimeException('LLM chat request failed: ' . $exception->getMessage(), 0, $exception);
|
||||
}
|
||||
|
||||
$payload = json_decode((string) $response->getBody(), true);
|
||||
$content = $payload['choices'][0]['message']['content'] ?? null;
|
||||
|
||||
if (!is_string($content) || trim($content) === '') {
|
||||
throw new RuntimeException('LLM chat response is empty.');
|
||||
}
|
||||
|
||||
return $content;
|
||||
}
|
||||
|
||||
private function extractJson(string $content): string
|
||||
{
|
||||
$content = trim($content);
|
||||
$content = preg_replace('/^```(?:json)?\s*/i', '', $content) ?? $content;
|
||||
$content = preg_replace('/\s*```$/', '', $content) ?? $content;
|
||||
|
||||
$start = strpos($content, '{');
|
||||
$end = strrpos($content, '}');
|
||||
if ($start !== false && $end !== false && $end > $start) {
|
||||
return substr($content, $start, $end - $start + 1);
|
||||
}
|
||||
|
||||
return $content;
|
||||
}
|
||||
|
||||
private function requestException(RequestException $exception): LLMRequestException
|
||||
{
|
||||
$statusCode = $exception->getResponse()?->getStatusCode();
|
||||
$body = $exception->getResponse() ? (string) $exception->getResponse()->getBody() : '';
|
||||
$payload = json_decode($body, true);
|
||||
$providerCode = null;
|
||||
$providerMessage = null;
|
||||
|
||||
if (is_array($payload)) {
|
||||
$providerCode = isset($payload['error']['code']) ? (string) $payload['error']['code'] : null;
|
||||
$providerMessage = isset($payload['error']['message']) ? (string) $payload['error']['message'] : null;
|
||||
}
|
||||
|
||||
$message = 'LLM chat request failed';
|
||||
if ($statusCode !== null) {
|
||||
$message .= " with HTTP {$statusCode}";
|
||||
}
|
||||
if ($providerCode !== null) {
|
||||
$message .= " and provider code {$providerCode}";
|
||||
}
|
||||
if ($providerMessage !== null) {
|
||||
$message .= ": {$providerMessage}";
|
||||
} else {
|
||||
$message .= ': ' . $exception->getMessage();
|
||||
}
|
||||
|
||||
return new LLMRequestException($message, $statusCode, $providerCode, is_array($payload) ? $payload : null);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user