暂存
This commit is contained in:
@@ -37,9 +37,23 @@ class ArticleImportController
|
||||
try {
|
||||
$service->persistSnapshot($result['data']);
|
||||
(new \app\service\ArchiveRepository())->saveImport($result['data']);
|
||||
if (($result['data']['queue']['needs_ai_metadata'] ?? false) === true) {
|
||||
(new \app\service\AiMetadataQueue())->push($result['data']['archive']['archive_uid']);
|
||||
$result['data']['queue']['ai_metadata_enqueued'] = true;
|
||||
if (($result['data']['queue']['needs_ai_metadata'] ?? false) === true && (new \app\service\ArchiveMetadataEnrichmentService())->isEnabled()) {
|
||||
$archiveUid = $result['data']['archive']['archive_uid'];
|
||||
(new \app\service\ArchiveRepository())->markAiMetadataStatus($archiveUid, 'queued');
|
||||
|
||||
try {
|
||||
(new \app\service\AiMetadataQueue())->push($archiveUid);
|
||||
$result['data']['queue']['ai_metadata_enqueued'] = true;
|
||||
} catch (Throwable $queueException) {
|
||||
(new \app\service\ArchiveRepository())->markAiMetadataStatus($archiveUid, 'failed_retryable', $queueException->getMessage());
|
||||
$result['data']['queue']['ai_metadata_enqueue_error'] = $queueException->getMessage();
|
||||
}
|
||||
} elseif (($result['data']['queue']['needs_ai_metadata'] ?? false) === true) {
|
||||
(new \app\service\ArchiveRepository())->markAiMetadataStatus(
|
||||
$result['data']['archive']['archive_uid'],
|
||||
'disabled',
|
||||
'AI metadata enrichment is not enabled.'
|
||||
);
|
||||
}
|
||||
} catch (Throwable $exception) {
|
||||
return $this->jsonResponse([
|
||||
|
||||
@@ -24,6 +24,17 @@ class AiMetadata
|
||||
public function onWorkerStart(): void
|
||||
{
|
||||
Timer::add(10, fn (): int => $this->queue->releaseDueDelayed());
|
||||
Timer::add(max(30, (int) config('queue.ai_metadata.dispatcher_interval_seconds', 60)), function (): void {
|
||||
try {
|
||||
if (!$this->enrichment->isEnabled()) {
|
||||
return;
|
||||
}
|
||||
foreach ($this->archives->queuePendingAiMetadataArchives((int) config('queue.ai_metadata.dispatcher_batch_size', 20)) as $archiveUid) {
|
||||
$this->queue->push($archiveUid);
|
||||
}
|
||||
} catch (Throwable) {
|
||||
}
|
||||
});
|
||||
|
||||
while (true) {
|
||||
$this->queue->releaseDueDelayed();
|
||||
@@ -47,10 +58,18 @@ class AiMetadata
|
||||
}
|
||||
|
||||
if (!$this->archives->archiveNeedsMetadata($archive)) {
|
||||
$this->archives->markAiMetadataStatus($archiveUid, 'completed', null, ['attempted' => false]);
|
||||
$this->queue->clearRetry($archiveUid);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!$this->enrichment->isEnabled()) {
|
||||
$this->archives->markAiMetadataStatus($archiveUid, 'disabled', 'AI metadata enrichment is not enabled.');
|
||||
$this->queue->clearRetry($archiveUid);
|
||||
return;
|
||||
}
|
||||
|
||||
$this->archives->markAiMetadataStatus($archiveUid, 'processing');
|
||||
$payload = $archive;
|
||||
$payload['content'] = $this->archives->findChunksText($archiveUid);
|
||||
|
||||
@@ -58,7 +77,15 @@ class AiMetadata
|
||||
$aiMeta = $enriched['metadata']['ai_enrichment'] ?? [];
|
||||
|
||||
if (($aiMeta['attempted'] ?? false) !== true || ($aiMeta['error'] ?? null)) {
|
||||
$this->queue->retryLater($archiveUid, $aiMeta['error'] ?? 'AI metadata enrichment did not complete.');
|
||||
$retryable = ($aiMeta['enabled'] ?? true) === true && ($aiMeta['error'] ?? null) !== null;
|
||||
$status = $retryable ? 'failed_retryable' : 'failed_terminal';
|
||||
$error = $aiMeta['error'] ?? 'AI metadata enrichment did not complete.';
|
||||
$this->archives->markAiMetadataStatus($archiveUid, $status, $error, $aiMeta);
|
||||
if ($retryable) {
|
||||
$this->queue->retryLater($archiveUid, $error);
|
||||
} else {
|
||||
$this->queue->clearRetry($archiveUid);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -70,9 +97,16 @@ class AiMetadata
|
||||
}
|
||||
|
||||
$this->archives->updateMetadata($archiveUid, $fields, $aiMeta);
|
||||
$this->archives->markAiMetadataStatus($archiveUid, 'completed', null, $aiMeta);
|
||||
$this->queue->clearRetry($archiveUid);
|
||||
} catch (Throwable $exception) {
|
||||
$this->queue->retryLater($archiveUid, $exception->getMessage());
|
||||
$status = $this->archives->isAiMetadataRetryable($exception) ? 'failed_retryable' : 'failed_terminal';
|
||||
$this->archives->markAiMetadataStatus($archiveUid, $status, $exception->getMessage());
|
||||
if ($status === 'failed_retryable') {
|
||||
$this->queue->retryLater($archiveUid, $exception->getMessage());
|
||||
} else {
|
||||
$this->queue->clearRetry($archiveUid);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -72,7 +72,7 @@ class ArchiveAdminService
|
||||
}
|
||||
|
||||
$updates = [];
|
||||
foreach (['title', 'summary', 'author', 'source', 'series', 'content', 'raw'] as $field) {
|
||||
foreach (['title', 'summary', 'author', 'source', 'series'] as $field) {
|
||||
if (array_key_exists($field, $payload)) {
|
||||
$updates[$field] = $this->nullableText($payload[$field]);
|
||||
}
|
||||
@@ -134,8 +134,6 @@ class ArchiveAdminService
|
||||
{
|
||||
$data = $this->listItem($row);
|
||||
$data['metadata'] = $this->decodeJson($row->metadata ?? null, []);
|
||||
$data['content'] = $row->content;
|
||||
$data['raw'] = $row->raw;
|
||||
$data['chunks'] = $this->decodeJson($row->chunks ?? null, []);
|
||||
|
||||
return $data;
|
||||
|
||||
@@ -119,6 +119,14 @@ class MaintenanceScriptService
|
||||
'doc_name' => 'setup_opensearch.md',
|
||||
'args_hint' => '无参数',
|
||||
],
|
||||
'reembed_chunks' => [
|
||||
'name' => 'reembed_chunks',
|
||||
'file' => 'reembed_chunks.php',
|
||||
'label' => '重新生成向量',
|
||||
'description' => '对 chunks 重新执行 embedding,支持 resume 与 --reset。',
|
||||
'doc_name' => 'reembed_chunks.md',
|
||||
'args_hint' => '--archive_uid=01... 或 --reset',
|
||||
],
|
||||
'reindex_opensearch' => [
|
||||
'name' => 'reindex_opensearch',
|
||||
'file' => 'reindex_opensearch.php',
|
||||
@@ -127,14 +135,6 @@ class MaintenanceScriptService
|
||||
'doc_name' => 'reindex_opensearch.md',
|
||||
'args_hint' => '--archive_uid=01...',
|
||||
],
|
||||
'backfill_archive_content' => [
|
||||
'name' => 'backfill_archive_content',
|
||||
'file' => 'backfill_archive_content.php',
|
||||
'label' => '回填 archive content',
|
||||
'description' => '从 raw 或 chunks 回填 archives.content。',
|
||||
'doc_name' => 'backfill_archive_content.md',
|
||||
'args_hint' => '--archive_uid=01...',
|
||||
],
|
||||
'setup_admin_users' => [
|
||||
'name' => 'setup_admin_users',
|
||||
'file' => 'setup_admin_users.php',
|
||||
|
||||
@@ -20,9 +20,9 @@ class ArchiveMetadataEnrichmentService
|
||||
public function enrich(array $payload): array
|
||||
{
|
||||
$missing = $this->missingFields($payload);
|
||||
if ($missing === [] || !$this->enabled()) {
|
||||
if ($missing === [] || !$this->isEnabled()) {
|
||||
return $this->withAiMeta($payload, [
|
||||
'enabled' => $this->enabled(),
|
||||
'enabled' => $this->isEnabled(),
|
||||
'attempted' => false,
|
||||
'filled' => [],
|
||||
'missing' => $missing,
|
||||
@@ -80,7 +80,7 @@ class ArchiveMetadataEnrichmentService
|
||||
return array_values(array_filter($fields, fn (string $field): bool => !$this->hasUsefulValue($payload, $field)));
|
||||
}
|
||||
|
||||
private function enabled(): bool
|
||||
public function isEnabled(): bool
|
||||
{
|
||||
return (bool) config('LLMapi.metadata.enabled', true) && $this->client->isConfigured();
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
namespace app\service;
|
||||
|
||||
use Throwable;
|
||||
use support\Db;
|
||||
|
||||
class ArchiveRepository
|
||||
@@ -24,8 +25,6 @@ class ArchiveRepository
|
||||
'series' => $archive['series'] ?? null,
|
||||
'tags' => json_encode($archive['tags'] ?? [], JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
|
||||
'metadata' => json_encode($archive['metadata'] ?? [], JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
|
||||
'content' => $archive['content'] ?? null,
|
||||
'raw' => $archive['raw'] ?? null,
|
||||
'chunks' => json_encode($chunkUids, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
|
||||
]
|
||||
);
|
||||
@@ -194,6 +193,79 @@ class ArchiveRepository
|
||||
Db::table('archives')->where('archive_uid', $archiveUid)->update($updates);
|
||||
}
|
||||
|
||||
public function queuePendingAiMetadataArchives(int $limit): array
|
||||
{
|
||||
$limit = max(1, $limit);
|
||||
$rows = Db::table('archives')
|
||||
->where(function ($query): void {
|
||||
$query
|
||||
->whereNull('title')
|
||||
->orWhere('title', '')
|
||||
->orWhereNull('summary')
|
||||
->orWhere('summary', '')
|
||||
->orWhereNull('author')
|
||||
->orWhere('author', '')
|
||||
->orWhereNull('year')
|
||||
->orWhere('year', '<=', 0)
|
||||
->orWhereRaw("tags = '[]'::jsonb")
|
||||
->orWhereRaw("metadata #>> '{title_source}' = 'fallback'");
|
||||
})
|
||||
->orderByDesc('updated_time')
|
||||
->limit(max($limit * 5, 50))
|
||||
->get()
|
||||
->all();
|
||||
|
||||
$archiveUids = [];
|
||||
foreach ($rows as $row) {
|
||||
$archive = $this->archiveToArray($row);
|
||||
if (!$this->archiveNeedsMetadata($archive) || !$this->shouldQueueAiMetadata($archive)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$archiveUids[] = (string) $archive['archive_uid'];
|
||||
$this->markAiMetadataStatus((string) $archive['archive_uid'], 'queued', null);
|
||||
|
||||
if (count($archiveUids) >= $limit) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return $archiveUids;
|
||||
}
|
||||
|
||||
public function markAiMetadataStatus(string $archiveUid, string $status, ?string $error = null, array $extra = []): void
|
||||
{
|
||||
$archive = $this->findArchive($archiveUid);
|
||||
if ($archive === null) {
|
||||
return;
|
||||
}
|
||||
|
||||
$metadata = is_array($archive['metadata'] ?? null) ? $archive['metadata'] : [];
|
||||
$aiMeta = is_array($metadata['ai_enrichment'] ?? null) ? $metadata['ai_enrichment'] : [];
|
||||
$attempt = ((int) ($aiMeta['attempt'] ?? 0)) + ($status === 'processing' ? 1 : 0);
|
||||
|
||||
$metadata['ai_enrichment'] = array_merge($aiMeta, $extra, [
|
||||
'status' => $status,
|
||||
'attempt' => $attempt,
|
||||
'updated_at' => date(DATE_ATOM),
|
||||
'error' => $error,
|
||||
]);
|
||||
|
||||
Db::table('archives')->where('archive_uid', $archiveUid)->update([
|
||||
'metadata' => json_encode($metadata, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
|
||||
]);
|
||||
}
|
||||
|
||||
public function isAiMetadataRetryable(Throwable $exception): bool
|
||||
{
|
||||
if (!$exception instanceof \app\service\LLM\LLMRequestException) {
|
||||
return true;
|
||||
}
|
||||
|
||||
$statusCode = $exception->statusCode();
|
||||
return $statusCode === null || $statusCode === 429 || $statusCode >= 500;
|
||||
}
|
||||
|
||||
public function archiveNeedsMetadata(array $archive): bool
|
||||
{
|
||||
foreach (['title', 'year', 'author', 'tags', 'summary'] as $field) {
|
||||
@@ -227,12 +299,29 @@ class ArchiveRepository
|
||||
'series' => $archive->series,
|
||||
'tags' => json_decode($archive->tags ?? '[]', true) ?: [],
|
||||
'metadata' => json_decode($archive->metadata ?? '{}', true) ?: [],
|
||||
'content' => $archive->content,
|
||||
'raw' => $archive->raw,
|
||||
'chunks' => json_decode($archive->chunks ?? '[]', true) ?: [],
|
||||
];
|
||||
}
|
||||
|
||||
private function shouldQueueAiMetadata(array $archive): bool
|
||||
{
|
||||
$aiMeta = is_array($archive['metadata']['ai_enrichment'] ?? null) ? $archive['metadata']['ai_enrichment'] : [];
|
||||
$status = (string) ($aiMeta['status'] ?? '');
|
||||
if ($status === 'failed_terminal') {
|
||||
return false;
|
||||
}
|
||||
|
||||
$updatedAt = strtotime((string) ($aiMeta['updated_at'] ?? '')) ?: 0;
|
||||
$staleAfter = max(60, (int) config('queue.ai_metadata.stale_after_seconds', 900));
|
||||
$isFresh = $updatedAt > 0 && (time() - $updatedAt) < $staleAfter;
|
||||
|
||||
if (in_array($status, ['queued', 'processing'], true) && $isFresh) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private function chunkRowToArray(object $row): array
|
||||
{
|
||||
return [
|
||||
|
||||
@@ -70,16 +70,6 @@ class ArticleImportService
|
||||
}
|
||||
}
|
||||
|
||||
public function normalizeArchiveContentString(string $content): ?string
|
||||
{
|
||||
return $this->nullableClean($this->cleanMarkdownPage($content));
|
||||
}
|
||||
|
||||
public function normalizeArchiveRawString(string $content): ?string
|
||||
{
|
||||
return $this->nullableClean($content);
|
||||
}
|
||||
|
||||
private function validate(array $payload): array
|
||||
{
|
||||
$errors = [];
|
||||
@@ -192,8 +182,6 @@ class ArticleImportService
|
||||
'tags' => is_array($payload['tags'] ?? null) ? array_values($payload['tags']) : [],
|
||||
'summary' => $this->nullableClean($payload['summary'] ?? null),
|
||||
'metadata' => $payload['metadata'] ?? [],
|
||||
'content' => $this->normalizedArchiveContent($payload),
|
||||
'raw' => $this->rawArchiveContent($payload),
|
||||
];
|
||||
}
|
||||
|
||||
@@ -210,57 +198,6 @@ class ArticleImportService
|
||||
return $this->pageBlocksFromItems($payload, preg_split('/\R{2,}/u', $payload['content']));
|
||||
}
|
||||
|
||||
private function normalizedArchiveContent(array $payload): ?string
|
||||
{
|
||||
if (isset($payload['pages']) && is_array($payload['pages'])) {
|
||||
$parts = [];
|
||||
foreach ($payload['pages'] as $page) {
|
||||
if (!is_array($page) || !isset($page['content']) || !is_string($page['content'])) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$content = $this->cleanMarkdownPage($page['content']);
|
||||
if ($content !== '') {
|
||||
$parts[] = $content;
|
||||
}
|
||||
}
|
||||
|
||||
return $this->nullableClean(implode("\n\n", $parts));
|
||||
}
|
||||
|
||||
if (isset($payload['paragraphs']) && is_array($payload['paragraphs'])) {
|
||||
$parts = [];
|
||||
foreach ($payload['paragraphs'] as $paragraph) {
|
||||
$content = is_array($paragraph) ? ($paragraph['content'] ?? '') : $paragraph;
|
||||
if (!is_string($content)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$content = $this->clean($content);
|
||||
if ($content !== '') {
|
||||
$parts[] = $content;
|
||||
}
|
||||
}
|
||||
|
||||
return $this->nullableClean(implode("\n\n", $parts));
|
||||
}
|
||||
|
||||
if (isset($payload['content']) && is_string($payload['content'])) {
|
||||
return $this->normalizeArchiveContentString($payload['content']);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private function rawArchiveContent(array $payload): ?string
|
||||
{
|
||||
if (isset($payload['content']) && is_string($payload['content'])) {
|
||||
return $this->normalizeArchiveRawString($payload['content']);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private function pageBlocksFromPages(array $payload): array
|
||||
{
|
||||
$pageBlocks = [];
|
||||
|
||||
@@ -22,24 +22,29 @@ class ChunkEmbeddingHandler
|
||||
$this->retryQueue = $retryQueue ?? new LLMRetryQueue();
|
||||
}
|
||||
|
||||
public function handle(array $task): void
|
||||
public function handle(array $task): int
|
||||
{
|
||||
if (($task['target_type'] ?? null) !== 'archive') {
|
||||
return;
|
||||
return 0;
|
||||
}
|
||||
|
||||
$archiveUid = trim((string) ($task['target_uid'] ?? ''));
|
||||
if ($archiveUid === '') {
|
||||
return;
|
||||
return 0;
|
||||
}
|
||||
|
||||
$batchSize = (int) config('LLMapi.embedding.batch_size', 32);
|
||||
$chunks = $this->chunks->findQueuedChunks($archiveUid, $batchSize);
|
||||
if ($chunks === []) {
|
||||
return;
|
||||
return 0;
|
||||
}
|
||||
|
||||
$chunkUids = array_column($chunks, 'chunk_uid');
|
||||
if (!$this->client->isConfigured()) {
|
||||
$this->chunks->markFailed($chunkUids, 'BigModel embedding API is not configured.', false);
|
||||
return count($chunkUids);
|
||||
}
|
||||
|
||||
$this->chunks->markProcessing($chunkUids);
|
||||
|
||||
try {
|
||||
@@ -52,6 +57,7 @@ class ChunkEmbeddingHandler
|
||||
);
|
||||
|
||||
$this->persistEmbeddings($chunks, $payload);
|
||||
return count($chunkUids);
|
||||
} catch (Throwable $exception) {
|
||||
$this->chunks->markFailed($chunkUids, $exception->getMessage(), $this->isRetryable($exception));
|
||||
throw $exception;
|
||||
|
||||
@@ -6,11 +6,70 @@ use support\Db;
|
||||
|
||||
class ChunkEmbeddingRepository
|
||||
{
|
||||
public function countChunks(?string $archiveUid = null): int
|
||||
{
|
||||
$query = Db::table('chunks');
|
||||
|
||||
if ($archiveUid !== null && trim($archiveUid) !== '') {
|
||||
$query->where('archive_uid', trim($archiveUid));
|
||||
}
|
||||
|
||||
return (int) $query->count();
|
||||
}
|
||||
|
||||
public function countChunksByStatuses(array $statuses, ?string $archiveUid = null): int
|
||||
{
|
||||
$query = Db::table('chunks')->whereIn('embedding_status', $statuses);
|
||||
|
||||
if ($archiveUid !== null && trim($archiveUid) !== '') {
|
||||
$query->where('archive_uid', trim($archiveUid));
|
||||
}
|
||||
|
||||
return (int) $query->count();
|
||||
}
|
||||
|
||||
public function resetAllChunksToPending(?string $archiveUid = null): int
|
||||
{
|
||||
$query = Db::table('chunks');
|
||||
|
||||
if ($archiveUid !== null && trim($archiveUid) !== '') {
|
||||
$query->where('archive_uid', trim($archiveUid));
|
||||
}
|
||||
|
||||
return $query->update($this->pendingResetPayload());
|
||||
}
|
||||
|
||||
public function resetRecoverableChunksToPending(?string $archiveUid = null): int
|
||||
{
|
||||
$query = Db::table('chunks')
|
||||
->whereIn('embedding_status', [
|
||||
EmbeddingStatus::QUEUED,
|
||||
EmbeddingStatus::PROCESSING,
|
||||
EmbeddingStatus::FAILED_RETRYABLE,
|
||||
]);
|
||||
|
||||
if ($archiveUid !== null && trim($archiveUid) !== '') {
|
||||
$query->where('archive_uid', trim($archiveUid));
|
||||
}
|
||||
|
||||
return $query->update($this->pendingResetPayload());
|
||||
}
|
||||
|
||||
public function queuePendingArchiveTasks(int $limit): array
|
||||
{
|
||||
$statuses = [EmbeddingStatus::PENDING, EmbeddingStatus::QUEUED, EmbeddingStatus::FAILED_RETRYABLE];
|
||||
$staleBefore = date('Y-m-d H:i:s', time() - max(60, (int) config('queue.tasks.stale_after_seconds', 900)));
|
||||
$archiveUids = Db::table('chunks')
|
||||
->whereIn('embedding_status', $statuses)
|
||||
->where(function ($query) use ($staleBefore): void {
|
||||
$query
|
||||
->whereIn('embedding_status', [EmbeddingStatus::PENDING, EmbeddingStatus::FAILED_RETRYABLE])
|
||||
->orWhere(function ($stale) use ($staleBefore): void {
|
||||
$stale
|
||||
->whereIn('embedding_status', [EmbeddingStatus::QUEUED, EmbeddingStatus::PROCESSING])
|
||||
->where(function ($time) use ($staleBefore): void {
|
||||
$time->whereNull('embedding_updated_at')->orWhere('embedding_updated_at', '<', $staleBefore);
|
||||
});
|
||||
});
|
||||
})
|
||||
->select('archive_uid')
|
||||
->groupBy('archive_uid')
|
||||
->orderByRaw('MIN(id)')
|
||||
@@ -22,7 +81,17 @@ class ChunkEmbeddingRepository
|
||||
foreach ($archiveUids as $archiveUid) {
|
||||
Db::table('chunks')
|
||||
->where('archive_uid', $archiveUid)
|
||||
->whereIn('embedding_status', $statuses)
|
||||
->where(function ($query) use ($staleBefore): void {
|
||||
$query
|
||||
->whereIn('embedding_status', [EmbeddingStatus::PENDING, EmbeddingStatus::FAILED_RETRYABLE])
|
||||
->orWhere(function ($stale) use ($staleBefore): void {
|
||||
$stale
|
||||
->whereIn('embedding_status', [EmbeddingStatus::QUEUED, EmbeddingStatus::PROCESSING])
|
||||
->where(function ($time) use ($staleBefore): void {
|
||||
$time->whereNull('embedding_updated_at')->orWhere('embedding_updated_at', '<', $staleBefore);
|
||||
});
|
||||
});
|
||||
})
|
||||
->update([
|
||||
'embedding_status' => EmbeddingStatus::QUEUED,
|
||||
'embedding_error' => null,
|
||||
@@ -94,4 +163,18 @@ class ChunkEmbeddingRepository
|
||||
'embedding_updated_at' => Db::raw('CURRENT_TIMESTAMP'),
|
||||
]);
|
||||
}
|
||||
|
||||
private function pendingResetPayload(): array
|
||||
{
|
||||
return [
|
||||
'embedding_status' => EmbeddingStatus::PENDING,
|
||||
'embedding_ref' => null,
|
||||
'embedding_model' => null,
|
||||
'embedding_error' => null,
|
||||
'embedding_updated_at' => null,
|
||||
'search_index_status' => 0,
|
||||
'search_index_error' => null,
|
||||
'search_index_updated_at' => null,
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,20 +17,20 @@ class ChunkSearchIndexHandler
|
||||
$this->index = $index ?? new OpenSearchChunkIndex();
|
||||
}
|
||||
|
||||
public function handle(array $task): void
|
||||
public function handle(array $task): int
|
||||
{
|
||||
if (($task['target_type'] ?? null) !== 'archive') {
|
||||
return;
|
||||
return 0;
|
||||
}
|
||||
|
||||
$archiveUid = trim((string) ($task['target_uid'] ?? ''));
|
||||
if ($archiveUid === '') {
|
||||
return;
|
||||
return 0;
|
||||
}
|
||||
|
||||
$documents = $this->chunks->findQueuedDocuments($archiveUid, (int) config('opensearch.bulk.chunk_size', 500));
|
||||
if ($documents === []) {
|
||||
return;
|
||||
return 0;
|
||||
}
|
||||
|
||||
$chunkUids = array_column($documents, 'chunk_uid');
|
||||
@@ -39,7 +39,7 @@ class ChunkSearchIndexHandler
|
||||
try {
|
||||
$documents = $this->validatedDocuments($documents);
|
||||
if ($documents === []) {
|
||||
return;
|
||||
return 0;
|
||||
}
|
||||
|
||||
$chunkUids = array_column($documents, 'chunk_uid');
|
||||
@@ -53,6 +53,7 @@ class ChunkSearchIndexHandler
|
||||
|
||||
$indexedChunkUids = array_values(array_diff($chunkUids, $failedChunkUids));
|
||||
$this->chunks->markIndexed($indexedChunkUids);
|
||||
return count($chunkUids);
|
||||
} catch (Throwable $exception) {
|
||||
$this->chunks->markFailed($chunkUids, $exception->getMessage(), true);
|
||||
throw $exception;
|
||||
|
||||
@@ -7,6 +7,31 @@ use support\Db;
|
||||
|
||||
class ChunkSearchIndexRepository
|
||||
{
|
||||
public function countEmbeddedChunks(?string $archiveUid = null): int
|
||||
{
|
||||
$query = Db::table('chunks')
|
||||
->where('embedding_status', EmbeddingStatus::EMBEDDED);
|
||||
|
||||
if ($archiveUid !== null && trim($archiveUid) !== '') {
|
||||
$query->where('archive_uid', trim($archiveUid));
|
||||
}
|
||||
|
||||
return (int) $query->count();
|
||||
}
|
||||
|
||||
public function countIndexedChunks(?string $archiveUid = null): int
|
||||
{
|
||||
$query = Db::table('chunks')
|
||||
->where('embedding_status', EmbeddingStatus::EMBEDDED)
|
||||
->where('search_index_status', SearchIndexStatus::INDEXED);
|
||||
|
||||
if ($archiveUid !== null && trim($archiveUid) !== '') {
|
||||
$query->where('archive_uid', trim($archiveUid));
|
||||
}
|
||||
|
||||
return (int) $query->count();
|
||||
}
|
||||
|
||||
public function resetEmbeddedChunksToPending(?string $archiveUid = null): int
|
||||
{
|
||||
$query = Db::table('chunks')
|
||||
@@ -23,18 +48,44 @@ class ChunkSearchIndexRepository
|
||||
]);
|
||||
}
|
||||
|
||||
public function resetRecoverableChunksToPending(?string $archiveUid = null): int
|
||||
{
|
||||
$query = Db::table('chunks')
|
||||
->where('embedding_status', EmbeddingStatus::EMBEDDED)
|
||||
->whereIn('search_index_status', [
|
||||
SearchIndexStatus::QUEUED,
|
||||
SearchIndexStatus::INDEXING,
|
||||
SearchIndexStatus::FAILED_RETRYABLE,
|
||||
]);
|
||||
|
||||
if ($archiveUid !== null && trim($archiveUid) !== '') {
|
||||
$query->where('archive_uid', trim($archiveUid));
|
||||
}
|
||||
|
||||
return $query->update([
|
||||
'search_index_status' => SearchIndexStatus::PENDING,
|
||||
'search_index_error' => null,
|
||||
'search_index_updated_at' => null,
|
||||
]);
|
||||
}
|
||||
|
||||
public function queuePendingArchiveTasks(int $limit): array
|
||||
{
|
||||
$statuses = [
|
||||
SearchIndexStatus::PENDING,
|
||||
SearchIndexStatus::QUEUED,
|
||||
SearchIndexStatus::INDEXING,
|
||||
SearchIndexStatus::FAILED_RETRYABLE,
|
||||
];
|
||||
$staleBefore = date('Y-m-d H:i:s', time() - max(60, (int) config('queue.tasks.stale_after_seconds', 900)));
|
||||
|
||||
$archiveUids = Db::table('chunks')
|
||||
->where('embedding_status', EmbeddingStatus::EMBEDDED)
|
||||
->whereIn('search_index_status', $statuses)
|
||||
->where(function ($query) use ($staleBefore): void {
|
||||
$query
|
||||
->whereIn('search_index_status', [SearchIndexStatus::PENDING, SearchIndexStatus::FAILED_RETRYABLE])
|
||||
->orWhere(function ($stale) use ($staleBefore): void {
|
||||
$stale
|
||||
->whereIn('search_index_status', [SearchIndexStatus::QUEUED, SearchIndexStatus::INDEXING])
|
||||
->where(function ($time) use ($staleBefore): void {
|
||||
$time->whereNull('search_index_updated_at')->orWhere('search_index_updated_at', '<', $staleBefore);
|
||||
});
|
||||
});
|
||||
})
|
||||
->select('archive_uid')
|
||||
->groupBy('archive_uid')
|
||||
->orderByRaw('MIN(id)')
|
||||
@@ -47,7 +98,17 @@ class ChunkSearchIndexRepository
|
||||
Db::table('chunks')
|
||||
->where('archive_uid', $archiveUid)
|
||||
->where('embedding_status', EmbeddingStatus::EMBEDDED)
|
||||
->whereIn('search_index_status', $statuses)
|
||||
->where(function ($query) use ($staleBefore): void {
|
||||
$query
|
||||
->whereIn('search_index_status', [SearchIndexStatus::PENDING, SearchIndexStatus::FAILED_RETRYABLE])
|
||||
->orWhere(function ($stale) use ($staleBefore): void {
|
||||
$stale
|
||||
->whereIn('search_index_status', [SearchIndexStatus::QUEUED, SearchIndexStatus::INDEXING])
|
||||
->where(function ($time) use ($staleBefore): void {
|
||||
$time->whereNull('search_index_updated_at')->orWhere('search_index_updated_at', '<', $staleBefore);
|
||||
});
|
||||
});
|
||||
})
|
||||
->update([
|
||||
'search_index_status' => SearchIndexStatus::QUEUED,
|
||||
'search_index_error' => null,
|
||||
|
||||
@@ -139,12 +139,6 @@
|
||||
<label class="field-label" for="archive-metadata">metadata</label>
|
||||
<textarea class="text-area admin-code-area" id="archive-metadata" name="metadata" rows="8" placeholder='{"key":"value"}'></textarea>
|
||||
|
||||
<label class="field-label" for="archive-content">content</label>
|
||||
<textarea class="text-area admin-code-area" id="archive-content" name="content" rows="10"></textarea>
|
||||
|
||||
<label class="field-label" for="archive-raw">raw</label>
|
||||
<textarea class="text-area admin-code-area" id="archive-raw" name="raw" rows="10"></textarea>
|
||||
|
||||
<div class="admin-form-actions">
|
||||
<button class="button primary" type="submit">保存档案</button>
|
||||
<button class="button" type="button" id="archive-delete">删除档案</button>
|
||||
@@ -525,8 +519,6 @@ async function loadArchiveDetail(archiveUid) {
|
||||
field(els.archiveForm, 'tags').value = (data.tags || []).join(', ');
|
||||
field(els.archiveForm, 'summary').value = data.summary || '';
|
||||
field(els.archiveForm, 'metadata').value = JSON.stringify(data.metadata || {}, null, 2);
|
||||
field(els.archiveForm, 'content').value = data.content || '';
|
||||
field(els.archiveForm, 'raw').value = data.raw || '';
|
||||
}
|
||||
|
||||
async function saveArchive(event) {
|
||||
@@ -553,9 +545,7 @@ async function saveArchive(event) {
|
||||
series: field(els.archiveForm, 'series').value,
|
||||
tags: field(els.archiveForm, 'tags').value,
|
||||
summary: field(els.archiveForm, 'summary').value,
|
||||
metadata,
|
||||
content: field(els.archiveForm, 'content').value,
|
||||
raw: field(els.archiveForm, 'raw').value
|
||||
metadata
|
||||
};
|
||||
|
||||
await api(`/api/admin/archives/${encodeURIComponent(state.archiveUid)}`, {
|
||||
|
||||
Reference in New Issue
Block a user