暂存
This commit is contained in:
@@ -0,0 +1,99 @@
|
||||
<?php
|
||||
|
||||
namespace app\service\Search;
|
||||
|
||||
use Throwable;
|
||||
|
||||
class ChunkSearchIndexHandler
|
||||
{
|
||||
private ChunkSearchIndexRepository $chunks;
|
||||
private OpenSearchChunkIndex $index;
|
||||
|
||||
public function __construct(
|
||||
?ChunkSearchIndexRepository $chunks = null,
|
||||
?OpenSearchChunkIndex $index = null
|
||||
) {
|
||||
$this->chunks = $chunks ?? new ChunkSearchIndexRepository();
|
||||
$this->index = $index ?? new OpenSearchChunkIndex();
|
||||
}
|
||||
|
||||
public function handle(array $task): void
|
||||
{
|
||||
if (($task['target_type'] ?? null) !== 'archive') {
|
||||
return;
|
||||
}
|
||||
|
||||
$archiveUid = trim((string) ($task['target_uid'] ?? ''));
|
||||
if ($archiveUid === '') {
|
||||
return;
|
||||
}
|
||||
|
||||
$documents = $this->chunks->findQueuedDocuments($archiveUid, (int) config('opensearch.bulk.chunk_size', 500));
|
||||
if ($documents === []) {
|
||||
return;
|
||||
}
|
||||
|
||||
$chunkUids = array_column($documents, 'chunk_uid');
|
||||
$this->chunks->markIndexing($chunkUids);
|
||||
|
||||
try {
|
||||
$documents = $this->validatedDocuments($documents);
|
||||
if ($documents === []) {
|
||||
return;
|
||||
}
|
||||
|
||||
$chunkUids = array_column($documents, 'chunk_uid');
|
||||
$this->index->ensureExists();
|
||||
$response = $this->index->bulkIndex($documents);
|
||||
$failedChunkUids = $this->failedChunkUids($response);
|
||||
|
||||
if ($failedChunkUids !== []) {
|
||||
$this->chunks->markFailed($failedChunkUids, 'OpenSearch bulk index returned item errors.', true);
|
||||
}
|
||||
|
||||
$indexedChunkUids = array_values(array_diff($chunkUids, $failedChunkUids));
|
||||
$this->chunks->markIndexed($indexedChunkUids);
|
||||
} catch (Throwable $exception) {
|
||||
$this->chunks->markFailed($chunkUids, $exception->getMessage(), true);
|
||||
throw $exception;
|
||||
}
|
||||
}
|
||||
|
||||
private function validatedDocuments(array $documents): array
|
||||
{
|
||||
$dimensions = (int) config('opensearch.vector.dimensions', 2048);
|
||||
$valid = [];
|
||||
foreach ($documents as $document) {
|
||||
if (($document['embedding_dimensions'] ?? 0) !== $dimensions) {
|
||||
$this->chunks->markFailed([$document['chunk_uid']], sprintf(
|
||||
'Chunk %s embedding dimension mismatch: expected %d, got %d.',
|
||||
$document['chunk_uid'] ?? '',
|
||||
$dimensions,
|
||||
$document['embedding_dimensions'] ?? 0
|
||||
), false);
|
||||
continue;
|
||||
}
|
||||
|
||||
$valid[] = $document;
|
||||
}
|
||||
|
||||
return $valid;
|
||||
}
|
||||
|
||||
private function failedChunkUids(array $response): array
|
||||
{
|
||||
if (($response['errors'] ?? false) !== true) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$failed = [];
|
||||
foreach ($response['items'] ?? [] as $item) {
|
||||
$result = $item['index'] ?? [];
|
||||
if (isset($result['error'])) {
|
||||
$failed[] = (string) ($result['_id'] ?? '');
|
||||
}
|
||||
}
|
||||
|
||||
return array_values(array_filter($failed));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
<?php
|
||||
|
||||
namespace app\service\Search;
|
||||
|
||||
use app\service\Embedding\EmbeddingStatus;
|
||||
use support\Db;
|
||||
|
||||
class ChunkSearchIndexRepository
|
||||
{
|
||||
public function queuePendingArchiveTasks(int $limit): array
|
||||
{
|
||||
$statuses = [
|
||||
SearchIndexStatus::PENDING,
|
||||
SearchIndexStatus::QUEUED,
|
||||
SearchIndexStatus::INDEXING,
|
||||
SearchIndexStatus::FAILED_RETRYABLE,
|
||||
];
|
||||
|
||||
$archiveUids = Db::table('chunks')
|
||||
->where('embedding_status', EmbeddingStatus::EMBEDDED)
|
||||
->whereIn('search_index_status', $statuses)
|
||||
->select('archive_uid')
|
||||
->groupBy('archive_uid')
|
||||
->orderByRaw('MIN(id)')
|
||||
->limit($limit)
|
||||
->pluck('archive_uid')
|
||||
->all();
|
||||
|
||||
$archiveUids = array_values(array_filter(array_map('strval', $archiveUids)));
|
||||
foreach ($archiveUids as $archiveUid) {
|
||||
Db::table('chunks')
|
||||
->where('archive_uid', $archiveUid)
|
||||
->where('embedding_status', EmbeddingStatus::EMBEDDED)
|
||||
->whereIn('search_index_status', $statuses)
|
||||
->update([
|
||||
'search_index_status' => SearchIndexStatus::QUEUED,
|
||||
'search_index_error' => null,
|
||||
'search_index_updated_at' => Db::raw('CURRENT_TIMESTAMP'),
|
||||
]);
|
||||
}
|
||||
|
||||
return $archiveUids;
|
||||
}
|
||||
|
||||
public function findQueuedDocuments(string $archiveUid, int $limit): array
|
||||
{
|
||||
$rows = Db::table('chunks')
|
||||
->join('archives', 'chunks.archive_uid', '=', 'archives.archive_uid')
|
||||
->where('chunks.archive_uid', $archiveUid)
|
||||
->where('chunks.embedding_status', EmbeddingStatus::EMBEDDED)
|
||||
->whereIn('chunks.search_index_status', [SearchIndexStatus::QUEUED, SearchIndexStatus::INDEXING])
|
||||
->orderBy('chunks.chunk_index')
|
||||
->limit($limit)
|
||||
->get([
|
||||
'chunks.chunk_uid',
|
||||
'chunks.archive_uid',
|
||||
'chunks.chunk_index',
|
||||
'chunks.page_start',
|
||||
'chunks.page_end',
|
||||
'chunks.text',
|
||||
'chunks.embedding_ref',
|
||||
'chunks.embedding_model',
|
||||
'chunks.created_time',
|
||||
'chunks.updated_time',
|
||||
'archives.title',
|
||||
'archives.source',
|
||||
'archives.author',
|
||||
'archives.year',
|
||||
'archives.series',
|
||||
'archives.tags',
|
||||
])
|
||||
->all();
|
||||
|
||||
return array_map(fn (object $row): array => $this->documentFromRow($row), $rows);
|
||||
}
|
||||
|
||||
public function markIndexing(array $chunkUids): void
|
||||
{
|
||||
$this->updateStatus($chunkUids, SearchIndexStatus::INDEXING, null);
|
||||
}
|
||||
|
||||
public function markIndexed(array $chunkUids): void
|
||||
{
|
||||
$this->updateStatus($chunkUids, SearchIndexStatus::INDEXED, null);
|
||||
}
|
||||
|
||||
public function markFailed(array $chunkUids, string $error, bool $retryable): void
|
||||
{
|
||||
$this->updateStatus(
|
||||
$chunkUids,
|
||||
$retryable ? SearchIndexStatus::FAILED_RETRYABLE : SearchIndexStatus::FAILED_TERMINAL,
|
||||
$error
|
||||
);
|
||||
}
|
||||
|
||||
private function documentFromRow(object $row): array
|
||||
{
|
||||
$embeddingRef = $this->decodeJson($row->embedding_ref ?? null, []);
|
||||
$embedding = is_array($embeddingRef['embedding'] ?? null) ? $embeddingRef['embedding'] : [];
|
||||
|
||||
return [
|
||||
'chunk_uid' => (string) $row->chunk_uid,
|
||||
'archive_uid' => (string) $row->archive_uid,
|
||||
'chunk_index' => (int) $row->chunk_index,
|
||||
'page_start' => $row->page_start === null ? null : (int) $row->page_start,
|
||||
'page_end' => $row->page_end === null ? null : (int) $row->page_end,
|
||||
'title' => $row->title,
|
||||
'source' => $row->source,
|
||||
'author' => $row->author,
|
||||
'year' => $row->year === null ? null : (int) $row->year,
|
||||
'series' => $row->series,
|
||||
'tags' => $this->decodeJson($row->tags ?? null, []),
|
||||
'text' => (string) $row->text,
|
||||
'embedding' => array_map('floatval', $embedding),
|
||||
'embedding_model' => (string) $row->embedding_model,
|
||||
'embedding_dimensions' => count($embedding),
|
||||
'created_time' => $this->dateString($row->created_time ?? null),
|
||||
'updated_time' => $this->dateString($row->updated_time ?? null),
|
||||
];
|
||||
}
|
||||
|
||||
private function updateStatus(array $chunkUids, int $status, ?string $error): void
|
||||
{
|
||||
if ($chunkUids === []) {
|
||||
return;
|
||||
}
|
||||
|
||||
Db::table('chunks')->whereIn('chunk_uid', $chunkUids)->update([
|
||||
'search_index_status' => $status,
|
||||
'search_index_error' => $error === null ? null : mb_substr($error, 0, 4000),
|
||||
'search_index_updated_at' => Db::raw('CURRENT_TIMESTAMP'),
|
||||
]);
|
||||
}
|
||||
|
||||
private function decodeJson(mixed $value, array $fallback): array
|
||||
{
|
||||
if (is_array($value)) {
|
||||
return $value;
|
||||
}
|
||||
|
||||
if (!is_string($value) || trim($value) === '') {
|
||||
return $fallback;
|
||||
}
|
||||
|
||||
$decoded = json_decode($value, true);
|
||||
return is_array($decoded) ? $decoded : $fallback;
|
||||
}
|
||||
|
||||
private function dateString(mixed $value): ?string
|
||||
{
|
||||
if ($value === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return date(DATE_ATOM, strtotime((string) $value) ?: time());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
<?php
|
||||
|
||||
namespace app\service\Search;
|
||||
|
||||
use OpenSearch\Client;
|
||||
|
||||
class OpenSearchChunkIndex
|
||||
{
|
||||
public function __construct(private readonly ?Client $client = null)
|
||||
{
|
||||
}
|
||||
|
||||
public function ensureExists(): void
|
||||
{
|
||||
$client = $this->client();
|
||||
$index = $this->indexName();
|
||||
|
||||
if ($client->indices()->exists(['index' => $index])) {
|
||||
return;
|
||||
}
|
||||
|
||||
$client->indices()->create([
|
||||
'index' => $index,
|
||||
'body' => $this->mapping(),
|
||||
]);
|
||||
}
|
||||
|
||||
public function bulkIndex(array $documents): array
|
||||
{
|
||||
if ($documents === []) {
|
||||
return ['items' => [], 'errors' => false];
|
||||
}
|
||||
|
||||
$body = [];
|
||||
foreach ($documents as $document) {
|
||||
$body[] = [
|
||||
'index' => [
|
||||
'_index' => $this->indexName(),
|
||||
'_id' => $document['chunk_uid'],
|
||||
],
|
||||
];
|
||||
$body[] = $document;
|
||||
}
|
||||
|
||||
return $this->client()->bulk([
|
||||
'refresh' => config('opensearch.bulk.refresh', 'false'),
|
||||
'body' => $body,
|
||||
]);
|
||||
}
|
||||
|
||||
public function mapping(): array
|
||||
{
|
||||
return [
|
||||
'settings' => [
|
||||
'index' => [
|
||||
'knn' => true,
|
||||
],
|
||||
],
|
||||
'mappings' => [
|
||||
'properties' => [
|
||||
'chunk_uid' => ['type' => 'keyword'],
|
||||
'archive_uid' => ['type' => 'keyword'],
|
||||
'chunk_index' => ['type' => 'integer'],
|
||||
'page_start' => ['type' => 'integer'],
|
||||
'page_end' => ['type' => 'integer'],
|
||||
'title' => $this->textWithKeyword(),
|
||||
'source' => $this->textWithKeyword(),
|
||||
'author' => $this->textWithKeyword(),
|
||||
'year' => ['type' => 'integer'],
|
||||
'series' => $this->textWithKeyword(),
|
||||
'tags' => ['type' => 'keyword'],
|
||||
'text' => ['type' => 'text'],
|
||||
'embedding' => [
|
||||
'type' => 'knn_vector',
|
||||
'dimension' => (int) config('opensearch.vector.dimensions', 2048),
|
||||
'method' => [
|
||||
'name' => 'hnsw',
|
||||
'space_type' => config('opensearch.vector.space_type', 'cosinesimil'),
|
||||
'engine' => config('opensearch.vector.engine', 'lucene'),
|
||||
],
|
||||
],
|
||||
'embedding_model' => ['type' => 'keyword'],
|
||||
'embedding_dimensions' => ['type' => 'integer'],
|
||||
'created_time' => ['type' => 'date'],
|
||||
'updated_time' => ['type' => 'date'],
|
||||
],
|
||||
],
|
||||
];
|
||||
}
|
||||
|
||||
private function client(): Client
|
||||
{
|
||||
return $this->client ?? (new OpenSearchClientFactory())->make();
|
||||
}
|
||||
|
||||
private function indexName(): string
|
||||
{
|
||||
return config('opensearch.indices.chunks', 'proofdb_chunks');
|
||||
}
|
||||
|
||||
private function textWithKeyword(): array
|
||||
{
|
||||
return [
|
||||
'type' => 'text',
|
||||
'fields' => [
|
||||
'keyword' => [
|
||||
'type' => 'keyword',
|
||||
'ignore_above' => 512,
|
||||
],
|
||||
],
|
||||
];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
<?php
|
||||
|
||||
namespace app\service\Search;
|
||||
|
||||
use OpenSearch\Client;
|
||||
use OpenSearch\ClientBuilder;
|
||||
|
||||
class OpenSearchClientFactory
|
||||
{
|
||||
public function make(?array $config = null): Client
|
||||
{
|
||||
$config = $config ?? config('opensearch.default', []);
|
||||
|
||||
$builder = ClientBuilder::create()
|
||||
->setHosts($config['hosts'] ?? ['http://127.0.0.1:9200'])
|
||||
->setSSLVerification((bool) ($config['ssl_verify'] ?? true))
|
||||
->setConnectionParams([
|
||||
'client' => [
|
||||
'timeout' => (float) ($config['timeout'] ?? 30),
|
||||
'connect_timeout' => (float) ($config['connect_timeout'] ?? 5),
|
||||
],
|
||||
]);
|
||||
|
||||
$username = trim((string) ($config['username'] ?? ''));
|
||||
$password = trim((string) ($config['password'] ?? ''));
|
||||
if ($username !== '' && $password !== '') {
|
||||
$builder->setBasicAuthentication($username, $password);
|
||||
}
|
||||
|
||||
return $builder->build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,350 @@
|
||||
<?php
|
||||
|
||||
namespace app\service\Search;
|
||||
|
||||
use app\service\Embedding\BigModelEmbeddingClient;
|
||||
use InvalidArgumentException;
|
||||
use OpenSearch\Client;
|
||||
|
||||
class OpenSearchSearchService
|
||||
{
|
||||
private const DEFAULT_LIMIT = 10;
|
||||
private const MAX_LIMIT = 50;
|
||||
private const DEFAULT_RRF_K = 60;
|
||||
|
||||
public function __construct(
|
||||
private readonly ?Client $client = null,
|
||||
private readonly ?BigModelEmbeddingClient $embeddingClient = null,
|
||||
private readonly ?SearchKeywordService $keywordService = null
|
||||
) {
|
||||
}
|
||||
|
||||
public function fulltext(array $payload): array
|
||||
{
|
||||
$query = trim((string) ($payload['query'] ?? ''));
|
||||
if ($query === '') {
|
||||
throw new InvalidArgumentException('query is required.');
|
||||
}
|
||||
|
||||
$limit = $this->limit($payload['limit'] ?? self::DEFAULT_LIMIT);
|
||||
$filters = is_array($payload['filters'] ?? null) ? $payload['filters'] : [];
|
||||
$body = [
|
||||
'size' => $limit,
|
||||
'query' => [
|
||||
'bool' => [
|
||||
'must' => [
|
||||
[
|
||||
'multi_match' => [
|
||||
'query' => $query,
|
||||
'fields' => [
|
||||
'text^4',
|
||||
'title^3',
|
||||
'source^2',
|
||||
'author^2',
|
||||
'series^2',
|
||||
'tags^2',
|
||||
],
|
||||
'type' => 'best_fields',
|
||||
],
|
||||
],
|
||||
],
|
||||
'filter' => $this->filters($filters),
|
||||
],
|
||||
],
|
||||
'_source' => $this->sourceFields(),
|
||||
];
|
||||
|
||||
$response = $this->client()->search([
|
||||
'index' => config('opensearch.indices.chunks', 'proofdb_chunks'),
|
||||
'body' => $body,
|
||||
]);
|
||||
|
||||
return [
|
||||
'mode' => 'fulltext',
|
||||
'query' => $query,
|
||||
'limit' => $limit,
|
||||
'filters' => $filters,
|
||||
'total' => $this->total($response),
|
||||
'hits' => $this->hits($response),
|
||||
];
|
||||
}
|
||||
|
||||
public function vector(array $payload): array
|
||||
{
|
||||
$query = trim((string) ($payload['query'] ?? ''));
|
||||
if ($query === '') {
|
||||
throw new InvalidArgumentException('query is required.');
|
||||
}
|
||||
|
||||
$limit = $this->limit($payload['limit'] ?? self::DEFAULT_LIMIT);
|
||||
$k = $this->limit($payload['k'] ?? $limit);
|
||||
$filters = is_array($payload['filters'] ?? null) ? $payload['filters'] : [];
|
||||
$embedding = $this->queryEmbedding($query);
|
||||
|
||||
$response = $this->client()->search([
|
||||
'index' => config('opensearch.indices.chunks', 'proofdb_chunks'),
|
||||
'body' => [
|
||||
'size' => $limit,
|
||||
'query' => [
|
||||
'bool' => [
|
||||
'must' => [
|
||||
[
|
||||
'knn' => [
|
||||
'embedding' => [
|
||||
'vector' => $embedding,
|
||||
'k' => $k,
|
||||
],
|
||||
],
|
||||
],
|
||||
],
|
||||
'filter' => $this->filters($filters),
|
||||
],
|
||||
],
|
||||
'_source' => $this->sourceFields(),
|
||||
],
|
||||
]);
|
||||
|
||||
return [
|
||||
'mode' => 'vector',
|
||||
'query' => $query,
|
||||
'limit' => $limit,
|
||||
'k' => $k,
|
||||
'filters' => $filters,
|
||||
'embedding_model' => config('LLMapi.embedding.model', 'embedding-3'),
|
||||
'embedding_dimensions' => count($embedding),
|
||||
'total' => $this->total($response),
|
||||
'hits' => $this->hits($response),
|
||||
];
|
||||
}
|
||||
|
||||
public function hybrid(array $payload): array
|
||||
{
|
||||
$query = trim((string) ($payload['query'] ?? ''));
|
||||
if ($query === '') {
|
||||
throw new InvalidArgumentException('query is required.');
|
||||
}
|
||||
|
||||
$limit = $this->limit($payload['limit'] ?? self::DEFAULT_LIMIT);
|
||||
$candidateLimit = $this->limit($payload['candidate_limit'] ?? max($limit * 3, 20));
|
||||
$rrfK = max(1, (int) ($payload['rrf_k'] ?? self::DEFAULT_RRF_K));
|
||||
$filters = is_array($payload['filters'] ?? null) ? $payload['filters'] : [];
|
||||
$aiKeywords = null;
|
||||
$fulltextQuery = $query;
|
||||
if ($this->aiEnabled($payload)) {
|
||||
$aiKeywords = $this->keywordService()->generate($query);
|
||||
$fulltextQuery = trim((string) ($aiKeywords['query'] ?? '')) ?: $query;
|
||||
}
|
||||
|
||||
$basePayload = [
|
||||
'query' => $query,
|
||||
'limit' => $candidateLimit,
|
||||
'k' => $candidateLimit,
|
||||
'filters' => $filters,
|
||||
];
|
||||
$fulltextPayload = $basePayload;
|
||||
$fulltextPayload['query'] = $fulltextQuery;
|
||||
|
||||
$fulltext = $this->fulltext($fulltextPayload);
|
||||
$vector = $this->vector($basePayload);
|
||||
$hits = $this->rrf($fulltext['hits'], $vector['hits'], $rrfK);
|
||||
|
||||
return [
|
||||
'mode' => 'hybrid',
|
||||
'query' => $query,
|
||||
'limit' => $limit,
|
||||
'candidate_limit' => $candidateLimit,
|
||||
'rrf_k' => $rrfK,
|
||||
'filters' => $filters,
|
||||
'ai' => $aiKeywords !== null,
|
||||
'fulltext_query' => $fulltextQuery,
|
||||
'vector_query' => $query,
|
||||
'keywords' => $aiKeywords,
|
||||
'total' => count($hits),
|
||||
'hits' => array_slice($hits, 0, $limit),
|
||||
'sources' => [
|
||||
'fulltext_total' => $fulltext['total'],
|
||||
'vector_total' => $vector['total'],
|
||||
'fulltext_hits' => count($fulltext['hits']),
|
||||
'vector_hits' => count($vector['hits']),
|
||||
],
|
||||
];
|
||||
}
|
||||
|
||||
private function aiEnabled(array $payload): bool
|
||||
{
|
||||
return (bool) ($payload['ai'] ?? false) && (bool) config('LLMapi.search_keywords.enabled', true);
|
||||
}
|
||||
|
||||
private function filters(array $filters): array
|
||||
{
|
||||
$clauses = [];
|
||||
|
||||
foreach (['archive_uid', 'chunk_uid'] as $field) {
|
||||
if (!empty($filters[$field])) {
|
||||
$clauses[] = ['term' => [$field => (string) $filters[$field]]];
|
||||
}
|
||||
}
|
||||
|
||||
foreach (['source', 'author', 'series'] as $field) {
|
||||
if (!empty($filters[$field])) {
|
||||
$clauses[] = ['term' => [$field . '.keyword' => (string) $filters[$field]]];
|
||||
}
|
||||
}
|
||||
|
||||
if (isset($filters['year']) && is_numeric($filters['year'])) {
|
||||
$clauses[] = ['term' => ['year' => (int) $filters['year']]];
|
||||
}
|
||||
|
||||
if (!empty($filters['tags'])) {
|
||||
$tags = is_array($filters['tags']) ? $filters['tags'] : [$filters['tags']];
|
||||
$tags = array_values(array_filter(array_map('strval', $tags)));
|
||||
if ($tags !== []) {
|
||||
$clauses[] = ['terms' => ['tags' => $tags]];
|
||||
}
|
||||
}
|
||||
|
||||
return $clauses;
|
||||
}
|
||||
|
||||
private function hits(array $response): array
|
||||
{
|
||||
$hits = [];
|
||||
foreach ($response['hits']['hits'] ?? [] as $hit) {
|
||||
$source = is_array($hit['_source'] ?? null) ? $hit['_source'] : [];
|
||||
$hits[] = [
|
||||
'score' => (float) ($hit['_score'] ?? 0),
|
||||
'chunk_uid' => $source['chunk_uid'] ?? null,
|
||||
'archive_uid' => $source['archive_uid'] ?? null,
|
||||
'chunk_index' => $source['chunk_index'] ?? null,
|
||||
'page_start' => $source['page_start'] ?? null,
|
||||
'page_end' => $source['page_end'] ?? null,
|
||||
'title' => $source['title'] ?? null,
|
||||
'source' => $source['source'] ?? null,
|
||||
'author' => $source['author'] ?? null,
|
||||
'year' => $source['year'] ?? null,
|
||||
'series' => $source['series'] ?? null,
|
||||
'tags' => $source['tags'] ?? [],
|
||||
'text' => $source['text'] ?? '',
|
||||
'embedding_model' => $source['embedding_model'] ?? null,
|
||||
'embedding_dimensions' => $source['embedding_dimensions'] ?? null,
|
||||
];
|
||||
}
|
||||
|
||||
return $hits;
|
||||
}
|
||||
|
||||
private function rrf(array $fulltextHits, array $vectorHits, int $rrfK): array
|
||||
{
|
||||
$merged = [];
|
||||
$this->mergeRankedHits($merged, $fulltextHits, 'fulltext', $rrfK);
|
||||
$this->mergeRankedHits($merged, $vectorHits, 'vector', $rrfK);
|
||||
|
||||
usort($merged, static function (array $a, array $b): int {
|
||||
$scoreCompare = ($b['hybrid_score'] ?? 0) <=> ($a['hybrid_score'] ?? 0);
|
||||
if ($scoreCompare !== 0) {
|
||||
return $scoreCompare;
|
||||
}
|
||||
|
||||
return ($b['score'] ?? 0) <=> ($a['score'] ?? 0);
|
||||
});
|
||||
|
||||
return array_values($merged);
|
||||
}
|
||||
|
||||
private function mergeRankedHits(array &$merged, array $hits, string $source, int $rrfK): void
|
||||
{
|
||||
foreach ($hits as $index => $hit) {
|
||||
$chunkUid = (string) ($hit['chunk_uid'] ?? '');
|
||||
if ($chunkUid === '') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$rank = $index + 1;
|
||||
$contribution = 1 / ($rrfK + $rank);
|
||||
if (!isset($merged[$chunkUid])) {
|
||||
$merged[$chunkUid] = $hit;
|
||||
$merged[$chunkUid]['score'] = 0.0;
|
||||
$merged[$chunkUid]['hybrid_score'] = 0.0;
|
||||
$merged[$chunkUid]['rank_sources'] = [];
|
||||
}
|
||||
|
||||
$merged[$chunkUid]['hybrid_score'] += $contribution;
|
||||
$merged[$chunkUid]['score'] = max((float) ($merged[$chunkUid]['score'] ?? 0), (float) ($hit['score'] ?? 0));
|
||||
$merged[$chunkUid]['rank_sources'][$source] = [
|
||||
'rank' => $rank,
|
||||
'score' => (float) ($hit['score'] ?? 0),
|
||||
'rrf' => $contribution,
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
private function total(array $response): int
|
||||
{
|
||||
$total = $response['hits']['total'] ?? 0;
|
||||
if (is_array($total)) {
|
||||
return (int) ($total['value'] ?? 0);
|
||||
}
|
||||
|
||||
return (int) $total;
|
||||
}
|
||||
|
||||
private function limit(mixed $value): int
|
||||
{
|
||||
return min(self::MAX_LIMIT, max(1, (int) $value));
|
||||
}
|
||||
|
||||
private function queryEmbedding(string $query): array
|
||||
{
|
||||
$payload = $this->embeddingClient()->embed([$query], [
|
||||
'model' => config('LLMapi.embedding.model', 'embedding-3'),
|
||||
'dimensions' => config('LLMapi.embedding.dimensions', 2048),
|
||||
]);
|
||||
|
||||
$embedding = $payload['data'][0]['embedding'] ?? null;
|
||||
if (!is_array($embedding)) {
|
||||
throw new InvalidArgumentException('query embedding could not be generated.');
|
||||
}
|
||||
|
||||
$dimensions = (int) config('opensearch.vector.dimensions', 2048);
|
||||
if (count($embedding) !== $dimensions) {
|
||||
throw new InvalidArgumentException("query embedding dimensions must be {$dimensions}.");
|
||||
}
|
||||
|
||||
return array_map('floatval', $embedding);
|
||||
}
|
||||
|
||||
private function sourceFields(): array
|
||||
{
|
||||
return [
|
||||
'chunk_uid',
|
||||
'archive_uid',
|
||||
'chunk_index',
|
||||
'page_start',
|
||||
'page_end',
|
||||
'title',
|
||||
'source',
|
||||
'author',
|
||||
'year',
|
||||
'series',
|
||||
'tags',
|
||||
'text',
|
||||
'embedding_model',
|
||||
'embedding_dimensions',
|
||||
];
|
||||
}
|
||||
|
||||
private function client(): Client
|
||||
{
|
||||
return $this->client ?? (new OpenSearchClientFactory())->make();
|
||||
}
|
||||
|
||||
private function embeddingClient(): BigModelEmbeddingClient
|
||||
{
|
||||
return $this->embeddingClient ?? new BigModelEmbeddingClient();
|
||||
}
|
||||
|
||||
private function keywordService(): SearchKeywordService
|
||||
{
|
||||
return $this->keywordService ?? new SearchKeywordService();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
<?php
|
||||
|
||||
namespace app\service\Search;
|
||||
|
||||
class SearchIndexStatus
|
||||
{
|
||||
public const PENDING = 0;
|
||||
public const QUEUED = 1;
|
||||
public const INDEXING = 2;
|
||||
public const INDEXED = 3;
|
||||
public const FAILED_RETRYABLE = 4;
|
||||
public const FAILED_TERMINAL = 5;
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
<?php
|
||||
|
||||
namespace app\service\Search;
|
||||
|
||||
use app\service\LLM\LLMRetryQueue;
|
||||
use app\service\LLM\OpenAICompatibleClient;
|
||||
use Throwable;
|
||||
|
||||
class SearchKeywordService
|
||||
{
|
||||
private OpenAICompatibleClient $client;
|
||||
private LLMRetryQueue $retryQueue;
|
||||
|
||||
public function __construct(?OpenAICompatibleClient $client = null, ?LLMRetryQueue $retryQueue = null)
|
||||
{
|
||||
$this->client = $client ?? new OpenAICompatibleClient($this->clientConfig());
|
||||
$this->retryQueue = $retryQueue ?? new LLMRetryQueue();
|
||||
}
|
||||
|
||||
public function generate(string $query): array
|
||||
{
|
||||
if (!$this->client->isConfigured()) {
|
||||
return $this->fallback($query, 'LLM API is not configured.');
|
||||
}
|
||||
|
||||
try {
|
||||
$result = $this->retryQueue->run(
|
||||
fn (): array => $this->client->chatJson($this->messages($query), [
|
||||
'model' => config('LLMapi.search_keywords.model', config('LLMapi.metadata.model')),
|
||||
'temperature' => config('LLMapi.search_keywords.temperature', 0.1),
|
||||
'max_tokens' => config('LLMapi.search_keywords.max_tokens', 300),
|
||||
'stream' => false,
|
||||
'response_format' => config('LLMapi.search_keywords.response_format', ['type' => 'json_object']),
|
||||
'thinking' => config('LLMapi.search_keywords.thinking', ['type' => 'disabled']),
|
||||
'request_id' => 'search-keywords-' . substr(hash('sha256', $query), 0, 32),
|
||||
]),
|
||||
config('LLMapi.search_keywords.retry', config('LLMapi.metadata.retry', []))
|
||||
);
|
||||
} catch (Throwable $exception) {
|
||||
return $this->fallback($query, $exception->getMessage());
|
||||
}
|
||||
|
||||
$keywords = $this->keywords($result);
|
||||
if ($keywords === []) {
|
||||
return $this->fallback($query, 'LLM returned no usable keywords.');
|
||||
}
|
||||
|
||||
return [
|
||||
'enabled' => true,
|
||||
'attempted' => true,
|
||||
'error' => null,
|
||||
'keywords' => $keywords,
|
||||
'query' => implode(' ', $keywords),
|
||||
'model' => config('LLMapi.search_keywords.model', config('LLMapi.metadata.model')),
|
||||
];
|
||||
}
|
||||
|
||||
private function messages(string $query): array
|
||||
{
|
||||
return [
|
||||
[
|
||||
'role' => 'system',
|
||||
'content' => implode("\n", [
|
||||
'你是历史档案检索关键词生成助手。',
|
||||
'任务：把用户的自然语言问题改写为适合 BM25 全文检索的关键词。',
|
||||
'优先输出档案中可能出现的英文专名、政策名、事件名、人物名、地点名、缩写和年份。',
|
||||
'如果用户输入中文，请翻译或扩展为可能出现在英文档案中的关键词。',
|
||||
'只返回 JSON 对象，不要 Markdown，不要解释。',
|
||||
'JSON 格式：{"keywords":["keyword1","keyword2"],"query":"keyword1 keyword2"}。',
|
||||
'keywords 数量 3-12 个；不要编造过于具体而输入中没有依据的事实。',
|
||||
]),
|
||||
],
|
||||
[
|
||||
'role' => 'user',
|
||||
'content' => json_encode(['query' => $query], JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
|
||||
],
|
||||
];
|
||||
}
|
||||
|
||||
private function clientConfig(): array
|
||||
{
|
||||
$config = config('LLMapi.default', []);
|
||||
$config['timeout'] = config('LLMapi.search_keywords.timeout', 12);
|
||||
$config['connect_timeout'] = config('LLMapi.search_keywords.connect_timeout', 5);
|
||||
|
||||
return $config;
|
||||
}
|
||||
|
||||
private function keywords(array $result): array
|
||||
{
|
||||
$keywords = [];
|
||||
if (isset($result['keywords']) && is_array($result['keywords'])) {
|
||||
$keywords = $result['keywords'];
|
||||
} elseif (isset($result['query']) && is_string($result['query'])) {
|
||||
$keywords = preg_split('/\s+/', $result['query']) ?: [];
|
||||
}
|
||||
|
||||
return array_values(array_unique(array_filter(array_map(
|
||||
static fn (mixed $value): string => trim((string) $value),
|
||||
$keywords
|
||||
))));
|
||||
}
|
||||
|
||||
private function fallback(string $query, string $error): array
|
||||
{
|
||||
return [
|
||||
'enabled' => true,
|
||||
'attempted' => false,
|
||||
'error' => $error,
|
||||
'keywords' => [],
|
||||
'query' => $query,
|
||||
'model' => null,
|
||||
];
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user