暂存

2026-05-11 15:23:34 +08:00
parent ed70a140a2
commit 547cbbb4a5
25 changed files with 721 additions and 332 deletions
@@ -1,117 +1,5 @@
 #!/usr/bin/env php
 <?php

-use app\service\ArticleImportService;
-use support\Db;
-
-require __DIR__ . '/../vendor/autoload.php';
-require __DIR__ . '/../support/bootstrap.php';
-require __DIR__ . '/../vendor/webman/database/src/support/Db.php';
-
-$archiveUid = null;
-$force = false;
-$dryRun = false;
-
-foreach (array_slice($argv, 1) as $argument) {
-    if (str_starts_with($argument, '--archive_uid=')) {
-        $archiveUid = substr($argument, strlen('--archive_uid='));
-        continue;
-    }
-
-    if ($argument === '--force') {
-        $force = true;
-        continue;
-    }
-
-    if ($argument === '--dry-run') {
-        $dryRun = true;
-    }
-}
-
-$query = Db::table('archives')->orderBy('id');
-if ($archiveUid !== null && trim($archiveUid) !== '') {
-    $query->where('archive_uid', trim($archiveUid));
-}
-
-if (!$force) {
-    $query->where(function ($builder) {
-        $builder->whereNull('content')->orWhere('content', '');
-    });
-}
-
-$archives = $query->get(['archive_uid', 'title', 'content', 'raw'])->all();
-$normalizer = new ArticleImportService();
-
-$scanned = 0;
-$updated = 0;
-$fromRaw = 0;
-$fromChunks = 0;
-$skipped = 0;
-
-foreach ($archives as $archive) {
-    $scanned++;
-    $archiveUidValue = (string) $archive->archive_uid;
-    $raw = is_string($archive->raw ?? null) ? $archive->raw : null;
-    $content = null;
-    $source = 'none';
-
-    if (is_string($raw) && trim($raw) !== '') {
-        $content = $normalizer->normalizeArchiveContentString($raw);
-        $source = 'raw';
-    } else {
-        $chunks = Db::table('chunks')
-            ->where('archive_uid', $archiveUidValue)
-            ->orderBy('chunk_index')
-            ->pluck('text')
-            ->all();
-
-        $chunks = array_values(array_filter(array_map(
-            static fn ($value): string => trim((string) $value),
-            $chunks
-        ), static fn (string $value): bool => $value !== ''));
-
-        if ($chunks !== []) {
-            $content = trim(implode("\n\n", $chunks));
-            $source = 'chunks';
-        }
-    }
-
-    if ($content === null || $content === '') {
-        $skipped++;
-        echo "[skip] {$archiveUidValue} no usable raw/chunks" . PHP_EOL;
-        continue;
-    }
-
-    if ($dryRun) {
-        echo "[dry-run] {$archiveUidValue} source={$source} content_length=" . mb_strlen($content) . PHP_EOL;
-        if ($source === 'raw') {
-            $fromRaw++;
-        } else {
-            $fromChunks++;
-        }
-        continue;
-    }
-
-    Db::table('archives')
-        ->where('archive_uid', $archiveUidValue)
-        ->update(['content' => $content]);
-
-    $updated++;
-    if ($source === 'raw') {
-        $fromRaw++;
-    } else {
-        $fromChunks++;
-    }
-
-    echo "[updated] {$archiveUidValue} source={$source} content_length=" . mb_strlen($content) . PHP_EOL;
-}
-
-echo 'Archive content backfill completed.' . PHP_EOL;
-echo 'Archive filter: ' . ($archiveUid ?: 'auto') . PHP_EOL;
-echo 'Force mode: ' . ($force ? 'yes' : 'no') . PHP_EOL;
-echo 'Dry run: ' . ($dryRun ? 'yes' : 'no') . PHP_EOL;
-echo 'Scanned: ' . $scanned . PHP_EOL;
-echo 'Updated: ' . $updated . PHP_EOL;
-echo 'From raw: ' . $fromRaw . PHP_EOL;
-echo 'From chunks: ' . $fromChunks . PHP_EOL;
-echo 'Skipped: ' . $skipped . PHP_EOL;
+fwrite(STDERR, "Deprecated: archives.content and archives.raw have been removed from PostgreSQL. This script is no longer supported.\n");
+exit(1);
@@ -0,0 +1,135 @@
+#!/usr/bin/env php
+<?php
+
+use app\service\Embedding\ChunkEmbeddingHandler;
+use app\service\Embedding\ChunkEmbeddingRepository;
+use app\service\Embedding\EmbeddingStatus;
+
+require __DIR__ . '/../vendor/autoload.php';
+require __DIR__ . '/../support/bootstrap.php';
+
+$archiveUid = null;
+$forceReset = false;
+
+foreach (array_slice($argv, 1) as $argument) {
+    if (str_starts_with($argument, '--archive_uid=')) {
+        $archiveUid = substr($argument, strlen('--archive_uid='));
+        continue;
+    }
+
+    if ($argument === '--reset') {
+        $forceReset = true;
+    }
+}
+
+$repository = new ChunkEmbeddingRepository();
+$handler = new ChunkEmbeddingHandler();
+$batchSize = max(1, (int) config('LLMapi.embedding.batch_size', 32));
+
+try {
+    $totalChunks = $repository->countChunks($archiveUid);
+    if ($totalChunks === 0) {
+        echo 'Chunk re-embedding completed.' . PHP_EOL;
+        echo 'Archive filter: ' . ($archiveUid ?: '(all chunks)') . PHP_EOL;
+        echo 'Mode: nothing-to-do' . PHP_EOL;
+        echo 'Eligible chunks: 0' . PHP_EOL;
+        exit(0);
+    }
+
+    $mode = $forceReset ? 'reset' : 'resume';
+    $resetCount = $mode === 'reset'
+        ? $repository->resetAllChunksToPending($archiveUid)
+        : $repository->resetRecoverableChunksToPending($archiveUid);
+
+    $batchCount = 0;
+    $processedArchives = [];
+    $progress = completedCount($repository, $archiveUid);
+    echo 'Progress granularity: embedding request batches (up to ' . $batchSize . ' chunks each)' . PHP_EOL;
+    renderProgress($progress, $totalChunks, 'Re-embedding');
+
+    while (true) {
+        $archiveUids = $repository->queuePendingArchiveTasks(100);
+        if ($archiveUid !== null && trim($archiveUid) !== '') {
+            $archiveUids = array_values(array_filter($archiveUids, static fn (string $uid): bool => $uid === trim($archiveUid)));
+        }
+
+        if ($archiveUids === []) {
+            break;
+        }
+
+        foreach ($archiveUids as $uid) {
+            $processedChunkCount = $handler->handle([
+                'task_type' => 'embedding',
+                'target_type' => 'archive',
+                'target_uid' => $uid,
+                'attempt' => 1,
+            ]);
+            if ($processedChunkCount <= 0) {
+                continue;
+            }
+
+            $batchCount++;
+            $processedArchives[] = $uid;
+            $progress = completedCount($repository, $archiveUid);
+            renderProgress($progress, $totalChunks, 'Re-embedding');
+            fwrite(STDOUT, PHP_EOL . sprintf(
+                'Batch #%d archive=%s chunks=%d progress=%d/%d%s',
+                $batchCount,
+                $uid,
+                $processedChunkCount,
+                $progress,
+                $totalChunks,
+                PHP_EOL
+            ));
+        }
+    }
+
+    $embeddedChunks = $repository->countChunksByStatuses([EmbeddingStatus::EMBEDDED], $archiveUid);
+    $terminalFailures = $repository->countChunksByStatuses([EmbeddingStatus::FAILED_TERMINAL], $archiveUid);
+    renderProgress($embeddedChunks + $terminalFailures, $totalChunks, 'Re-embedding', true);
+
+    echo 'Chunk re-embedding completed.' . PHP_EOL;
+    echo 'Archive filter: ' . ($archiveUid ?: '(all chunks)') . PHP_EOL;
+    echo 'Mode: ' . $mode . ($forceReset ? ' (--reset)' : '') . PHP_EOL;
+    echo 'Eligible chunks: ' . $totalChunks . PHP_EOL;
+    echo 'Embedding batch size: ' . $batchSize . PHP_EOL;
+    echo 'Reset chunks: ' . $resetCount . PHP_EOL;
+    echo 'Processed archives: ' . count(array_unique($processedArchives)) . PHP_EOL;
+    echo 'Processed batches: ' . $batchCount . PHP_EOL;
+    echo 'Embedded chunk rows now marked embedded: ' . $embeddedChunks . PHP_EOL;
+    echo 'Terminal failures: ' . $terminalFailures . PHP_EOL;
+    if ($processedArchives !== []) {
+        echo 'Archives: ' . implode(', ', $processedArchives) . PHP_EOL;
+    }
+    echo 'Next step: refresh OpenSearch vectors with `php scripts/reindex_opensearch.php'
+        . ($archiveUid ? ' --archive_uid=' . $archiveUid : '')
+        . ($forceReset ? ' --reset' : '')
+        . '`' . PHP_EOL;
+} catch (Throwable $exception) {
+    fwrite(STDERR, PHP_EOL . $exception::class . ': ' . $exception->getMessage() . PHP_EOL);
+    exit(1);
+}
+
+function completedCount(ChunkEmbeddingRepository $repository, ?string $archiveUid): int
+{
+    return $repository->countChunksByStatuses([
+        EmbeddingStatus::EMBEDDED,
+        EmbeddingStatus::FAILED_TERMINAL,
+    ], $archiveUid);
+}
+
+function renderProgress(int $done, int $total, string $label, bool $final = false): void
+{
+    $total = max(1, $total);
+    $done = max(0, min($done, $total));
+    $width = 32;
+    $filled = (int) floor(($done / $total) * $width);
+    $bar = str_repeat('=', $filled) . str_repeat(' ', max(0, $width - $filled));
+    $percent = str_pad(number_format(($done / $total) * 100, 1), 5, ' ', STR_PAD_LEFT);
+    $line = sprintf("\r%s [%s] %s%% (%d/%d)", $label, $bar, $percent, $done, $total);
+    fwrite(STDOUT, $line);
+
+    if ($final || $done >= $total) {
+        fwrite(STDOUT, PHP_EOL);
+    }
+}
@@ -3,31 +3,59 @@

 use app\service\Search\ChunkSearchIndexHandler;
 use app\service\Search\ChunkSearchIndexRepository;
+use app\service\Search\OpenSearchClientFactory;
 use app\service\Search\OpenSearchChunkIndex;
-use support\Db;

 require __DIR__ . '/../vendor/autoload.php';
 require __DIR__ . '/../support/bootstrap.php';
-require __DIR__ . '/../vendor/webman/database/src/support/Db.php';

 $archiveUid = null;
+$forceReset = false;
 foreach (array_slice($argv, 1) as $argument) {
    if (str_starts_with($argument, '--archive_uid=')) {
        $archiveUid = substr($argument, strlen('--archive_uid='));
+        continue;
+    }
+
+    if ($argument === '--reset') {
+        $forceReset = true;
    }
 }

 $repository = new ChunkSearchIndexRepository();
 $handler = new ChunkSearchIndexHandler();
 $index = new OpenSearchChunkIndex();
+$clientFactory = new OpenSearchClientFactory();
+$bulkSize = max(1, (int) config('opensearch.bulk.chunk_size', 500));

 try {
+    $client = $clientFactory->make();
+    $indexName = config('opensearch.indices.chunks', 'proofdb_chunks');
+    $indexExists = (bool) $client->indices()->exists(['index' => $indexName]);
+
    $index->ensureExists();

-    $resetCount = $repository->resetEmbeddedChunksToPending($archiveUid);
-    $archiveCount = 0;
+    $totalChunks = $repository->countEmbeddedChunks($archiveUid);
+    if ($totalChunks === 0) {
+        echo 'OpenSearch reindex completed.' . PHP_EOL;
+        echo 'Index: ' . $indexName . PHP_EOL;
+        echo 'Archive filter: ' . ($archiveUid ?: '(all embedded archives)') . PHP_EOL;
+        echo 'Mode: nothing-to-do' . PHP_EOL;
+        echo 'Eligible embedded chunks: 0' . PHP_EOL;
+        exit(0);
+    }
+
+    $mode = $forceReset || !$indexExists ? 'reset' : 'resume';
+    $resetCount = $mode === 'reset'
+        ? $repository->resetEmbeddedChunksToPending($archiveUid)
+        : $repository->resetRecoverableChunksToPending($archiveUid);
+
+    $batchCount = 0;
    $indexedArchives = [];
-    $indexedChunks = 0;
+    $progress = $repository->countIndexedChunks($archiveUid);
+
+    echo 'Progress granularity: OpenSearch bulk batches (up to ' . $bulkSize . ' chunks each)' . PHP_EOL;
+    renderProgress($progress, $totalChunks, 'Reindexing');

    while (true) {
        $archiveUids = $repository->queuePendingArchiveTasks(100);
@@ -36,28 +64,44 @@ try {
        }

        foreach ($archiveUids as $uid) {
-            $handler->handle([
+            $processedChunkCount = $handler->handle([
                'task_type' => 'search_index',
                'target_type' => 'archive',
                'target_uid' => $uid,
                'attempt' => 1,
            ]);
-            $archiveCount++;
+            if ($processedChunkCount <= 0) {
+                continue;
+            }
+
+            $batchCount++;
            $indexedArchives[] = $uid;
+            $progress = $repository->countIndexedChunks($archiveUid);
+            renderProgress($progress, $totalChunks, 'Reindexing');
+            fwrite(STDOUT, PHP_EOL . sprintf(
+                'Batch #%d archive=%s chunks=%d progress=%d/%d%s',
+                $batchCount,
+                $uid,
+                $processedChunkCount,
+                $progress,
+                $totalChunks,
+                PHP_EOL
+            ));
        }
    }

-    $indexedChunksQuery = Db::table('chunks')->where('search_index_status', 3);
-    if ($archiveUid !== null && trim($archiveUid) !== '') {
-        $indexedChunksQuery->where('archive_uid', trim($archiveUid));
-    }
-    $indexedChunks = (int) $indexedChunksQuery->count();
+    $indexedChunks = $repository->countIndexedChunks($archiveUid);
+    renderProgress($indexedChunks, $totalChunks, 'Reindexing', true);

    echo 'OpenSearch reindex completed.' . PHP_EOL;
-    echo 'Index: ' . config('opensearch.indices.chunks', 'proofdb_chunks') . PHP_EOL;
+    echo 'Index: ' . $indexName . PHP_EOL;
    echo 'Archive filter: ' . ($archiveUid ?: '(all embedded archives)') . PHP_EOL;
+    echo 'Mode: ' . $mode . ($forceReset ? ' (--reset)' : (!$indexExists ? ' (index was missing)' : '')) . PHP_EOL;
+    echo 'Eligible embedded chunks: ' . $totalChunks . PHP_EOL;
+    echo 'OpenSearch bulk size: ' . $bulkSize . PHP_EOL;
    echo 'Reset chunks: ' . $resetCount . PHP_EOL;
-    echo 'Indexed archives: ' . $archiveCount . PHP_EOL;
+    echo 'Indexed archives: ' . count(array_unique($indexedArchives)) . PHP_EOL;
+    echo 'Processed batches: ' . $batchCount . PHP_EOL;
    echo 'Indexed chunk rows now marked indexed: ' . $indexedChunks . PHP_EOL;
    if ($indexedArchives !== []) {
        echo 'Archives: ' . implode(', ', $indexedArchives) . PHP_EOL;
@@ -66,3 +110,19 @@ try {
    fwrite(STDERR, $exception::class . ': ' . $exception->getMessage() . PHP_EOL);
    exit(1);
 }
+
+function renderProgress(int $done, int $total, string $label, bool $final = false): void
+{
+    $total = max(1, $total);
+    $done = max(0, min($done, $total));
+    $width = 32;
+    $filled = (int) floor(($done / $total) * $width);
+    $bar = str_repeat('=', $filled) . str_repeat(' ', max(0, $width - $filled));
+    $percent = str_pad(number_format(($done / $total) * 100, 1), 5, ' ', STR_PAD_LEFT);
+    $line = sprintf("\r%s [%s] %s%% (%d/%d)", $label, $bar, $percent, $done, $total);
+    fwrite(STDOUT, $line);
+
+    if ($final || $done >= $total) {
+        fwrite(STDOUT, PHP_EOL);
+    }
+}
@@ -20,14 +20,14 @@ CREATE TABLE IF NOT EXISTS archives (
    series TEXT,
    tags JSONB NOT NULL DEFAULT '[]'::jsonb,
    metadata JSONB NOT NULL DEFAULT '{}'::jsonb,
-    content TEXT,
-    raw TEXT,
    chunks JSONB NOT NULL DEFAULT '[]'::jsonb,
    created_time TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
    updated_time TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
 )
 SQL,
    'ALTER TABLE archives ADD COLUMN IF NOT EXISTS summary TEXT',
+    'ALTER TABLE archives DROP COLUMN IF EXISTS content',
+    'ALTER TABLE archives DROP COLUMN IF EXISTS raw',
    <<<SQL
 CREATE TABLE IF NOT EXISTS chunks (
    id BIGSERIAL PRIMARY KEY,