feat(ai): unify AI architecture, implement RAG and legacy migration
This commit is contained in:
@@ -2,12 +2,14 @@ import { Test, TestingModule } from '@nestjs/testing';
|
||||
import { ServiceUnavailableException } from '@nestjs/common';
|
||||
import { getRepositoryToken } from '@nestjs/typeorm';
|
||||
|
||||
import { getQueueToken } from '@nestjs/bullmq';
|
||||
import { RagService } from '../rag.service';
|
||||
import { QdrantService } from '../qdrant.service';
|
||||
import { EmbeddingService } from '../embedding.service';
|
||||
import { TyphoonService } from '../typhoon.service';
|
||||
import { IngestionService } from '../ingestion.service';
|
||||
import { DocumentChunk } from '../entities/document-chunk.entity';
|
||||
import { QUEUE_AI_VECTOR_DELETION } from '../../common/constants/queue.constants';
|
||||
|
||||
const DEFAULT_REDIS_TOKEN = 'default_IORedisModuleConnectionToken';
|
||||
|
||||
@@ -41,6 +43,10 @@ const mockRedis = {
|
||||
setex: jest.fn(),
|
||||
};
|
||||
|
||||
const mockVectorDeletionQueue = {
|
||||
add: jest.fn().mockResolvedValue({ id: 'mock-job-id' }),
|
||||
};
|
||||
|
||||
describe('RagService', () => {
|
||||
let service: RagService;
|
||||
|
||||
@@ -54,6 +60,10 @@ describe('RagService', () => {
|
||||
{ provide: IngestionService, useValue: mockIngestion },
|
||||
{ provide: getRepositoryToken(DocumentChunk), useValue: mockChunkRepo },
|
||||
{ provide: DEFAULT_REDIS_TOKEN, useValue: mockRedis },
|
||||
{
|
||||
provide: getQueueToken(QUEUE_AI_VECTOR_DELETION),
|
||||
useValue: mockVectorDeletionQueue,
|
||||
},
|
||||
],
|
||||
}).compile();
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import { BullModule } from '@nestjs/bullmq';
|
||||
import { ConfigModule } from '@nestjs/config';
|
||||
|
||||
import { DocumentChunk } from './entities/document-chunk.entity';
|
||||
import { QUEUE_AI_VECTOR_DELETION } from '../common/constants/queue.constants';
|
||||
import { EmbeddingService } from './embedding.service';
|
||||
import { QdrantService } from './qdrant.service';
|
||||
import { TyphoonService } from './typhoon.service';
|
||||
@@ -30,7 +31,9 @@ const DLQ_DEFAULTS = {
|
||||
BullModule.registerQueue(
|
||||
{ name: 'rag-ocr', defaultJobOptions: DLQ_DEFAULTS },
|
||||
{ name: 'rag-thai-preprocess', defaultJobOptions: DLQ_DEFAULTS },
|
||||
{ name: 'rag-embedding', defaultJobOptions: DLQ_DEFAULTS }
|
||||
{ name: 'rag-embedding', defaultJobOptions: DLQ_DEFAULTS },
|
||||
// T028: Producer สำหรับ dispatch vector deletion jobs (ADR-023 FR-008)
|
||||
{ name: QUEUE_AI_VECTOR_DELETION }
|
||||
),
|
||||
],
|
||||
controllers: [RagController],
|
||||
|
||||
@@ -6,6 +6,10 @@ import {
|
||||
} from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository } from 'typeorm';
|
||||
import { InjectQueue } from '@nestjs/bullmq';
|
||||
import { Queue } from 'bullmq';
|
||||
import { QUEUE_AI_VECTOR_DELETION } from '../common/constants/queue.constants';
|
||||
import { AiVectorDeletionJobPayload } from '../ai/ai-queue.service';
|
||||
import { InjectRedis } from '@nestjs-modules/ioredis';
|
||||
import Redis from 'ioredis';
|
||||
import { createHash } from 'crypto';
|
||||
@@ -32,7 +36,9 @@ export class RagService {
|
||||
private readonly ingestionService: IngestionService,
|
||||
@InjectRepository(DocumentChunk)
|
||||
private readonly chunkRepo: Repository<DocumentChunk>,
|
||||
@InjectRedis() private readonly redis: Redis
|
||||
@InjectRedis() private readonly redis: Redis,
|
||||
@InjectQueue(QUEUE_AI_VECTOR_DELETION)
|
||||
private readonly vectorDeletionQueue: Queue<AiVectorDeletionJobPayload>
|
||||
) {}
|
||||
|
||||
async query(
|
||||
@@ -184,19 +190,24 @@ export class RagService {
|
||||
await this.qdrant.onModuleInit();
|
||||
}
|
||||
|
||||
async deleteVectors(attachmentPublicId: string): Promise<void> {
|
||||
async deleteVectors(
|
||||
attachmentPublicId: string,
|
||||
requestedByUserPublicId = 'system'
|
||||
): Promise<void> {
|
||||
// ลบ DocumentChunk ออกจาก DB แบบ synchronous (รวดเร็ว ไม่มี external dependency)
|
||||
await this.chunkRepo.delete({ documentId: attachmentPublicId });
|
||||
try {
|
||||
await this.qdrant.deleteByDocumentId(attachmentPublicId);
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
`Qdrant delete failed for ${attachmentPublicId}`,
|
||||
err instanceof Error ? err.stack : String(err)
|
||||
);
|
||||
}
|
||||
await this.chunkRepo.manager.query(
|
||||
`UPDATE attachments SET rag_status = 'PENDING', rag_last_error = NULL WHERE public_id = ?`,
|
||||
[attachmentPublicId]
|
||||
// T028: เปลี่ยน Qdrant deletion เป็น async ผ่าน BullMQ เพื่อ eventual consistency (FR-008)
|
||||
await this.vectorDeletionQueue.add(
|
||||
'delete-document-vectors',
|
||||
{ documentPublicId: attachmentPublicId, requestedByUserPublicId },
|
||||
{
|
||||
jobId: attachmentPublicId,
|
||||
attempts: 3,
|
||||
backoff: { type: 'exponential', delay: 5000 },
|
||||
}
|
||||
);
|
||||
this.logger.log(
|
||||
`Vector deletion queued for attachment=${attachmentPublicId}`
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user