feat(ai-admin-console): complete implementation and resolve lint compilation errors
This commit is contained in:
@@ -0,0 +1,151 @@
|
||||
// File: src/modules/ai/processors/ai-batch.processor.spec.ts
|
||||
// Change Log
|
||||
// - 2026-05-21: สร้าง Unit Test สำหรับ AiBatchProcessor ครอบคลุม embed-document และ sandbox-rag (T032).
|
||||
// - 2026-05-21: เพิ่มการทดสอบ sandbox-extract พร้อม mock OcrService, OllamaService และ Redis (T039).
|
||||
// - 2026-05-21: แก้ไข ESLint unexpected any และ unsafe member access โดยกำหนด type ให้ redis เป็น Record<string, jest.Mock>
|
||||
|
||||
import { Test, TestingModule } from '@nestjs/testing';
|
||||
import { getRepositoryToken } from '@nestjs/typeorm';
|
||||
import { Repository } from 'typeorm';
|
||||
import { Job } from 'bullmq';
|
||||
import { AiBatchProcessor, AiBatchJobData } from './ai-batch.processor';
|
||||
import { EmbeddingService } from '../services/embedding.service';
|
||||
import { AiRagService } from '../ai-rag.service';
|
||||
import { Attachment } from '../../../common/file-storage/entities/attachment.entity';
|
||||
import { OcrService } from '../services/ocr.service';
|
||||
import { OllamaService } from '../services/ollama.service';
|
||||
|
||||
describe('AiBatchProcessor', () => {
|
||||
let processor: AiBatchProcessor;
|
||||
let embeddingService: jest.Mocked<EmbeddingService>;
|
||||
let ragService: jest.Mocked<AiRagService>;
|
||||
let ocrService: jest.Mocked<OcrService>;
|
||||
let ollamaService: jest.Mocked<OllamaService>;
|
||||
let redis: Record<string, jest.Mock>;
|
||||
let attachmentRepo: jest.Mocked<Repository<Attachment>>;
|
||||
const DEFAULT_REDIS_TOKEN = 'default_IORedisModuleConnectionToken';
|
||||
const mockEmbeddingService = {
|
||||
embedDocument: jest
|
||||
.fn()
|
||||
.mockResolvedValue({ success: true, chunksEmbedded: 5 }),
|
||||
};
|
||||
const mockRagService = {
|
||||
processQuery: jest.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
const mockOcrService = {
|
||||
detectAndExtract: jest
|
||||
.fn()
|
||||
.mockResolvedValue({ text: 'OCR text LCBP3-CIV-001 Civil' }),
|
||||
};
|
||||
const mockOllamaService = {
|
||||
generate: jest.fn().mockResolvedValue(
|
||||
JSON.stringify({
|
||||
documentNumber: 'LCBP3-CIV-001',
|
||||
subject: 'Foundation Inspection Report',
|
||||
discipline: 'Civil',
|
||||
date: '2026-05-20',
|
||||
confidence: 0.95,
|
||||
})
|
||||
),
|
||||
};
|
||||
const mockRedis = {
|
||||
setex: jest.fn().mockResolvedValue('OK'),
|
||||
};
|
||||
const mockAttachmentRepo = {
|
||||
update: jest.fn().mockResolvedValue({ affected: 1 }),
|
||||
};
|
||||
beforeEach(async () => {
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
providers: [
|
||||
AiBatchProcessor,
|
||||
{ provide: EmbeddingService, useValue: mockEmbeddingService },
|
||||
{ provide: AiRagService, useValue: mockRagService },
|
||||
{ provide: OcrService, useValue: mockOcrService },
|
||||
{ provide: OllamaService, useValue: mockOllamaService },
|
||||
{ provide: DEFAULT_REDIS_TOKEN, useValue: mockRedis },
|
||||
{
|
||||
provide: getRepositoryToken(Attachment),
|
||||
useValue: mockAttachmentRepo,
|
||||
},
|
||||
],
|
||||
}).compile();
|
||||
processor = module.get<AiBatchProcessor>(AiBatchProcessor);
|
||||
embeddingService = module.get(EmbeddingService);
|
||||
ragService = module.get(AiRagService);
|
||||
ocrService = module.get(OcrService);
|
||||
ollamaService = module.get(OllamaService);
|
||||
redis = module.get(DEFAULT_REDIS_TOKEN);
|
||||
attachmentRepo = module.get(getRepositoryToken(Attachment));
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
it('ควรสามารถเรียก process embed-document และอัปเดตสถานะใน database', async () => {
|
||||
const job = {
|
||||
id: 'job-embed',
|
||||
data: {
|
||||
jobType: 'embed-document',
|
||||
documentPublicId: 'doc-uuid-123',
|
||||
projectPublicId: 'proj-uuid-456',
|
||||
payload: { pdfPath: '/files/test.pdf' },
|
||||
idempotencyKey: 'idem-123',
|
||||
},
|
||||
} as unknown as Job<AiBatchJobData>;
|
||||
await processor.process(job);
|
||||
expect(embeddingService.embedDocument).toHaveBeenCalledTimes(1);
|
||||
expect(attachmentRepo.update).toHaveBeenCalledWith(
|
||||
{ publicId: 'doc-uuid-123' },
|
||||
{ aiProcessingStatus: 'PROCESSING' }
|
||||
);
|
||||
expect(attachmentRepo.update).toHaveBeenCalledWith(
|
||||
{ publicId: 'doc-uuid-123' },
|
||||
{ aiProcessingStatus: 'DONE' }
|
||||
);
|
||||
});
|
||||
it('ควรประมวลผล sandbox-rag โดยการเรียก ragService.processQuery และข้ามการอัปเดต database', async () => {
|
||||
const job = {
|
||||
id: 'job-sandbox',
|
||||
data: {
|
||||
jobType: 'sandbox-rag',
|
||||
documentPublicId: 'idem-sandbox-123',
|
||||
projectPublicId: 'proj-uuid-456',
|
||||
payload: {
|
||||
query: 'ทดสอบคำถาม sandbox RAG',
|
||||
userPublicId: 'user-uuid-789',
|
||||
},
|
||||
idempotencyKey: 'idem-sandbox-123',
|
||||
},
|
||||
} as unknown as Job<AiBatchJobData>;
|
||||
await processor.process(job);
|
||||
expect(ragService.processQuery).toHaveBeenCalledTimes(1);
|
||||
expect(ragService.processQuery).toHaveBeenCalledWith(
|
||||
'idem-sandbox-123',
|
||||
'ทดสอบคำถาม sandbox RAG',
|
||||
'proj-uuid-456',
|
||||
'user-uuid-789',
|
||||
expect.any(AbortSignal)
|
||||
);
|
||||
expect(attachmentRepo.update).not.toHaveBeenCalled();
|
||||
});
|
||||
it('ควรประมวลผล sandbox-extract โดยใช้ OcrService, OllamaService และเก็บค่าลง Redis', async () => {
|
||||
const job = {
|
||||
id: 'job-extract',
|
||||
data: {
|
||||
jobType: 'sandbox-extract',
|
||||
documentPublicId: 'idem-extract-123',
|
||||
projectPublicId: 'proj-uuid-456',
|
||||
payload: { pdfPath: '/files/test.pdf' },
|
||||
idempotencyKey: 'idem-extract-123',
|
||||
},
|
||||
} as unknown as Job<AiBatchJobData>;
|
||||
await processor.process(job);
|
||||
expect(ocrService.detectAndExtract).toHaveBeenCalledWith({
|
||||
pdfPath: '/files/test.pdf',
|
||||
});
|
||||
expect(ollamaService.generate).toHaveBeenCalledTimes(1);
|
||||
expect(redis.setex).toHaveBeenCalledTimes(2);
|
||||
expect(redis.setex).toHaveBeenLastCalledWith(
|
||||
'ai:rag:result:idem-extract-123',
|
||||
3600,
|
||||
expect.stringContaining('completed')
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -2,17 +2,30 @@
|
||||
// Change Log
|
||||
// - 2026-05-15: เพิ่ม processor สำหรับ ai-batch queue ตาม ADR-023A.
|
||||
// - 2026-05-15: เพิ่ม EmbeddingService สำหรับ embed-document logic (T022).
|
||||
// - 2026-05-21: เพิ่มการรองรับ sandbox-rag และ sandbox-extract สำหรับ Superadmin sandbox.
|
||||
// - 2026-05-21: พัฒนาระบบประมวลผล sandbox-extract พร้อมเชื่อมต่อ OcrService, OllamaService และ Redis cache
|
||||
// - 2026-05-21: แก้ไข ESLint unused variable สำหรับ parseError ใน catch block
|
||||
|
||||
import { Processor, WorkerHost } from '@nestjs/bullmq';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { Job } from 'bullmq';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository } from 'typeorm';
|
||||
import { InjectRedis } from '@nestjs-modules/ioredis';
|
||||
import Redis from 'ioredis';
|
||||
import { Attachment } from '../../../common/file-storage/entities/attachment.entity';
|
||||
import { QUEUE_AI_BATCH } from '../../common/constants/queue.constants';
|
||||
import { EmbeddingService } from '../services/embedding.service';
|
||||
import { AiRagService } from '../ai-rag.service';
|
||||
import { OcrService } from '../services/ocr.service';
|
||||
import { OllamaService } from '../services/ollama.service';
|
||||
|
||||
export type AiBatchJobType = 'ocr' | 'extract-metadata' | 'embed-document';
|
||||
export type AiBatchJobType =
|
||||
| 'ocr'
|
||||
| 'extract-metadata'
|
||||
| 'embed-document'
|
||||
| 'sandbox-rag'
|
||||
| 'sandbox-extract';
|
||||
|
||||
export interface AiBatchJobData {
|
||||
jobType: AiBatchJobType;
|
||||
@@ -27,36 +40,62 @@ export interface AiBatchJobData {
|
||||
@Processor(QUEUE_AI_BATCH, { concurrency: 1 })
|
||||
export class AiBatchProcessor extends WorkerHost {
|
||||
private readonly logger = new Logger(AiBatchProcessor.name);
|
||||
private readonly abortControllers = new Map<string, AbortController>();
|
||||
|
||||
constructor(
|
||||
@InjectRepository(Attachment)
|
||||
private readonly attachmentRepo: Repository<Attachment>,
|
||||
private readonly embeddingService: EmbeddingService
|
||||
private readonly embeddingService: EmbeddingService,
|
||||
private readonly ragService: AiRagService,
|
||||
private readonly ocrService: OcrService,
|
||||
private readonly ollamaService: OllamaService,
|
||||
@InjectRedis() private readonly redis: Redis
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
/** Dispatch งาน batch ตาม jobType */
|
||||
async process(job: Job<AiBatchJobData>): Promise<void> {
|
||||
await this.setAiProcessingStatus(job.data.documentPublicId, 'PROCESSING');
|
||||
const isSandbox =
|
||||
job.data.jobType === 'sandbox-rag' ||
|
||||
job.data.jobType === 'sandbox-extract';
|
||||
if (!isSandbox) {
|
||||
await this.setAiProcessingStatus(job.data.documentPublicId, 'PROCESSING');
|
||||
}
|
||||
try {
|
||||
switch (job.data.jobType) {
|
||||
case 'ocr':
|
||||
this.logger.log(`OCR batch job processing — jobId=${String(job.id)}`);
|
||||
// OCR logic handled by OcrService in ai-realtime processor
|
||||
await this.setAiProcessingStatus(job.data.documentPublicId, 'DONE');
|
||||
if (!isSandbox) {
|
||||
await this.setAiProcessingStatus(job.data.documentPublicId, 'DONE');
|
||||
}
|
||||
return;
|
||||
case 'extract-metadata':
|
||||
this.logger.log(
|
||||
`Metadata extraction job processing — jobId=${String(job.id)}`
|
||||
);
|
||||
// Metadata extraction handled in ai-realtime processor
|
||||
await this.setAiProcessingStatus(job.data.documentPublicId, 'DONE');
|
||||
if (!isSandbox) {
|
||||
await this.setAiProcessingStatus(job.data.documentPublicId, 'DONE');
|
||||
}
|
||||
return;
|
||||
case 'embed-document':
|
||||
this.logger.log(`Embedding job processing — jobId=${String(job.id)}`);
|
||||
await this.processEmbedDocument(job.data);
|
||||
await this.setAiProcessingStatus(job.data.documentPublicId, 'DONE');
|
||||
if (!isSandbox) {
|
||||
await this.setAiProcessingStatus(job.data.documentPublicId, 'DONE');
|
||||
}
|
||||
return;
|
||||
case 'sandbox-rag':
|
||||
this.logger.log(
|
||||
`Sandbox RAG job processing — jobId=${String(job.id)}`
|
||||
);
|
||||
await this.processSandboxRag(job.data);
|
||||
return;
|
||||
case 'sandbox-extract':
|
||||
this.logger.log(
|
||||
`Sandbox Extract job processing — jobId=${String(job.id)}`
|
||||
);
|
||||
await this.processSandboxExtract(job.data);
|
||||
return;
|
||||
default: {
|
||||
const unreachable: never = job.data.jobType;
|
||||
@@ -70,7 +109,9 @@ export class AiBatchProcessor extends WorkerHost {
|
||||
`Batch job failed — jobType=${job.data.jobType}, documentPublicId=${job.data.documentPublicId}`,
|
||||
err instanceof Error ? err.stack : String(err)
|
||||
);
|
||||
await this.setAiProcessingStatus(job.data.documentPublicId, 'FAILED');
|
||||
if (!isSandbox) {
|
||||
await this.setAiProcessingStatus(job.data.documentPublicId, 'FAILED');
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
@@ -80,27 +121,43 @@ export class AiBatchProcessor extends WorkerHost {
|
||||
const { documentPublicId, projectPublicId, payload } = data;
|
||||
const pdfPath = payload.pdfPath as string;
|
||||
const extractedText = payload.extractedText as string | undefined;
|
||||
|
||||
if (!pdfPath) {
|
||||
throw new Error('pdfPath is required for embed-document job');
|
||||
}
|
||||
|
||||
const result = await this.embeddingService.embedDocument(
|
||||
pdfPath,
|
||||
documentPublicId,
|
||||
projectPublicId,
|
||||
extractedText
|
||||
);
|
||||
|
||||
if (!result.success) {
|
||||
throw new Error(`Embedding failed: ${result.error ?? 'Unknown error'}`);
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`Embedding completed for document ${documentPublicId} — ${result.chunksEmbedded} chunks embedded`
|
||||
);
|
||||
}
|
||||
|
||||
/** ประมวลผล sandbox RAG query */
|
||||
private async processSandboxRag(data: AiBatchJobData): Promise<void> {
|
||||
const { projectPublicId, idempotencyKey, payload } = data;
|
||||
const query = payload.query as string;
|
||||
const userPublicId = payload.userPublicId as string;
|
||||
const controller = new AbortController();
|
||||
this.abortControllers.set(idempotencyKey, controller);
|
||||
try {
|
||||
await this.ragService.processQuery(
|
||||
idempotencyKey,
|
||||
query,
|
||||
projectPublicId,
|
||||
userPublicId,
|
||||
controller.signal
|
||||
);
|
||||
} finally {
|
||||
this.abortControllers.delete(idempotencyKey);
|
||||
}
|
||||
}
|
||||
|
||||
private async setAiProcessingStatus(
|
||||
documentPublicId: string,
|
||||
status: 'PENDING' | 'PROCESSING' | 'DONE' | 'FAILED'
|
||||
@@ -110,4 +167,85 @@ export class AiBatchProcessor extends WorkerHost {
|
||||
{ aiProcessingStatus: status }
|
||||
);
|
||||
}
|
||||
|
||||
/** ประมวลผล sandbox OCR + Metadata extraction โดยไม่บันทึกลง database */
|
||||
private async processSandboxExtract(data: AiBatchJobData): Promise<void> {
|
||||
const { idempotencyKey, payload } = data;
|
||||
const pdfPath = payload.pdfPath as string;
|
||||
if (!pdfPath) {
|
||||
throw new Error('pdfPath is required for sandbox-extract job');
|
||||
}
|
||||
await this.redis.setex(
|
||||
`ai:rag:result:${idempotencyKey}`,
|
||||
3600,
|
||||
JSON.stringify({
|
||||
requestPublicId: idempotencyKey,
|
||||
status: 'processing',
|
||||
})
|
||||
);
|
||||
try {
|
||||
const ocrResult = await this.ocrService.detectAndExtract({ pdfPath });
|
||||
const prompt = `You are an expert document extraction system.
|
||||
Analyze the following OCR text extracted from a project document and extract the metadata fields.
|
||||
|
||||
OCR TEXT:
|
||||
${ocrResult.text}
|
||||
|
||||
Extract these fields:
|
||||
1. documentNumber: The official document number or code. If not found, return null.
|
||||
2. subject: The main subject, title, or topic of the document. If not found, return null.
|
||||
3. discipline: Must be exactly one of: "Civil", "Mechanical", "Electrical", "Architectural", or null if not specified.
|
||||
4. date: The issue date in YYYY-MM-DD format. If not found, return null.
|
||||
5. confidence: A float between 0.0 and 1.0 indicating your confidence in this extraction.
|
||||
|
||||
Return ONLY a valid JSON object matching this schema. Do NOT include markdown code blocks, HTML, or any conversational text. Example:
|
||||
{
|
||||
"documentNumber": "LCBP3-CIV-001",
|
||||
"subject": "Foundation Inspection Report",
|
||||
"discipline": "Civil",
|
||||
"date": "2026-05-20",
|
||||
"confidence": 0.95
|
||||
}`;
|
||||
const response = await this.ollamaService.generate(prompt);
|
||||
const cleanedResponse = response
|
||||
.replace(/```json/g, '')
|
||||
.replace(/```/g, '')
|
||||
.trim();
|
||||
let extractedMetadata: Record<string, unknown>;
|
||||
try {
|
||||
extractedMetadata = JSON.parse(cleanedResponse) as Record<
|
||||
string,
|
||||
unknown
|
||||
>;
|
||||
} catch {
|
||||
throw new Error(
|
||||
`Failed to parse LLM response as JSON: ${cleanedResponse}`
|
||||
);
|
||||
}
|
||||
await this.redis.setex(
|
||||
`ai:rag:result:${idempotencyKey}`,
|
||||
3600,
|
||||
JSON.stringify({
|
||||
requestPublicId: idempotencyKey,
|
||||
status: 'completed',
|
||||
answer: JSON.stringify(extractedMetadata, null, 2),
|
||||
completedAt: new Date().toISOString(),
|
||||
})
|
||||
);
|
||||
} catch (err: unknown) {
|
||||
const errMsg = err instanceof Error ? err.message : String(err);
|
||||
this.logger.error(`Sandbox extract failed: ${errMsg}`);
|
||||
await this.redis.setex(
|
||||
`ai:rag:result:${idempotencyKey}`,
|
||||
3600,
|
||||
JSON.stringify({
|
||||
requestPublicId: idempotencyKey,
|
||||
status: 'failed',
|
||||
errorMessage: errMsg,
|
||||
completedAt: new Date().toISOString(),
|
||||
})
|
||||
);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user