690419:1831 feat: update CI/CD to use SSH key authentication #05
This commit is contained in:
@@ -0,0 +1,57 @@
|
||||
# ========================================
|
||||
# LCBP3 Backend — Environment Variables
|
||||
# Copy to .env and fill in real values
|
||||
# หมายเหตุ: ค่า DB_PASSWORD, REDIS_PASSWORD,
|
||||
# ELASTICSEARCH_PASSWORD ต้องตรงกับที่ตั้งไว้ใน
|
||||
# services stack (MariaDB/Redis/Elasticsearch
|
||||
# ดูจาก .env ของ services stack ที่รันอยู่แล้วบน QNAP
|
||||
# ========================================
|
||||
|
||||
# Database
|
||||
DB_HOST=localhost
|
||||
DB_PORT=3306
|
||||
DB_USERNAME=admin
|
||||
DB_PASSWORD=Center2025
|
||||
DB_DATABASE=lcbp3_dev
|
||||
|
||||
# Redis
|
||||
REDIS_HOST=localhost
|
||||
REDIS_PORT=16379
|
||||
REDIS_PASSWORD=Center2025
|
||||
|
||||
# JWT
|
||||
JWT_SECRET=change-me-in-production
|
||||
JWT_EXPIRES_IN=7d
|
||||
|
||||
# File Storage
|
||||
UPLOAD_DEST=./uploads
|
||||
MAX_FILE_SIZE=52428800
|
||||
|
||||
# ClamAV
|
||||
CLAMAV_HOST=localhost
|
||||
CLAMAV_PORT=3310
|
||||
|
||||
# ========================================
|
||||
# ADR-022 RAG — Retrieval-Augmented Generation
|
||||
# ========================================
|
||||
|
||||
# Qdrant vector store (local docker-compose or QNAP)
|
||||
QDRANT_URL=http://localhost:6333
|
||||
|
||||
# Ollama (Admin Desktop Desk-5439 — ADR-018 AI boundary)
|
||||
OLLAMA_EMBED_MODEL=nomic-embed-text
|
||||
OLLAMA_RAG_MODEL=gemma3:12b
|
||||
OLLAMA_URL=http://192.168.10.100:11434
|
||||
|
||||
# Thai preprocessing microservice (PyThaiNLP — Admin Desktop)
|
||||
THAI_PREPROCESS_URL=http://192.168.10.100:8765
|
||||
|
||||
# Typhoon API (cloud LLM — PUBLIC/INTERNAL only, never CONFIDENTIAL)
|
||||
TYPHOON_API_KEY=your-typhoon-api-key-here
|
||||
TYPHOON_API_URL=https://api.opentyphoon.ai/v1
|
||||
|
||||
# RAG query config
|
||||
RAG_TOPK=20
|
||||
RAG_FINAL_K=5
|
||||
RAG_TIMEOUT_MS=5000
|
||||
RAG_QUERY_CACHE_TTL=300
|
||||
@@ -54,10 +54,24 @@ services:
|
||||
- esdata:/usr/share/elasticsearch/data
|
||||
networks:
|
||||
- lcbp3-net
|
||||
# ADR-022 RAG: Qdrant vector store (tiered multitenancy for project isolation)
|
||||
qdrant:
|
||||
image: qdrant/qdrant:v1.16.1
|
||||
container_name: lcbp3-qdrant-local
|
||||
restart: always
|
||||
ports:
|
||||
- '6333:6333' # REST API
|
||||
- '6334:6334' # gRPC
|
||||
volumes:
|
||||
- qdrant_data:/qdrant/storage
|
||||
networks:
|
||||
- lcbp3-net
|
||||
|
||||
volumes:
|
||||
db_data:
|
||||
redis_data: # เพิ่ม Volume
|
||||
esdata:
|
||||
qdrant_data: # ADR-022 RAG vector store
|
||||
|
||||
networks:
|
||||
lcbp3-net:
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large
Load Diff
@@ -24,19 +24,19 @@
|
||||
"seed": "ts-node -r tsconfig-paths/register src/database/seeds/run-seed.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@casl/ability": "^6.7.5",
|
||||
"@casl/ability": "6.8.0",
|
||||
"@elastic/elasticsearch": "^8.13.0",
|
||||
"@nestjs-modules/ioredis": "^2.0.2",
|
||||
"@nestjs/axios": "^4.0.1",
|
||||
"@nestjs/bullmq": "^11.0.4",
|
||||
"@nestjs/cache-manager": "^3.0.1",
|
||||
"@nestjs/common": "^11.0.1",
|
||||
"@nestjs/common": "^11.1.19",
|
||||
"@nestjs/config": "^4.0.2",
|
||||
"@nestjs/core": "^11.0.1",
|
||||
"@nestjs/core": "^11.1.19",
|
||||
"@nestjs/elasticsearch": "^11.1.0",
|
||||
"@nestjs/jwt": "^11.0.1",
|
||||
"@nestjs/passport": "^11.0.5",
|
||||
"@nestjs/platform-express": "^11.0.1",
|
||||
"@nestjs/platform-express": "^11.1.19",
|
||||
"@nestjs/platform-socket.io": "^11.1.9",
|
||||
"@nestjs/schedule": "^6.0.1",
|
||||
"@nestjs/swagger": "^11.2.3",
|
||||
@@ -44,12 +44,13 @@
|
||||
"@nestjs/throttler": "^6.4.0",
|
||||
"@nestjs/typeorm": "^11.0.0",
|
||||
"@nestjs/websockets": "^11.1.9",
|
||||
"@qdrant/js-client-rest": "^1.17.0",
|
||||
"@types/nodemailer": "^7.0.4",
|
||||
"@willsoto/nestjs-prometheus": "^6.0.2",
|
||||
"ajv": "^8.17.1",
|
||||
"ajv-formats": "^3.0.1",
|
||||
"async-retry": "^1.3.3",
|
||||
"axios": "^1.13.2",
|
||||
"axios": "^1.15.0",
|
||||
"bcrypt": "^6.0.0",
|
||||
"bullmq": "^5.63.2",
|
||||
"cache-manager": "^7.2.5",
|
||||
@@ -81,7 +82,7 @@
|
||||
"zod": "^4.1.13"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@compodoc/compodoc": "^1.1.32",
|
||||
"@compodoc/compodoc": "^1.1.23",
|
||||
"@eslint/eslintrc": "^3.2.0",
|
||||
"@eslint/js": "^9.18.0",
|
||||
"@nestjs/cli": "^11.0.0",
|
||||
|
||||
@@ -52,6 +52,7 @@ import { SearchModule } from './modules/search/search.module';
|
||||
import { AuditLogModule } from './modules/audit-log/audit-log.module';
|
||||
import { MigrationModule } from './modules/migration/migration.module';
|
||||
import { AiModule } from './modules/ai/ai.module';
|
||||
import { RagModule } from './modules/rag/rag.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
@@ -189,6 +190,7 @@ import { AiModule } from './modules/ai/ai.module';
|
||||
AuditLogModule,
|
||||
MigrationModule,
|
||||
AiModule,
|
||||
RagModule,
|
||||
],
|
||||
controllers: [AppController],
|
||||
providers: [
|
||||
|
||||
@@ -2,7 +2,7 @@ import { Test, TestingModule } from '@nestjs/testing';
|
||||
import { AbilityFactory, ScopeContext } from './ability.factory';
|
||||
import { User } from '../../../modules/user/entities/user.entity';
|
||||
import { UserAssignment } from '../../../modules/user/entities/user-assignment.entity';
|
||||
import { Role } from '../../../modules/auth/entities/role.entity';
|
||||
import { Role } from '../../../modules/user/entities/role.entity';
|
||||
|
||||
describe('AbilityFactory', () => {
|
||||
let factory: AbilityFactory;
|
||||
|
||||
@@ -14,6 +14,7 @@ export enum ErrorType {
|
||||
DATABASE_ERROR = 'DATABASE_ERROR',
|
||||
EXTERNAL_SERVICE = 'EXTERNAL_SERVICE',
|
||||
INFRASTRUCTURE = 'INFRASTRUCTURE',
|
||||
SERVICE_UNAVAILABLE = 'SERVICE_UNAVAILABLE', // 503 — ระบบไม่พร้อมให้บริการชั่วคราว (Redlock fail, Redis down)
|
||||
}
|
||||
|
||||
// ระดับความรุนแรงของ Error
|
||||
@@ -49,6 +50,8 @@ export function getStatusCode(type: ErrorType): number {
|
||||
case ErrorType.EXTERNAL_SERVICE:
|
||||
case ErrorType.INFRASTRUCTURE:
|
||||
return HttpStatus.INTERNAL_SERVER_ERROR;
|
||||
case ErrorType.SERVICE_UNAVAILABLE:
|
||||
return HttpStatus.SERVICE_UNAVAILABLE; // 503
|
||||
default:
|
||||
return HttpStatus.INTERNAL_SERVER_ERROR;
|
||||
}
|
||||
@@ -233,3 +236,27 @@ export class DatabaseException extends BaseException {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Service Unavailable (503) - ระบบไม่พร้อมให้บริการชั่วคราว
|
||||
// ADR-021 C1: Redlock Fail-closed — retry ครบแล้ว ยัง acquire ไม่ได้
|
||||
export class ServiceUnavailableException extends BaseException {
|
||||
constructor(
|
||||
code: string,
|
||||
message: string,
|
||||
userMessage?: string,
|
||||
recoveryActions?: string[]
|
||||
) {
|
||||
super(
|
||||
ErrorType.SERVICE_UNAVAILABLE,
|
||||
code,
|
||||
message,
|
||||
userMessage || 'ระบบยุ่งชั่วคราว กรุณาลองใหม่ภายหลัง',
|
||||
ErrorSeverity.HIGH,
|
||||
undefined,
|
||||
recoveryActions || [
|
||||
'รอสักครู่แล้วลองใหม่',
|
||||
'แจ้งผู้ดูแลระบบหากยังพบปัญหา',
|
||||
]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ export {
|
||||
WorkflowException,
|
||||
SystemException,
|
||||
DatabaseException,
|
||||
ServiceUnavailableException,
|
||||
} from './base.exception';
|
||||
|
||||
export type { ValidationErrorDetail, ErrorPayload } from './base.exception';
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
import { BullModule } from '@nestjs/bullmq';
|
||||
import { ScheduleModule } from '@nestjs/schedule'; // ✅ Import
|
||||
import { FileStorageService } from './file-storage.service.js';
|
||||
import { FileStorageController } from './file-storage.controller.js';
|
||||
@@ -12,6 +13,7 @@ import { UserModule } from '../../modules/user/user.module';
|
||||
TypeOrmModule.forFeature([Attachment]),
|
||||
ScheduleModule.forRoot(), // ✅ เปิดใช้งาน Cron Job],
|
||||
UserModule,
|
||||
BullModule.registerQueue({ name: 'rag:ocr' }),
|
||||
],
|
||||
controllers: [FileStorageController],
|
||||
providers: [
|
||||
|
||||
@@ -4,10 +4,13 @@ import {
|
||||
NotFoundException,
|
||||
BadRequestException,
|
||||
Logger,
|
||||
Optional,
|
||||
} from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository, In } from 'typeorm';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { InjectQueue } from '@nestjs/bullmq';
|
||||
import { Queue } from 'bullmq';
|
||||
import * as fs from 'fs-extra';
|
||||
import * as path from 'path';
|
||||
import * as crypto from 'crypto';
|
||||
@@ -24,7 +27,8 @@ export class FileStorageService {
|
||||
constructor(
|
||||
@InjectRepository(Attachment)
|
||||
private attachmentRepository: Repository<Attachment>,
|
||||
private configService: ConfigService
|
||||
private configService: ConfigService,
|
||||
@Optional() @InjectQueue('rag:ocr') private readonly ragOcrQueue?: Queue
|
||||
) {
|
||||
// ใช้ env vars จาก docker-compose สำหรับ Production
|
||||
// ถ้าไม่ได้กำหนดจะ fallback เป็น ./uploads/temp และ ./uploads/permanent
|
||||
@@ -90,7 +94,18 @@ export class FileStorageService {
|
||||
*/
|
||||
async commit(
|
||||
tempIds: string[],
|
||||
options?: { issueDate?: Date; documentType?: string }
|
||||
options?: {
|
||||
issueDate?: Date;
|
||||
documentType?: string;
|
||||
ragMeta?: {
|
||||
docType: string;
|
||||
docNumber: string | null;
|
||||
revision: string | null;
|
||||
projectCode: string;
|
||||
projectPublicId: string;
|
||||
classification: 'PUBLIC' | 'INTERNAL' | 'CONFIDENTIAL';
|
||||
};
|
||||
}
|
||||
): Promise<Attachment[]> {
|
||||
if (!tempIds || tempIds.length === 0) {
|
||||
return [];
|
||||
@@ -149,7 +164,27 @@ export class FileStorageService {
|
||||
att.expiresAt = undefined; // เคลียร์วันหมดอายุ
|
||||
att.referenceDate = effectiveDate; // Save reference date
|
||||
|
||||
committedAttachments.push(await this.attachmentRepository.save(att));
|
||||
const saved = await this.attachmentRepository.save(att);
|
||||
committedAttachments.push(saved);
|
||||
|
||||
if (this.ragOcrQueue && options?.ragMeta) {
|
||||
await this.ragOcrQueue
|
||||
.add(
|
||||
'ocr',
|
||||
{
|
||||
attachmentPublicId: saved.publicId,
|
||||
filePath: saved.filePath,
|
||||
...options.ragMeta,
|
||||
},
|
||||
{ jobId: saved.publicId }
|
||||
)
|
||||
.catch((err: unknown) => {
|
||||
this.logger.error(
|
||||
`Failed to enqueue rag:ocr for ${saved.publicId}`,
|
||||
err instanceof Error ? err.stack : String(err)
|
||||
);
|
||||
});
|
||||
}
|
||||
} else {
|
||||
this.logger.error(`File missing during commit: ${oldPath}`);
|
||||
throw new NotFoundException(
|
||||
|
||||
@@ -10,10 +10,15 @@ import { of, lastValueFrom } from 'rxjs';
|
||||
import { Request } from 'express';
|
||||
import type { Socket } from 'net';
|
||||
|
||||
type MockAuditLogRepo = {
|
||||
create: jest.Mock;
|
||||
save: jest.Mock;
|
||||
};
|
||||
|
||||
describe('AuditLogInterceptor', () => {
|
||||
let interceptor: AuditLogInterceptor;
|
||||
let reflector: Reflector;
|
||||
let auditLogRepo: jest.Mocked<Partial<typeof AuditLog.prototype.constructor>>;
|
||||
let auditLogRepo: MockAuditLogRepo;
|
||||
|
||||
const createMockUser = (userId: number): User => {
|
||||
const user = new User();
|
||||
@@ -55,7 +60,7 @@ describe('AuditLogInterceptor', () => {
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
const mockRepository = {
|
||||
const mockRepository: MockAuditLogRepo = {
|
||||
create: jest.fn().mockReturnValue({}),
|
||||
save: jest.fn().mockResolvedValue({}),
|
||||
};
|
||||
@@ -78,7 +83,7 @@ describe('AuditLogInterceptor', () => {
|
||||
|
||||
interceptor = module.get<AuditLogInterceptor>(AuditLogInterceptor);
|
||||
reflector = module.get<Reflector>(Reflector);
|
||||
auditLogRepo = module.get(getRepositoryToken(AuditLog));
|
||||
auditLogRepo = module.get<MockAuditLogRepo>(getRepositoryToken(AuditLog));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
|
||||
@@ -99,7 +99,17 @@ describe('CorrespondenceController', () => {
|
||||
mockResult
|
||||
);
|
||||
|
||||
const mockReq = { user: { user_id: 1, roles: [] } };
|
||||
const mockReq = {
|
||||
user: {
|
||||
user_id: 1,
|
||||
username: 'testuser',
|
||||
password: 'hashedpassword',
|
||||
email: 'test@example.com',
|
||||
publicId: '019505a1-7c3e-7000-8000-abc123def456',
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
},
|
||||
};
|
||||
(mockCorrespondenceService.findOneByUuid as jest.Mock).mockResolvedValue({
|
||||
id: 1,
|
||||
uuid: 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11',
|
||||
|
||||
@@ -2,6 +2,7 @@ import { Test, TestingModule } from '@nestjs/testing';
|
||||
import { MigrationController } from './migration.controller';
|
||||
import { MigrationService } from './migration.service';
|
||||
import { ImportCorrespondenceDto } from './dto/import-correspondence.dto';
|
||||
import { User } from '../user/entities/user.entity';
|
||||
|
||||
describe('MigrationController', () => {
|
||||
let controller: MigrationController;
|
||||
@@ -32,17 +33,29 @@ describe('MigrationController', () => {
|
||||
|
||||
it('should call importCorrespondence on service', async () => {
|
||||
const dto: ImportCorrespondenceDto = {
|
||||
document_number: 'DOC-001',
|
||||
documentNumber: 'DOC-001',
|
||||
subject: 'Legacy Record',
|
||||
category: 'Correspondence',
|
||||
source_file_path: '/staging_ai/test.pdf',
|
||||
migrated_by: 'SYSTEM_IMPORT',
|
||||
batch_id: 'batch1',
|
||||
project_id: 1,
|
||||
sourceFilePath: '/staging_ai/test.pdf',
|
||||
migratedBy: 'SYSTEM_IMPORT',
|
||||
batchId: 'batch1',
|
||||
projectId: 1,
|
||||
};
|
||||
|
||||
const idempotencyKey = 'key123';
|
||||
const user = { userId: 5 };
|
||||
const user: User = {
|
||||
user_id: 5,
|
||||
username: 'testuser',
|
||||
password: 'hashedpassword',
|
||||
email: 'test@example.com',
|
||||
publicId: '019505a1-7c3e-7000-8000-abc123def456',
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
isActive: true,
|
||||
failedAttempts: 0,
|
||||
primaryOrganizationPublicId: undefined,
|
||||
generatePublicId: jest.fn(),
|
||||
};
|
||||
|
||||
const result = await controller.importCorrespondence(
|
||||
dto,
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
import { Test, TestingModule } from '@nestjs/testing';
|
||||
|
||||
import { IngestionService } from '../ingestion.service';
|
||||
|
||||
const QUEUE_TOKEN = 'BullQueue_rag:ocr';
|
||||
|
||||
const mockOcrQueue = {
|
||||
getJob: jest.fn(),
|
||||
add: jest.fn(),
|
||||
};
|
||||
|
||||
const baseJobData = {
|
||||
attachmentPublicId: 'att-uuid-001',
|
||||
filePath: '/uploads/permanent/CORR/2026/04/file.pdf',
|
||||
docType: 'CORR',
|
||||
docNumber: 'REF-001',
|
||||
revision: null,
|
||||
projectCode: 'PRJ-001',
|
||||
projectPublicId: 'proj-uuid-001',
|
||||
classification: 'INTERNAL' as const,
|
||||
};
|
||||
|
||||
describe('IngestionService', () => {
|
||||
let service: IngestionService;
|
||||
|
||||
beforeEach(async () => {
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
providers: [
|
||||
IngestionService,
|
||||
{ provide: QUEUE_TOKEN, useValue: mockOcrQueue },
|
||||
],
|
||||
}).compile();
|
||||
|
||||
service = module.get<IngestionService>(IngestionService);
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it('should enqueue rag:ocr job with attachmentPublicId as jobId', async () => {
|
||||
mockOcrQueue.getJob.mockResolvedValue(null);
|
||||
mockOcrQueue.add.mockResolvedValue({ id: baseJobData.attachmentPublicId });
|
||||
|
||||
await service.enqueue(baseJobData);
|
||||
|
||||
expect(mockOcrQueue.add).toHaveBeenCalledWith('ocr', baseJobData, {
|
||||
jobId: baseJobData.attachmentPublicId,
|
||||
});
|
||||
});
|
||||
|
||||
it('EC-RAG-001: duplicate enqueue when job is active → second call is no-op (log only)', async () => {
|
||||
const mockJob = { getState: jest.fn().mockResolvedValue('active') };
|
||||
mockOcrQueue.getJob.mockResolvedValue(mockJob);
|
||||
|
||||
await service.enqueue(baseJobData);
|
||||
|
||||
expect(mockOcrQueue.add).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('EC-RAG-001: duplicate enqueue when job is waiting → second call is no-op', async () => {
|
||||
const mockJob = { getState: jest.fn().mockResolvedValue('waiting') };
|
||||
mockOcrQueue.getJob.mockResolvedValue(mockJob);
|
||||
|
||||
await service.enqueue(baseJobData);
|
||||
|
||||
expect(mockOcrQueue.add).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should re-enqueue if job exists but is completed (state=completed)', async () => {
|
||||
const mockJob = { getState: jest.fn().mockResolvedValue('completed') };
|
||||
mockOcrQueue.getJob.mockResolvedValue(mockJob);
|
||||
mockOcrQueue.add.mockResolvedValue({ id: baseJobData.attachmentPublicId });
|
||||
|
||||
await service.enqueue(baseJobData);
|
||||
|
||||
expect(mockOcrQueue.add).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('should re-enqueue if job exists but is failed (state=failed)', async () => {
|
||||
const mockJob = { getState: jest.fn().mockResolvedValue('failed') };
|
||||
mockOcrQueue.getJob.mockResolvedValue(mockJob);
|
||||
mockOcrQueue.add.mockResolvedValue({ id: baseJobData.attachmentPublicId });
|
||||
|
||||
await service.enqueue(baseJobData);
|
||||
|
||||
expect(mockOcrQueue.add).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,206 @@
|
||||
import { Test, TestingModule } from '@nestjs/testing';
|
||||
import { ServiceUnavailableException } from '@nestjs/common';
|
||||
import { getRepositoryToken } from '@nestjs/typeorm';
|
||||
|
||||
import { RagService } from '../rag.service';
|
||||
import { QdrantService } from '../qdrant.service';
|
||||
import { EmbeddingService } from '../embedding.service';
|
||||
import { TyphoonService } from '../typhoon.service';
|
||||
import { IngestionService } from '../ingestion.service';
|
||||
import { DocumentChunk } from '../entities/document-chunk.entity';
|
||||
|
||||
const DEFAULT_REDIS_TOKEN = 'default_IORedisModuleConnectionToken';
|
||||
|
||||
const mockQdrant = {
|
||||
isReady: jest.fn(),
|
||||
hybridSearch: jest.fn(),
|
||||
deleteByDocumentId: jest.fn(),
|
||||
};
|
||||
|
||||
const mockEmbedding = {
|
||||
embed: jest.fn(),
|
||||
};
|
||||
|
||||
const mockTyphoon = {
|
||||
generate: jest.fn(),
|
||||
sanitizeInput: jest.fn((t: string) => t),
|
||||
};
|
||||
|
||||
const mockIngestion = { enqueue: jest.fn() };
|
||||
|
||||
const mockChunkRepo = {
|
||||
count: jest.fn(),
|
||||
delete: jest.fn(),
|
||||
manager: {
|
||||
query: jest.fn(),
|
||||
},
|
||||
};
|
||||
|
||||
const mockRedis = {
|
||||
get: jest.fn(),
|
||||
setex: jest.fn(),
|
||||
};
|
||||
|
||||
describe('RagService', () => {
|
||||
let service: RagService;
|
||||
|
||||
beforeEach(async () => {
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
providers: [
|
||||
RagService,
|
||||
{ provide: QdrantService, useValue: mockQdrant },
|
||||
{ provide: EmbeddingService, useValue: mockEmbedding },
|
||||
{ provide: TyphoonService, useValue: mockTyphoon },
|
||||
{ provide: IngestionService, useValue: mockIngestion },
|
||||
{ provide: getRepositoryToken(DocumentChunk), useValue: mockChunkRepo },
|
||||
{ provide: DEFAULT_REDIS_TOKEN, useValue: mockRedis },
|
||||
],
|
||||
}).compile();
|
||||
|
||||
service = module.get<RagService>(RagService);
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('query()', () => {
|
||||
const dto = {
|
||||
question: 'เอกสารเกี่ยวกับอะไร?',
|
||||
projectPublicId: 'proj-uuid-1234',
|
||||
};
|
||||
const memberPerms: string[] = [];
|
||||
const adminPerms = ['system.manage_all'];
|
||||
|
||||
it('should return answer with citations on PUBLIC cache miss → write cache', async () => {
|
||||
mockQdrant.isReady.mockReturnValue(true);
|
||||
mockRedis.get.mockResolvedValue(null);
|
||||
mockEmbedding.embed.mockResolvedValue(new Array(768).fill(0.1));
|
||||
mockQdrant.hybridSearch.mockResolvedValue([
|
||||
{
|
||||
chunkId: 'chunk-1',
|
||||
publicId: 'att-1',
|
||||
docType: 'CORR',
|
||||
docNumber: 'REF-001',
|
||||
revision: null,
|
||||
projectCode: 'PRJ-001',
|
||||
contentPreview: 'เนื้อหาเอกสาร',
|
||||
score: 0.92,
|
||||
},
|
||||
]);
|
||||
mockTyphoon.generate.mockResolvedValue({
|
||||
answer: 'คำตอบ',
|
||||
usedFallbackModel: false,
|
||||
});
|
||||
|
||||
const result = await service.query(dto, memberPerms);
|
||||
|
||||
expect(result.answer).toBe('คำตอบ');
|
||||
expect(result.citations).toHaveLength(1);
|
||||
expect(result.usedFallbackModel).toBe(false);
|
||||
expect(mockRedis.setex).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('should return cached result without calling Qdrant on cache hit', async () => {
|
||||
mockQdrant.isReady.mockReturnValue(true);
|
||||
const cached = JSON.stringify({
|
||||
answer: 'cached answer',
|
||||
citations: [],
|
||||
confidence: 0.9,
|
||||
usedFallbackModel: false,
|
||||
});
|
||||
mockRedis.get.mockResolvedValue(cached);
|
||||
|
||||
const result = await service.query(dto, memberPerms);
|
||||
|
||||
expect(result.answer).toBe('cached answer');
|
||||
expect(mockQdrant.hybridSearch).not.toHaveBeenCalled();
|
||||
expect(mockEmbedding.embed).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('CONFIDENTIAL: must use Ollama only, skip cache read and write', async () => {
|
||||
mockQdrant.isReady.mockReturnValue(true);
|
||||
mockEmbedding.embed.mockResolvedValue(new Array(768).fill(0.1));
|
||||
mockQdrant.hybridSearch.mockResolvedValue([]);
|
||||
mockTyphoon.generate.mockResolvedValue({
|
||||
answer: 'ลับมาก',
|
||||
usedFallbackModel: true,
|
||||
});
|
||||
|
||||
const result = await service.query(dto, adminPerms);
|
||||
|
||||
expect(mockRedis.get).not.toHaveBeenCalled();
|
||||
expect(mockRedis.setex).not.toHaveBeenCalled();
|
||||
expect(mockTyphoon.generate).toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
true
|
||||
);
|
||||
expect(result.usedFallbackModel).toBe(true);
|
||||
});
|
||||
|
||||
it('collectionReady=false → throw ServiceUnavailableException RAG_NOT_READY', async () => {
|
||||
mockQdrant.isReady.mockReturnValue(false);
|
||||
|
||||
await expect(service.query(dto, memberPerms)).rejects.toThrow(
|
||||
ServiceUnavailableException
|
||||
);
|
||||
});
|
||||
|
||||
it('cross-project cache isolation: same question different projectPublicId → different cache key', async () => {
|
||||
mockQdrant.isReady.mockReturnValue(true);
|
||||
mockRedis.get.mockResolvedValue(null);
|
||||
mockEmbedding.embed.mockResolvedValue(new Array(768).fill(0.1));
|
||||
mockQdrant.hybridSearch.mockResolvedValue([]);
|
||||
mockTyphoon.generate.mockResolvedValue({
|
||||
answer: 'A',
|
||||
usedFallbackModel: false,
|
||||
});
|
||||
|
||||
await service.query(
|
||||
{ question: 'Q?', projectPublicId: 'proj-A' },
|
||||
memberPerms
|
||||
);
|
||||
await service.query(
|
||||
{ question: 'Q?', projectPublicId: 'proj-B' },
|
||||
memberPerms
|
||||
);
|
||||
|
||||
const calls = mockRedis.setex.mock.calls as [string, ...unknown[]][];
|
||||
expect(calls[0][0]).not.toBe(calls[1][0]);
|
||||
});
|
||||
|
||||
it('classification ceiling derived from role, not from request body', async () => {
|
||||
mockQdrant.isReady.mockReturnValue(true);
|
||||
mockRedis.get.mockResolvedValue(null);
|
||||
mockEmbedding.embed.mockResolvedValue(new Array(768).fill(0.1));
|
||||
mockQdrant.hybridSearch.mockResolvedValue([]);
|
||||
mockTyphoon.generate.mockResolvedValue({
|
||||
anwer: 'ok',
|
||||
usedFallbackModel: false,
|
||||
});
|
||||
|
||||
await service.query(dto, memberPerms);
|
||||
expect(mockQdrant.hybridSearch).toHaveBeenCalledWith(
|
||||
expect.any(Array),
|
||||
dto.projectPublicId,
|
||||
'INTERNAL',
|
||||
20
|
||||
);
|
||||
|
||||
jest.clearAllMocks();
|
||||
mockQdrant.isReady.mockReturnValue(true);
|
||||
mockRedis.get.mockResolvedValue(null);
|
||||
mockEmbedding.embed.mockResolvedValue(new Array(768).fill(0.1));
|
||||
mockQdrant.hybridSearch.mockResolvedValue([]);
|
||||
mockTyphoon.generate.mockResolvedValue({
|
||||
answer: 'ok',
|
||||
usedFallbackModel: true,
|
||||
});
|
||||
|
||||
await service.query(dto, adminPerms);
|
||||
expect(mockQdrant.hybridSearch).toHaveBeenCalledWith(
|
||||
expect.any(Array),
|
||||
dto.projectPublicId,
|
||||
'CONFIDENTIAL',
|
||||
20
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,11 @@
|
||||
import { IsNotEmpty, IsString, IsUUID, MaxLength } from 'class-validator';
|
||||
|
||||
export class RagQueryDto {
|
||||
@IsString()
|
||||
@IsNotEmpty()
|
||||
@MaxLength(500)
|
||||
question!: string;
|
||||
|
||||
@IsUUID()
|
||||
projectPublicId!: string;
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
export interface RagCitation {
|
||||
chunkId: string;
|
||||
docNumber: string | null;
|
||||
docType: string;
|
||||
revision: string | null;
|
||||
snippet: string;
|
||||
score: number;
|
||||
}
|
||||
|
||||
export class RagResponseDto {
|
||||
answer!: string;
|
||||
citations!: RagCitation[];
|
||||
confidence!: number;
|
||||
usedFallbackModel!: boolean;
|
||||
cachedAt?: string;
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import axios from 'axios';
|
||||
|
||||
@Injectable()
|
||||
export class EmbeddingService {
|
||||
private readonly logger = new Logger(EmbeddingService.name);
|
||||
private readonly ollamaUrl: string;
|
||||
private readonly model: string;
|
||||
|
||||
constructor(private readonly configService: ConfigService) {
|
||||
this.ollamaUrl = this.configService.get<string>(
|
||||
'OLLAMA_URL',
|
||||
'http://localhost:11434'
|
||||
);
|
||||
this.model = this.configService.get<string>(
|
||||
'OLLAMA_EMBED_MODEL',
|
||||
'nomic-embed-text'
|
||||
);
|
||||
}
|
||||
|
||||
async embed(text: string): Promise<number[]> {
|
||||
try {
|
||||
const response = await axios.post<{ embedding: number[] }>(
|
||||
`${this.ollamaUrl}/api/embeddings`,
|
||||
{ model: this.model, prompt: text },
|
||||
{ timeout: 30000 }
|
||||
);
|
||||
return response.data.embedding;
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
'Embedding failed',
|
||||
err instanceof Error ? err.stack : String(err)
|
||||
);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async embedBatch(texts: string[]): Promise<number[][]> {
|
||||
return Promise.all(texts.map((t) => this.embed(t)));
|
||||
}
|
||||
|
||||
getModelName(): string {
|
||||
return this.model;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import { Column, CreateDateColumn, Entity, PrimaryColumn } from 'typeorm';
|
||||
|
||||
@Entity('document_chunks')
|
||||
export class DocumentChunk {
|
||||
@PrimaryColumn({ type: 'char', length: 36 })
|
||||
id!: string;
|
||||
|
||||
@Column({ type: 'char', length: 36, name: 'document_id' })
|
||||
documentId!: string;
|
||||
|
||||
@Column({ name: 'chunk_index' })
|
||||
chunkIndex!: number;
|
||||
|
||||
@Column({ type: 'text' })
|
||||
content!: string;
|
||||
|
||||
@Column({ length: 20, name: 'doc_type' })
|
||||
docType!: string;
|
||||
|
||||
@Column({ length: 100, name: 'doc_number', nullable: true })
|
||||
docNumber!: string | null;
|
||||
|
||||
@Column({ length: 20, nullable: true })
|
||||
revision!: string | null;
|
||||
|
||||
@Column({ length: 50, name: 'project_code' })
|
||||
projectCode!: string;
|
||||
|
||||
@Column({ length: 36, name: 'project_public_id' })
|
||||
projectPublicId!: string;
|
||||
|
||||
@Column({
|
||||
type: 'enum',
|
||||
enum: ['PUBLIC', 'INTERNAL', 'CONFIDENTIAL'],
|
||||
default: 'INTERNAL',
|
||||
})
|
||||
classification!: 'PUBLIC' | 'INTERNAL' | 'CONFIDENTIAL';
|
||||
|
||||
@Column({ length: 20, nullable: true })
|
||||
version!: string | null;
|
||||
|
||||
@Column({ length: 100, name: 'embedding_model', default: 'nomic-embed-text' })
|
||||
embeddingModel!: string;
|
||||
|
||||
@CreateDateColumn({ name: 'created_at', precision: 3 })
|
||||
createdAt!: Date;
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { InjectQueue } from '@nestjs/bullmq';
|
||||
import { Queue } from 'bullmq';
|
||||
|
||||
import { OcrJobData } from './processors/ocr.processor';
|
||||
|
||||
@Injectable()
|
||||
export class IngestionService {
|
||||
private readonly logger = new Logger(IngestionService.name);
|
||||
|
||||
constructor(@InjectQueue('rag:ocr') private readonly ocrQueue: Queue) {}
|
||||
|
||||
async enqueue(data: OcrJobData): Promise<void> {
|
||||
const jobId = data.attachmentPublicId;
|
||||
|
||||
const existing = await this.ocrQueue.getJob(jobId);
|
||||
if (existing) {
|
||||
const state = await existing.getState();
|
||||
if (state === 'active' || state === 'waiting' || state === 'delayed') {
|
||||
this.logger.log(
|
||||
`rag:ocr job already queued for ${jobId} (state: ${state})`
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
await this.ocrQueue.add('ocr', data, { jobId });
|
||||
this.logger.log(`Enqueued rag:ocr for attachment ${jobId}`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
import { Processor, WorkerHost } from '@nestjs/bullmq';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository } from 'typeorm';
|
||||
import { Job } from 'bullmq';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
|
||||
import { EmbeddingService } from '../embedding.service';
|
||||
import { QdrantService, VectorMetadata } from '../qdrant.service';
|
||||
import { DocumentChunk } from '../entities/document-chunk.entity';
|
||||
import { EmbeddingJobData } from './thai-preprocess.processor';
|
||||
|
||||
const CHUNK_SIZE = 512;
|
||||
const CHUNK_OVERLAP = 50;
|
||||
|
||||
@Processor('rag:embedding')
|
||||
export class EmbeddingProcessor extends WorkerHost {
|
||||
private readonly logger = new Logger(EmbeddingProcessor.name);
|
||||
|
||||
constructor(
|
||||
private readonly embeddingService: EmbeddingService,
|
||||
private readonly qdrantService: QdrantService,
|
||||
@InjectRepository(DocumentChunk)
|
||||
private readonly chunkRepo: Repository<DocumentChunk>
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
async process(job: Job<EmbeddingJobData>): Promise<void> {
|
||||
const {
|
||||
attachmentPublicId,
|
||||
normalizedText,
|
||||
docType,
|
||||
docNumber,
|
||||
revision,
|
||||
projectCode,
|
||||
projectPublicId,
|
||||
classification,
|
||||
} = job.data;
|
||||
|
||||
const chunks = this.chunkText(normalizedText);
|
||||
const model = this.embeddingService.getModelName();
|
||||
|
||||
const upsertPoints: Parameters<QdrantService['upsertBatch']>[0] = [];
|
||||
const chunkEntities: DocumentChunk[] = [];
|
||||
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
const chunkId = uuidv4();
|
||||
const vector = await this.embeddingService.embed(chunks[i]);
|
||||
|
||||
const payload: VectorMetadata = {
|
||||
chunk_id: chunkId,
|
||||
public_id: attachmentPublicId,
|
||||
project_public_id: projectPublicId,
|
||||
doc_type: docType,
|
||||
doc_number: docNumber,
|
||||
revision,
|
||||
project_code: projectCode,
|
||||
classification,
|
||||
content_preview: chunks[i].slice(0, 500),
|
||||
embedding_model: model,
|
||||
};
|
||||
|
||||
upsertPoints.push({ id: chunkId, vector, payload });
|
||||
|
||||
const entity = this.chunkRepo.create({
|
||||
id: chunkId,
|
||||
documentId: attachmentPublicId,
|
||||
chunkIndex: i,
|
||||
content: chunks[i],
|
||||
docType,
|
||||
docNumber,
|
||||
revision,
|
||||
projectCode,
|
||||
projectPublicId,
|
||||
classification,
|
||||
embeddingModel: model,
|
||||
});
|
||||
chunkEntities.push(entity);
|
||||
}
|
||||
|
||||
if (upsertPoints.length > 0) {
|
||||
await this.qdrantService.upsertBatch(upsertPoints);
|
||||
await this.chunkRepo.save(chunkEntities);
|
||||
}
|
||||
|
||||
await this.chunkRepo.manager.query(
|
||||
`UPDATE attachments SET rag_status = 'INDEXED', rag_last_error = NULL WHERE public_id = ?`,
|
||||
[attachmentPublicId]
|
||||
);
|
||||
|
||||
this.logger.log(
|
||||
`Embedded ${chunks.length} chunks for ${attachmentPublicId}`
|
||||
);
|
||||
}
|
||||
|
||||
private chunkText(text: string): string[] {
|
||||
const words = text.split(/\s+/);
|
||||
const chunks: string[] = [];
|
||||
let start = 0;
|
||||
|
||||
while (start < words.length) {
|
||||
const end = Math.min(start + CHUNK_SIZE, words.length);
|
||||
chunks.push(words.slice(start, end).join(' '));
|
||||
start += CHUNK_SIZE - CHUNK_OVERLAP;
|
||||
}
|
||||
|
||||
return chunks.filter((c) => c.trim().length > 0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
import { Processor, WorkerHost } from '@nestjs/bullmq';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository } from 'typeorm';
|
||||
import { Job } from 'bullmq';
|
||||
import * as fs from 'fs';
|
||||
import { InjectQueue } from '@nestjs/bullmq';
|
||||
import { Queue } from 'bullmq';
|
||||
|
||||
import { DocumentChunk } from '../entities/document-chunk.entity';
|
||||
|
||||
export interface OcrJobData {
|
||||
attachmentPublicId: string;
|
||||
filePath: string;
|
||||
docType: string;
|
||||
docNumber: string | null;
|
||||
revision: string | null;
|
||||
projectCode: string;
|
||||
projectPublicId: string;
|
||||
classification: 'PUBLIC' | 'INTERNAL' | 'CONFIDENTIAL';
|
||||
}
|
||||
|
||||
@Processor('rag:ocr')
|
||||
export class OcrProcessor extends WorkerHost {
|
||||
private readonly logger = new Logger(OcrProcessor.name);
|
||||
|
||||
constructor(
|
||||
@InjectQueue('rag:thai-preprocess') private readonly thaiQueue: Queue,
|
||||
@InjectRepository(DocumentChunk)
|
||||
private readonly chunkRepo: Repository<DocumentChunk>
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
async process(job: Job<OcrJobData>): Promise<void> {
|
||||
const { attachmentPublicId, filePath } = job.data;
|
||||
|
||||
const existing = await this.chunkRepo.count({
|
||||
where: { documentId: attachmentPublicId },
|
||||
});
|
||||
if (existing > 0) {
|
||||
this.logger.log(
|
||||
`rag:ocr job already indexed for ${attachmentPublicId}, skipping`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
await this.chunkRepo.manager.query(
|
||||
`UPDATE attachments SET rag_status = 'PROCESSING' WHERE public_id = ?`,
|
||||
[attachmentPublicId]
|
||||
);
|
||||
|
||||
let rawText: string;
|
||||
try {
|
||||
rawText = fs.readFileSync(filePath, 'utf-8');
|
||||
} catch {
|
||||
rawText = `[binary:${attachmentPublicId}]`;
|
||||
}
|
||||
|
||||
await this.thaiQueue.add(
|
||||
'preprocess',
|
||||
{ ...job.data, rawText },
|
||||
{ jobId: `thai:${attachmentPublicId}` }
|
||||
);
|
||||
|
||||
this.logger.log(`OCR enqueued thai-preprocess for ${attachmentPublicId}`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
import { Processor, WorkerHost, InjectQueue } from '@nestjs/bullmq';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { Queue, Job } from 'bullmq';
|
||||
import axios from 'axios';
|
||||
|
||||
import { OcrJobData } from './ocr.processor';
|
||||
|
||||
export interface ThaiPreprocessJobData extends OcrJobData {
|
||||
rawText: string;
|
||||
}
|
||||
|
||||
export interface EmbeddingJobData extends ThaiPreprocessJobData {
|
||||
normalizedText: string;
|
||||
}
|
||||
|
||||
@Processor('rag:thai-preprocess')
|
||||
export class ThaiPreprocessProcessor extends WorkerHost {
|
||||
private readonly logger = new Logger(ThaiPreprocessProcessor.name);
|
||||
private readonly thaiUrl: string;
|
||||
|
||||
constructor(
|
||||
private readonly configService: ConfigService,
|
||||
@InjectQueue('rag:embedding') private readonly embeddingQueue: Queue
|
||||
) {
|
||||
super();
|
||||
this.thaiUrl = this.configService.get<string>(
|
||||
'THAI_PREPROCESS_URL',
|
||||
'http://localhost:8765'
|
||||
);
|
||||
}
|
||||
|
||||
async process(job: Job<ThaiPreprocessJobData>): Promise<void> {
|
||||
const { rawText, attachmentPublicId } = job.data;
|
||||
|
||||
let normalizedText = rawText;
|
||||
try {
|
||||
const response = await axios.post<{ normalized: string }>(
|
||||
`${this.thaiUrl}/normalize`,
|
||||
{ text: rawText },
|
||||
{ timeout: 30000 }
|
||||
);
|
||||
normalizedText = response.data.normalized ?? rawText;
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`Thai preprocess failed for ${attachmentPublicId}, using raw text: ${err instanceof Error ? err.message : String(err)}`
|
||||
);
|
||||
}
|
||||
|
||||
await this.embeddingQueue.add(
|
||||
'embed',
|
||||
{ ...job.data, normalizedText } as EmbeddingJobData,
|
||||
{ jobId: `embed:${attachmentPublicId}` }
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { QdrantClient } from '@qdrant/js-client-rest';
|
||||
|
||||
export interface VectorMetadata extends Record<string, unknown> {
|
||||
chunk_id: string;
|
||||
public_id: string;
|
||||
project_public_id: string;
|
||||
doc_type: string;
|
||||
doc_number: string | null;
|
||||
revision: string | null;
|
||||
project_code: string;
|
||||
classification: 'PUBLIC' | 'INTERNAL' | 'CONFIDENTIAL';
|
||||
content_preview: string;
|
||||
embedding_model: string;
|
||||
}
|
||||
|
||||
export interface HybridSearchResult {
|
||||
chunkId: string;
|
||||
publicId: string;
|
||||
docType: string;
|
||||
docNumber: string | null;
|
||||
revision: string | null;
|
||||
projectCode: string;
|
||||
contentPreview: string;
|
||||
score: number;
|
||||
}
|
||||
|
||||
const COLLECTION_NAME = 'lcbp3_vectors';
|
||||
const VECTOR_SIZE = 768;
|
||||
|
||||
@Injectable()
|
||||
export class QdrantService implements OnModuleInit {
|
||||
private readonly logger = new Logger(QdrantService.name);
|
||||
private client: QdrantClient;
|
||||
private collectionReady = false;
|
||||
|
||||
constructor(private readonly configService: ConfigService) {
|
||||
const url = this.configService.get<string>(
|
||||
'QDRANT_URL',
|
||||
'http://localhost:6333'
|
||||
);
|
||||
this.client = new QdrantClient({ url });
|
||||
}
|
||||
|
||||
async onModuleInit(): Promise<void> {
|
||||
try {
|
||||
await this.initCollection();
|
||||
this.collectionReady = true;
|
||||
this.logger.log(`Qdrant collection '${COLLECTION_NAME}' ready`);
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
'Qdrant collection init failed — RAG queries will return 503',
|
||||
err instanceof Error ? err.stack : String(err)
|
||||
);
|
||||
this.collectionReady = false;
|
||||
}
|
||||
}
|
||||
|
||||
isReady(): boolean {
|
||||
return this.collectionReady;
|
||||
}
|
||||
|
||||
private async initCollection(): Promise<void> {
|
||||
const collections = await this.client.getCollections();
|
||||
const exists = collections.collections.some(
|
||||
(c) => c.name === COLLECTION_NAME
|
||||
);
|
||||
|
||||
if (!exists) {
|
||||
await this.client.createCollection(COLLECTION_NAME, {
|
||||
vectors: { size: VECTOR_SIZE, distance: 'Cosine' },
|
||||
hnsw_config: {
|
||||
payload_m: 16,
|
||||
m: 0,
|
||||
},
|
||||
optimizers_config: { indexing_threshold: 10000 },
|
||||
});
|
||||
this.logger.log(`Created Qdrant collection '${COLLECTION_NAME}'`);
|
||||
|
||||
await this.client.createPayloadIndex(COLLECTION_NAME, {
|
||||
field_name: 'project_public_id',
|
||||
field_schema: { type: 'keyword', is_tenant: true } as Parameters<
|
||||
QdrantClient['createPayloadIndex']
|
||||
>[1]['field_schema'],
|
||||
});
|
||||
await this.client.createPayloadIndex(COLLECTION_NAME, {
|
||||
field_name: 'classification',
|
||||
field_schema: 'keyword',
|
||||
});
|
||||
await this.client.createPayloadIndex(COLLECTION_NAME, {
|
||||
field_name: 'doc_type',
|
||||
field_schema: 'keyword',
|
||||
});
|
||||
await this.client.createPayloadIndex(COLLECTION_NAME, {
|
||||
field_name: 'doc_number',
|
||||
field_schema: 'keyword',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async upsertBatch(
|
||||
points: Array<{ id: string; vector: number[]; payload: VectorMetadata }>
|
||||
): Promise<void> {
|
||||
await this.client.upsert(COLLECTION_NAME, {
|
||||
wait: true,
|
||||
points: points.map((p) => ({
|
||||
id: p.id,
|
||||
vector: p.vector,
|
||||
payload: p.payload,
|
||||
})),
|
||||
});
|
||||
}
|
||||
|
||||
async hybridSearch(
|
||||
queryVector: number[],
|
||||
|
||||
projectPublicId: string,
|
||||
classificationCeiling: 'PUBLIC' | 'INTERNAL' | 'CONFIDENTIAL',
|
||||
topK: number
|
||||
): Promise<HybridSearchResult[]> {
|
||||
const classificationValues = this.getAllowedClassifications(
|
||||
classificationCeiling
|
||||
);
|
||||
|
||||
const vectorResults = await this.client.search(COLLECTION_NAME, {
|
||||
vector: queryVector,
|
||||
limit: topK,
|
||||
filter: {
|
||||
must: [
|
||||
{ key: 'project_public_id', match: { value: projectPublicId } },
|
||||
{ key: 'classification', match: { any: classificationValues } },
|
||||
],
|
||||
},
|
||||
with_payload: true,
|
||||
});
|
||||
|
||||
return vectorResults.map((r) => {
|
||||
const payload = r.payload as unknown as VectorMetadata;
|
||||
return {
|
||||
chunkId: payload.chunk_id,
|
||||
publicId: payload.public_id,
|
||||
docType: payload.doc_type,
|
||||
docNumber: payload.doc_number,
|
||||
revision: payload.revision,
|
||||
projectCode: payload.project_code,
|
||||
contentPreview: payload.content_preview,
|
||||
score: r.score,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async deleteByDocumentId(documentId: string): Promise<void> {
|
||||
await this.client.delete(COLLECTION_NAME, {
|
||||
wait: true,
|
||||
filter: {
|
||||
must: [{ key: 'public_id', match: { value: documentId } }],
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async forceInitCollection(): Promise<void> {
|
||||
await this.initCollection();
|
||||
this.collectionReady = true;
|
||||
this.logger.log(`Qdrant collection force-initialized`);
|
||||
}
|
||||
|
||||
private getAllowedClassifications(
|
||||
ceiling: 'PUBLIC' | 'INTERNAL' | 'CONFIDENTIAL'
|
||||
): string[] {
|
||||
const order: Array<'PUBLIC' | 'INTERNAL' | 'CONFIDENTIAL'> = [
|
||||
'PUBLIC',
|
||||
'INTERNAL',
|
||||
'CONFIDENTIAL',
|
||||
];
|
||||
const ceilIdx = order.indexOf(ceiling);
|
||||
return order.slice(0, ceilIdx + 1);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
import {
|
||||
Body,
|
||||
Controller,
|
||||
Delete,
|
||||
Get,
|
||||
Headers,
|
||||
HttpCode,
|
||||
HttpStatus,
|
||||
Logger,
|
||||
Param,
|
||||
Post,
|
||||
UseGuards,
|
||||
} from '@nestjs/common';
|
||||
import { ApiBearerAuth, ApiOperation, ApiTags } from '@nestjs/swagger';
|
||||
import { Throttle } from '@nestjs/throttler';
|
||||
|
||||
import { CurrentUser } from '../../common/decorators/current-user.decorator';
|
||||
import { RequirePermission } from '../../common/decorators/require-permission.decorator';
|
||||
import { JwtAuthGuard } from '../../common/guards/jwt-auth.guard';
|
||||
import { RbacGuard } from '../../common/guards/rbac.guard';
|
||||
import { ParseUuidPipe } from '../../common/pipes/parse-uuid.pipe';
|
||||
import { UserService } from '../user/user.service';
|
||||
import { User } from '../user/entities/user.entity';
|
||||
import { RagQueryDto } from './dto/rag-query.dto';
|
||||
import { RagService } from './rag.service';
|
||||
|
||||
@ApiTags('RAG')
|
||||
@ApiBearerAuth()
|
||||
@UseGuards(JwtAuthGuard, RbacGuard)
|
||||
@Throttle({ default: { limit: 30, ttl: 60000 } })
|
||||
@Controller('rag')
|
||||
export class RagController {
|
||||
private readonly logger = new Logger(RagController.name);
|
||||
|
||||
constructor(
|
||||
private readonly ragService: RagService,
|
||||
private readonly userService: UserService
|
||||
) {}
|
||||
|
||||
@Post('query')
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@ApiOperation({ summary: 'RAG Q&A — ค้นหาคำตอบจากเอกสารโครงการ' })
|
||||
@RequirePermission('rag.query')
|
||||
async query(
|
||||
@Body() dto: RagQueryDto,
|
||||
@CurrentUser() user: User,
|
||||
@Headers('Idempotency-Key') idempotencyKey: string
|
||||
) {
|
||||
if (!idempotencyKey) {
|
||||
this.logger.warn(`Missing Idempotency-Key from user ${user.user_id}`);
|
||||
}
|
||||
|
||||
const permissions = await this.userService.getUserPermissions(user.user_id);
|
||||
return this.ragService.query(dto, permissions);
|
||||
}
|
||||
|
||||
@Get('status/:attachmentId')
|
||||
@ApiOperation({ summary: 'ดูสถานะ RAG ingestion ของ attachment' })
|
||||
@RequirePermission('rag.query')
|
||||
async getStatus(@Param('attachmentId', ParseUuidPipe) attachmentId: string) {
|
||||
return this.ragService.getStatus(attachmentId);
|
||||
}
|
||||
|
||||
@Post('ingest/:attachmentId')
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@ApiOperation({ summary: 'Re-ingest attachment ที่ FAILED (Admin only)' })
|
||||
@RequirePermission('rag.manage')
|
||||
async reIngest(@Param('attachmentId', ParseUuidPipe) attachmentId: string) {
|
||||
await this.ragService.reIngest(attachmentId);
|
||||
return { message: 'Re-ingestion queued' };
|
||||
}
|
||||
|
||||
@Delete('vectors/:attachmentId')
|
||||
@HttpCode(HttpStatus.NO_CONTENT)
|
||||
@ApiOperation({ summary: 'ลบ vectors ของ attachment ออกจาก Qdrant' })
|
||||
@RequirePermission('rag.manage')
|
||||
async deleteVectors(
|
||||
@Param('attachmentId', ParseUuidPipe) attachmentId: string
|
||||
) {
|
||||
await this.ragService.deleteVectors(attachmentId);
|
||||
}
|
||||
|
||||
@Post('admin/init-collection')
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@ApiOperation({
|
||||
summary: 'T038: Init Qdrant collection lcbp3_vectors (admin only)',
|
||||
})
|
||||
@RequirePermission('rag.manage')
|
||||
async initCollection() {
|
||||
await this.ragService.initCollection();
|
||||
return { message: 'Qdrant collection initialized' };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
import { BullModule } from '@nestjs/bullmq';
|
||||
import { ConfigModule } from '@nestjs/config';
|
||||
|
||||
import { DocumentChunk } from './entities/document-chunk.entity';
|
||||
import { EmbeddingService } from './embedding.service';
|
||||
import { QdrantService } from './qdrant.service';
|
||||
import { TyphoonService } from './typhoon.service';
|
||||
import { RagService } from './rag.service';
|
||||
import { RagController } from './rag.controller';
|
||||
import { IngestionService } from './ingestion.service';
|
||||
import { OcrProcessor } from './processors/ocr.processor';
|
||||
import { ThaiPreprocessProcessor } from './processors/thai-preprocess.processor';
|
||||
import { EmbeddingProcessor } from './processors/embedding.processor';
|
||||
import { UserModule } from '../user/user.module';
|
||||
|
||||
const DLQ_DEFAULTS = {
|
||||
attempts: 3,
|
||||
backoff: { type: 'exponential' as const, delay: 2000 },
|
||||
removeOnComplete: 100,
|
||||
removeOnFail: 200,
|
||||
};
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
ConfigModule,
|
||||
UserModule,
|
||||
TypeOrmModule.forFeature([DocumentChunk]),
|
||||
BullModule.registerQueue(
|
||||
{ name: 'rag:ocr', defaultJobOptions: DLQ_DEFAULTS },
|
||||
{ name: 'rag:thai-preprocess', defaultJobOptions: DLQ_DEFAULTS },
|
||||
{ name: 'rag:embedding', defaultJobOptions: DLQ_DEFAULTS }
|
||||
),
|
||||
],
|
||||
controllers: [RagController],
|
||||
providers: [
|
||||
EmbeddingService,
|
||||
QdrantService,
|
||||
TyphoonService,
|
||||
RagService,
|
||||
IngestionService,
|
||||
OcrProcessor,
|
||||
ThaiPreprocessProcessor,
|
||||
EmbeddingProcessor,
|
||||
],
|
||||
exports: [
|
||||
EmbeddingService,
|
||||
QdrantService,
|
||||
TyphoonService,
|
||||
RagService,
|
||||
IngestionService,
|
||||
],
|
||||
})
|
||||
export class RagModule {}
|
||||
@@ -0,0 +1,255 @@
|
||||
import {
|
||||
Injectable,
|
||||
Logger,
|
||||
ServiceUnavailableException,
|
||||
BadRequestException,
|
||||
} from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository } from 'typeorm';
|
||||
import { InjectRedis } from '@nestjs-modules/ioredis';
|
||||
import Redis from 'ioredis';
|
||||
import { createHash } from 'crypto';
|
||||
|
||||
import { QdrantService } from './qdrant.service';
|
||||
import { EmbeddingService } from './embedding.service';
|
||||
import { TyphoonService } from './typhoon.service';
|
||||
import { IngestionService } from './ingestion.service';
|
||||
import { DocumentChunk } from './entities/document-chunk.entity';
|
||||
import { RagQueryDto } from './dto/rag-query.dto';
|
||||
import { RagResponseDto, RagCitation } from './dto/rag-response.dto';
|
||||
|
||||
const CACHE_TTL_SECONDS = 300;
|
||||
const PROMPT_CONTEXT_LIMIT = 3000;
|
||||
|
||||
@Injectable()
|
||||
export class RagService {
|
||||
private readonly logger = new Logger(RagService.name);
|
||||
|
||||
constructor(
|
||||
private readonly qdrant: QdrantService,
|
||||
private readonly embedding: EmbeddingService,
|
||||
private readonly typhoon: TyphoonService,
|
||||
private readonly ingestionService: IngestionService,
|
||||
@InjectRepository(DocumentChunk)
|
||||
private readonly chunkRepo: Repository<DocumentChunk>,
|
||||
@InjectRedis() private readonly redis: Redis
|
||||
) {}
|
||||
|
||||
async query(
|
||||
dto: RagQueryDto,
|
||||
userPermissions: string[]
|
||||
): Promise<RagResponseDto> {
|
||||
const { question, projectPublicId } = dto;
|
||||
|
||||
const classificationCeiling =
|
||||
this.deriveClassificationCeiling(userPermissions);
|
||||
const isConfidential = classificationCeiling === 'CONFIDENTIAL';
|
||||
|
||||
if (!this.qdrant.isReady()) {
|
||||
throw new ServiceUnavailableException('RAG_NOT_READY');
|
||||
}
|
||||
|
||||
const cacheKey = this.buildCacheKey(
|
||||
question,
|
||||
projectPublicId,
|
||||
classificationCeiling
|
||||
);
|
||||
|
||||
if (!isConfidential) {
|
||||
const cached = await this.redis.get(cacheKey);
|
||||
if (cached) {
|
||||
const parsed = JSON.parse(cached) as RagResponseDto;
|
||||
parsed.cachedAt = new Date().toISOString();
|
||||
return parsed;
|
||||
}
|
||||
}
|
||||
|
||||
const queryVector = await this.embedding.embed(question);
|
||||
const topK = 20;
|
||||
|
||||
const results = await this.qdrant.hybridSearch(
|
||||
queryVector,
|
||||
projectPublicId,
|
||||
classificationCeiling,
|
||||
topK
|
||||
);
|
||||
|
||||
const reranked = results.sort((a, b) => b.score - a.score).slice(0, 5);
|
||||
|
||||
const context = this.buildContext(reranked);
|
||||
|
||||
const safeQuestion = this.typhoon.sanitizeInput(question);
|
||||
const prompt = this.buildPrompt(safeQuestion, context);
|
||||
|
||||
const { answer, usedFallbackModel } = await this.typhoon.generate(
|
||||
prompt,
|
||||
isConfidential
|
||||
);
|
||||
|
||||
const citations: RagCitation[] = reranked.map((r) => ({
|
||||
chunkId: r.chunkId,
|
||||
docNumber: r.docNumber,
|
||||
docType: r.docType,
|
||||
revision: r.revision,
|
||||
snippet: r.contentPreview.slice(0, 200),
|
||||
score: r.score,
|
||||
}));
|
||||
|
||||
const confidence = reranked.length > 0 ? reranked[0].score : 0;
|
||||
|
||||
const response: RagResponseDto = {
|
||||
answer,
|
||||
citations,
|
||||
confidence,
|
||||
usedFallbackModel,
|
||||
};
|
||||
|
||||
if (!isConfidential) {
|
||||
await this.redis.setex(
|
||||
cacheKey,
|
||||
CACHE_TTL_SECONDS,
|
||||
JSON.stringify(response)
|
||||
);
|
||||
}
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
async getStatus(
|
||||
attachmentPublicId: string
|
||||
): Promise<{ ragStatus: string; chunkCount: number }> {
|
||||
const chunkCount = await this.chunkRepo.count({
|
||||
where: { documentId: attachmentPublicId },
|
||||
});
|
||||
|
||||
const result = await this.chunkRepo.manager.query<{ rag_status: string }[]>(
|
||||
`SELECT rag_status FROM attachments WHERE public_id = ? LIMIT 1`,
|
||||
[attachmentPublicId]
|
||||
);
|
||||
|
||||
const ragStatus = result[0]?.rag_status ?? 'PENDING';
|
||||
return { ragStatus, chunkCount };
|
||||
}
|
||||
|
||||
async reIngest(attachmentPublicId: string): Promise<void> {
|
||||
const statusResult = await this.chunkRepo.manager.query<
|
||||
{ rag_status: string; file_path: string }[]
|
||||
>(
|
||||
`SELECT rag_status, file_path FROM attachments WHERE public_id = ? LIMIT 1`,
|
||||
[attachmentPublicId]
|
||||
);
|
||||
|
||||
const current = statusResult[0]?.rag_status;
|
||||
if (current !== 'FAILED') {
|
||||
throw new BadRequestException(
|
||||
`Cannot re-ingest: current status is '${current ?? 'unknown'}', expected 'FAILED'`
|
||||
);
|
||||
}
|
||||
|
||||
const sample = await this.chunkRepo.findOne({
|
||||
where: { documentId: attachmentPublicId },
|
||||
});
|
||||
|
||||
await this.chunkRepo.delete({ documentId: attachmentPublicId });
|
||||
|
||||
try {
|
||||
await this.qdrant.deleteByDocumentId(attachmentPublicId);
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
`Qdrant delete failed for ${attachmentPublicId} — continuing`,
|
||||
err instanceof Error ? err.stack : String(err)
|
||||
);
|
||||
}
|
||||
|
||||
await this.chunkRepo.manager.query(
|
||||
`UPDATE attachments SET rag_status = 'PENDING', rag_last_error = NULL WHERE public_id = ?`,
|
||||
[attachmentPublicId]
|
||||
);
|
||||
|
||||
if (sample) {
|
||||
await this.ingestionService.enqueue({
|
||||
attachmentPublicId,
|
||||
filePath: statusResult[0]?.file_path ?? '',
|
||||
docType: sample.docType,
|
||||
docNumber: sample.docNumber,
|
||||
revision: sample.revision,
|
||||
projectCode: sample.projectCode,
|
||||
projectPublicId: sample.projectPublicId,
|
||||
classification: sample.classification,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async initCollection(): Promise<void> {
|
||||
await this.qdrant.onModuleInit();
|
||||
}
|
||||
|
||||
async deleteVectors(attachmentPublicId: string): Promise<void> {
|
||||
await this.chunkRepo.delete({ documentId: attachmentPublicId });
|
||||
try {
|
||||
await this.qdrant.deleteByDocumentId(attachmentPublicId);
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
`Qdrant delete failed for ${attachmentPublicId}`,
|
||||
err instanceof Error ? err.stack : String(err)
|
||||
);
|
||||
}
|
||||
await this.chunkRepo.manager.query(
|
||||
`UPDATE attachments SET rag_status = 'PENDING', rag_last_error = NULL WHERE public_id = ?`,
|
||||
[attachmentPublicId]
|
||||
);
|
||||
}
|
||||
|
||||
buildContext(
|
||||
results: Array<{
|
||||
docType: string;
|
||||
docNumber: string | null;
|
||||
revision: string | null;
|
||||
contentPreview: string;
|
||||
}>
|
||||
): string {
|
||||
let context = '';
|
||||
for (const r of results) {
|
||||
const header = `[${r.docType}${r.docNumber ? ` - ${r.docNumber}` : ''}${r.revision ? ` - ${r.revision}` : ''}]`;
|
||||
const snippet = `${header}\n${r.contentPreview}\n\n`;
|
||||
if ((context + snippet).length > PROMPT_CONTEXT_LIMIT) break;
|
||||
context += snippet;
|
||||
}
|
||||
return context.trim();
|
||||
}
|
||||
|
||||
private buildPrompt(question: string, context: string): string {
|
||||
return [
|
||||
'คุณเป็นผู้ช่วยผู้เชี่ยวชาญด้านเอกสารโครงการก่อสร้าง',
|
||||
'ตอบคำถามโดยอ้างอิงจากเอกสารที่ให้มาเท่านั้น ห้ามตอบจากความรู้ทั่วไป',
|
||||
'หากข้อมูลในเอกสารไม่เพียงพอ ให้แจ้งว่า "ไม่พบข้อมูลในเอกสารที่ระบุ"',
|
||||
'',
|
||||
'=== เอกสารอ้างอิง ===',
|
||||
context,
|
||||
'',
|
||||
'=== คำถาม ===',
|
||||
question,
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
private buildCacheKey(
|
||||
question: string,
|
||||
projectPublicId: string,
|
||||
classificationCeiling: string
|
||||
): string {
|
||||
const raw = `${question}|${projectPublicId}|${classificationCeiling}`;
|
||||
return `rag:query:${createHash('sha256').update(raw).digest('hex')}`;
|
||||
}
|
||||
|
||||
private deriveClassificationCeiling(
|
||||
permissions: string[]
|
||||
): 'PUBLIC' | 'INTERNAL' | 'CONFIDENTIAL' {
|
||||
if (
|
||||
permissions.includes('system.manage_all') ||
|
||||
permissions.includes('document.view_confidential')
|
||||
) {
|
||||
return 'CONFIDENTIAL';
|
||||
}
|
||||
return 'INTERNAL';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import axios from 'axios';
|
||||
|
||||
export interface LlmGenerateResult {
|
||||
answer: string;
|
||||
usedFallbackModel: boolean;
|
||||
}
|
||||
|
||||
interface TyphoonChatResponse {
|
||||
choices: Array<{ message: { content: string } }>;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class TyphoonService {
|
||||
private readonly logger = new Logger(TyphoonService.name);
|
||||
private readonly typhoonUrl: string;
|
||||
private readonly typhoonKey: string;
|
||||
private readonly ollamaUrl: string;
|
||||
private readonly ollamaModel: string;
|
||||
private readonly timeoutMs: number;
|
||||
|
||||
constructor(private readonly configService: ConfigService) {
|
||||
this.typhoonUrl = this.configService.get<string>(
|
||||
'TYPHOON_API_URL',
|
||||
'https://api.opentyphoon.ai/v1'
|
||||
);
|
||||
this.typhoonKey = this.configService.get<string>('TYPHOON_API_KEY', '');
|
||||
this.ollamaUrl = this.configService.get<string>(
|
||||
'OLLAMA_URL',
|
||||
'http://localhost:11434'
|
||||
);
|
||||
this.ollamaModel = this.configService.get<string>(
|
||||
'OLLAMA_RAG_MODEL',
|
||||
'gemma3:12b'
|
||||
);
|
||||
this.timeoutMs = this.configService.get<number>('RAG_TIMEOUT_MS', 5000);
|
||||
}
|
||||
|
||||
async generate(
|
||||
prompt: string,
|
||||
forceLocal: boolean
|
||||
): Promise<LlmGenerateResult> {
|
||||
if (forceLocal) {
|
||||
const answer = await this.generateOllama(prompt);
|
||||
return { answer, usedFallbackModel: true };
|
||||
}
|
||||
|
||||
try {
|
||||
const answer = await Promise.race([
|
||||
this.generateTyphoon(prompt),
|
||||
this.delay(this.timeoutMs).then(() => {
|
||||
throw new Error('Typhoon timeout');
|
||||
}),
|
||||
]);
|
||||
return { answer, usedFallbackModel: false };
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`Typhoon failed, falling back to Ollama: ${err instanceof Error ? err.message : String(err)}`
|
||||
);
|
||||
const answer = await this.generateOllama(prompt);
|
||||
return { answer, usedFallbackModel: true };
|
||||
}
|
||||
}
|
||||
|
||||
sanitizeInput(text: string): string {
|
||||
return text
|
||||
.replace(/<CONTEXT_START>|<CONTEXT_END>/gi, '')
|
||||
.replace(/ignore previous instructions/gi, '')
|
||||
.replace(/system:/gi, '')
|
||||
.slice(0, 1000);
|
||||
}
|
||||
|
||||
private async generateTyphoon(prompt: string): Promise<string> {
|
||||
const response = await axios.post<TyphoonChatResponse>(
|
||||
`${this.typhoonUrl}/chat/completions`,
|
||||
{
|
||||
model: 'typhoon-v2.1-12b-instruct',
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: `<CONTEXT_START>\n${prompt}\n<CONTEXT_END>`,
|
||||
},
|
||||
],
|
||||
max_tokens: 1024,
|
||||
temperature: 0.1,
|
||||
},
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.typhoonKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
timeout: this.timeoutMs,
|
||||
}
|
||||
);
|
||||
return response.data.choices[0]?.message?.content ?? '';
|
||||
}
|
||||
|
||||
private async generateOllama(prompt: string): Promise<string> {
|
||||
const response = await axios.post<{ response: string }>(
|
||||
`${this.ollamaUrl}/api/generate`,
|
||||
{
|
||||
model: this.ollamaModel,
|
||||
prompt,
|
||||
stream: false,
|
||||
},
|
||||
{ timeout: 30000 }
|
||||
);
|
||||
return response.data.response ?? '';
|
||||
}
|
||||
|
||||
private delay(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,10 @@
|
||||
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
import {
|
||||
makeCounterProvider,
|
||||
makeHistogramProvider,
|
||||
} from '@willsoto/nestjs-prometheus';
|
||||
|
||||
// Entities
|
||||
import { WorkflowDefinition } from './entities/workflow-definition.entity';
|
||||
@@ -37,6 +41,17 @@ import { WorkflowEngineController } from './workflow-engine.controller';
|
||||
WorkflowDslService,
|
||||
WorkflowEventService,
|
||||
WorkflowTransitionGuard,
|
||||
// ADR-021 S1: Redlock observability — Prometheus metrics
|
||||
makeHistogramProvider({
|
||||
name: 'workflow_redlock_acquire_duration_ms',
|
||||
help: 'เวลาที่ใช้ในการ acquire Redlock สำหรับ workflow transition (รวม retry)',
|
||||
labelNames: ['outcome'], // 'success' | 'failure'
|
||||
buckets: [50, 100, 250, 500, 1000, 2000, 5000, 10000],
|
||||
}),
|
||||
makeCounterProvider({
|
||||
name: 'workflow_redlock_acquire_failures_total',
|
||||
help: 'จำนวนครั้งที่ Redlock acquire ล้มเหลวหลัง retry ครบ (Fail-closed HTTP 503)',
|
||||
}),
|
||||
],
|
||||
exports: [WorkflowEngineService], // Export Service ให้ Module อื่น (Correspondence, RFA) เรียกใช้
|
||||
})
|
||||
|
||||
@@ -8,7 +8,6 @@ jest.mock('redlock', () =>
|
||||
}))
|
||||
);
|
||||
|
||||
import { ConflictException, ServiceUnavailableException } from '@nestjs/common';
|
||||
import { Test, TestingModule } from '@nestjs/testing';
|
||||
import { WorkflowEngineService } from './workflow-engine.service';
|
||||
import { getRepositoryToken } from '@nestjs/typeorm';
|
||||
@@ -23,7 +22,13 @@ import { Attachment } from '../../common/file-storage/entities/attachment.entity
|
||||
import { WorkflowDslService } from './workflow-dsl.service';
|
||||
import { WorkflowEventService } from './workflow-event.service';
|
||||
import { CACHE_MANAGER } from '@nestjs/cache-manager';
|
||||
import { NotFoundException, WorkflowException } from '../../common/exceptions';
|
||||
// ADR-007: \u0e43\u0e0a\u0e49 custom exceptions \u0e17\u0e31\u0e49\u0e07\u0e2b\u0e21\u0e14\u0e08\u0e32\u0e01 common/exceptions (\u0e44\u0e21\u0e48\u0e43\u0e0a\u0e49 @nestjs/common built-in)
|
||||
import {
|
||||
NotFoundException,
|
||||
WorkflowException,
|
||||
ConflictException,
|
||||
ServiceUnavailableException,
|
||||
} from '../../common/exceptions';
|
||||
import { CreateWorkflowDefinitionDto } from './dto/create-workflow-definition.dto';
|
||||
|
||||
// Token ของ @nestjs-modules/ioredis — default Redis connection
|
||||
@@ -132,6 +137,20 @@ describe('WorkflowEngineService', () => {
|
||||
// ไม่จำเป็นต้องมี method จริง เพราะ Redlock ถูก mock แล้ว
|
||||
},
|
||||
},
|
||||
// ADR-021 S1: Prometheus metrics mocks
|
||||
{
|
||||
provide: 'PROM_METRIC_WORKFLOW_REDLOCK_ACQUIRE_DURATION_MS',
|
||||
useValue: {
|
||||
labels: jest.fn().mockReturnThis(),
|
||||
observe: jest.fn(),
|
||||
},
|
||||
},
|
||||
{
|
||||
provide: 'PROM_METRIC_WORKFLOW_REDLOCK_ACQUIRE_FAILURES_TOTAL',
|
||||
useValue: {
|
||||
inc: jest.fn(),
|
||||
},
|
||||
},
|
||||
],
|
||||
}).compile();
|
||||
|
||||
@@ -496,6 +515,45 @@ describe('WorkflowEngineService', () => {
|
||||
expect(mockRedlockRelease).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('H1: should throw ConflictException when state changes between pre-check and pessimistic lock (TOCTOU)', async () => {
|
||||
// Arrange: pre-check พบ PENDING_REVIEW (stale read)
|
||||
(instanceRepo.findOne as jest.Mock).mockResolvedValue({
|
||||
id: 'inst-1',
|
||||
currentState: 'PENDING_REVIEW',
|
||||
});
|
||||
// แต่ภายใน transaction (pessimistic lock) state เปลี่ยนเป็น APPROVED แล้ว
|
||||
// (simulate: another request transition ไปก่อนที่ Redlock จะ release)
|
||||
mockQueryRunner.manager.findOne.mockResolvedValue({
|
||||
id: 'inst-1',
|
||||
currentState: 'APPROVED', // ← เปลี่ยนไปแล้ว
|
||||
status: WorkflowStatus.ACTIVE,
|
||||
definition: { compiled: mockCompiledWorkflow },
|
||||
context: {},
|
||||
});
|
||||
mockDslService.evaluate.mockReturnValue({
|
||||
nextState: 'APPROVED',
|
||||
events: [],
|
||||
});
|
||||
|
||||
await expect(
|
||||
service.processTransition(
|
||||
'inst-1',
|
||||
'APPROVE',
|
||||
1,
|
||||
undefined,
|
||||
{},
|
||||
attachmentPublicIds
|
||||
)
|
||||
).rejects.toThrow(ConflictException);
|
||||
|
||||
// ต้อง rollback transaction + release Redlock
|
||||
expect(mockQueryRunner.rollbackTransaction).toHaveBeenCalled();
|
||||
expect(mockQueryRunner.commitTransaction).not.toHaveBeenCalled();
|
||||
expect(mockRedlockRelease).toHaveBeenCalled();
|
||||
// attachment update ต้องไม่ถูกเรียก
|
||||
expect(mockQueryRunner.manager.update).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('C1: should release Redlock even when transition succeeds', async () => {
|
||||
(instanceRepo.findOne as jest.Mock).mockResolvedValue({
|
||||
id: 'inst-1',
|
||||
|
||||
@@ -1,21 +1,23 @@
|
||||
// File: src/modules/workflow-engine/workflow-engine.service.ts
|
||||
|
||||
import {
|
||||
Injectable,
|
||||
Inject,
|
||||
Logger,
|
||||
ConflictException,
|
||||
ServiceUnavailableException,
|
||||
} from '@nestjs/common';
|
||||
import { Injectable, Inject, Logger } from '@nestjs/common';
|
||||
import { CACHE_MANAGER } from '@nestjs/cache-manager';
|
||||
import type { Cache } from 'cache-manager';
|
||||
import { NotFoundException, WorkflowException } from '../../common/exceptions';
|
||||
// ADR-007: ใช้ custom exceptions ที่ extends BaseException เพื่อให้ payload ตรง layered structure
|
||||
import {
|
||||
NotFoundException,
|
||||
WorkflowException,
|
||||
ConflictException,
|
||||
ServiceUnavailableException,
|
||||
} from '../../common/exceptions';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { DataSource, In, Repository } from 'typeorm';
|
||||
// ADR-021 Clarify Q2: Redis Redlock for transition Fail-closed (Retry 3x → 503)
|
||||
import { InjectRedis } from '@nestjs-modules/ioredis';
|
||||
import Redis from 'ioredis';
|
||||
import Redlock, { Lock } from 'redlock';
|
||||
import { InjectMetric } from '@willsoto/nestjs-prometheus';
|
||||
import { Counter, Histogram } from 'prom-client';
|
||||
// Entities
|
||||
import { WorkflowDefinition } from './entities/workflow-definition.entity';
|
||||
import { WorkflowHistory } from './entities/workflow-history.entity';
|
||||
@@ -30,11 +32,7 @@ import { CreateWorkflowDefinitionDto } from './dto/create-workflow-definition.dt
|
||||
import { EvaluateWorkflowDto } from './dto/evaluate-workflow.dto';
|
||||
import { UpdateWorkflowDefinitionDto } from './dto/update-workflow-definition.dto';
|
||||
import { WorkflowHistoryItemDto } from './dto/workflow-history-item.dto';
|
||||
import {
|
||||
CompiledWorkflow,
|
||||
RawEvent,
|
||||
WorkflowDslService,
|
||||
} from './workflow-dsl.service';
|
||||
import { CompiledWorkflow, WorkflowDslService } from './workflow-dsl.service';
|
||||
import { WorkflowEventService } from './workflow-event.service'; // [NEW] Import Event Service
|
||||
|
||||
// Legacy Interface (Backward Compatibility)
|
||||
@@ -76,7 +74,12 @@ export class WorkflowEngineService {
|
||||
private readonly eventService: WorkflowEventService, // [NEW] Inject Service
|
||||
private readonly dataSource: DataSource, // ใช้สำหรับ Transaction
|
||||
@Inject(CACHE_MANAGER) private readonly cacheManager: Cache, // ADR-021 T024: History cache
|
||||
@InjectRedis() private readonly redis: Redis // ADR-021 Clarify Q2: Redlock
|
||||
@InjectRedis() private readonly redis: Redis, // ADR-021 Clarify Q2: Redlock
|
||||
// ADR-021 S1: Redlock observability metrics
|
||||
@InjectMetric('workflow_redlock_acquire_duration_ms')
|
||||
private readonly redlockAcquireDuration: Histogram<string>,
|
||||
@InjectMetric('workflow_redlock_acquire_failures_total')
|
||||
private readonly redlockAcquireFailures: Counter<string>
|
||||
) {
|
||||
// ADR-021 Clarify Q2 (C1): Redlock Fail-closed
|
||||
// Retry 3 ครั้ง × 500ms เพิ่ม jitter → ถ้ายังไม่ได้ throw HTTP 503
|
||||
@@ -340,6 +343,8 @@ export class WorkflowEngineService {
|
||||
// อนุญาตให้แนบไฟล์เฉพาะในสถานะ PENDING_REVIEW / PENDING_APPROVAL
|
||||
// ==============================================================
|
||||
if (hasAttachments) {
|
||||
// ADR-021 S2: `id` ใน WorkflowInstance เป็น CHAR(36) UUID direct PK
|
||||
// (ไม่ใช่ pattern UuidBaseEntity ที่ INT+publicId) — ADR-019 compliant เพราะ UUID ถูก expose โดยตรง
|
||||
const instancePreCheck = await this.instanceRepo.findOne({
|
||||
where: { id: instanceId },
|
||||
select: ['id', 'currentState'],
|
||||
@@ -352,12 +357,15 @@ export class WorkflowEngineService {
|
||||
instancePreCheck.currentState
|
||||
)
|
||||
) {
|
||||
throw new ConflictException({
|
||||
userMessage: 'ไม่สามารถอัปโหลดไฟล์ในสถานะนี้ได้',
|
||||
recoveryAction:
|
||||
throw new ConflictException(
|
||||
'WORKFLOW_STATE_LOCKED',
|
||||
`Upload rejected: currentState=${instancePreCheck.currentState} not in UPLOAD_ALLOWED_STATES`,
|
||||
'ไม่สามารถอัปโหลดไฟล์ในสถานะนี้ได้',
|
||||
[
|
||||
'อนุญาตเฉพาะสถานะ PENDING_REVIEW หรือ PENDING_APPROVAL เท่านั้น',
|
||||
currentState: instancePreCheck.currentState,
|
||||
});
|
||||
'รีเฟรชหน้าแล้วตรวจสถานะล่าสุด',
|
||||
]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -367,25 +375,34 @@ export class WorkflowEngineService {
|
||||
// ==============================================================
|
||||
const lockKey = `lock:wf:transition:${instanceId}`;
|
||||
let lock: Lock;
|
||||
const acquireStart = Date.now();
|
||||
try {
|
||||
lock = await this.redlock.acquire([lockKey], 10000); // 10s TTL
|
||||
// S1: บันทึก duration กรณี acquire สำเร็จ
|
||||
this.redlockAcquireDuration
|
||||
.labels({ outcome: 'success' })
|
||||
.observe(Date.now() - acquireStart);
|
||||
} catch (err) {
|
||||
// S1: บันทึก duration + failure counter
|
||||
this.redlockAcquireDuration
|
||||
.labels({ outcome: 'failure' })
|
||||
.observe(Date.now() - acquireStart);
|
||||
this.redlockAcquireFailures.inc();
|
||||
this.logger.error(
|
||||
`Redlock acquire failed after retries for ${instanceId}: ${(err as Error).message}`
|
||||
);
|
||||
throw new ServiceUnavailableException({
|
||||
userMessage: 'ระบบยุ่งชั่วคราว กรุณาลองใหม่ภายหลัง',
|
||||
recoveryAction: 'รอสักครู่แล้วลองใหม่',
|
||||
});
|
||||
throw new ServiceUnavailableException(
|
||||
'WORKFLOW_LOCK_UNAVAILABLE',
|
||||
`Redlock acquire failed after 3 retries on lock:wf:transition:${instanceId}`,
|
||||
'ระบบยุ่งชั่วคราว กรุณาลองใหม่ภายหลัง',
|
||||
['รอสักครู่แล้วลองใหม่', 'แจ้งผู้ดูแลระบบหากยังพบปัญหา']
|
||||
);
|
||||
}
|
||||
|
||||
const queryRunner = this.dataSource.createQueryRunner();
|
||||
await queryRunner.connect();
|
||||
await queryRunner.startTransaction();
|
||||
|
||||
let eventsToDispatch: RawEvent[] = [];
|
||||
let updatedContext: Record<string, unknown> = {};
|
||||
|
||||
try {
|
||||
// 1. Lock Instance เพื่อป้องกัน Race Condition (Pessimistic Write Lock)
|
||||
const instance = await queryRunner.manager.findOne(WorkflowInstance, {
|
||||
@@ -407,6 +424,22 @@ export class WorkflowEngineService {
|
||||
);
|
||||
}
|
||||
|
||||
// ==============================================================
|
||||
// ADR-021 (H1): Re-check state ภายใต้ pessimistic lock — ปิด TOCTOU race
|
||||
// pre-check ด้านหน้าเป็น optimistic fast-fail; เช็กที่นี่เป็น authoritative
|
||||
// ==============================================================
|
||||
if (
|
||||
hasAttachments &&
|
||||
!WorkflowEngineService.UPLOAD_ALLOWED_STATES.has(instance.currentState)
|
||||
) {
|
||||
throw new ConflictException(
|
||||
'WORKFLOW_STATE_CHANGED',
|
||||
`TOCTOU: state changed to ${instance.currentState} under pessimistic lock`,
|
||||
'ไม่สามารถอัปโหลดไฟล์ได้ (สถานะเอกสารได้เปลี่ยนไปก่อนหน้านี้)',
|
||||
['รีเฟรชหน้าแล้วตรวจสถานะล่าสุดของเอกสาร']
|
||||
);
|
||||
}
|
||||
|
||||
// 2. Evaluate Logic ผ่าน DSL Service
|
||||
const compiled = instance.definition
|
||||
.compiled as unknown as CompiledWorkflow;
|
||||
@@ -494,27 +527,23 @@ export class WorkflowEngineService {
|
||||
)
|
||||
);
|
||||
|
||||
// [NEW] เก็บค่าไว้ Dispatch หลัง Commit
|
||||
eventsToDispatch = evaluation.events;
|
||||
updatedContext = context;
|
||||
|
||||
this.logger.log(
|
||||
`Transition: ${instanceId} [${fromState}] --${action}--> [${toState}] by User:${userId}`
|
||||
);
|
||||
|
||||
// [NEW] Dispatch Events (Async) ผ่าน WorkflowEventService
|
||||
if (eventsToDispatch && eventsToDispatch.length > 0) {
|
||||
// Dispatch Events (Async, Fire-and-forget) ผ่าน WorkflowEventService
|
||||
if (evaluation.events.length > 0) {
|
||||
void this.eventService.dispatchEvents(
|
||||
instance.id,
|
||||
eventsToDispatch,
|
||||
updatedContext
|
||||
evaluation.events,
|
||||
context
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
nextState: toState,
|
||||
events: eventsToDispatch,
|
||||
events: evaluation.events,
|
||||
isCompleted: instance.status === WorkflowStatus.COMPLETED,
|
||||
};
|
||||
} catch (err) {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import _request from 'supertest';
|
||||
import { AppModule } from '../src/app.module';
|
||||
import { _RoutingTemplate } from '../src/modules/correspondence/entities/routing-template.entity';
|
||||
import { RoutingTemplate } from '../src/modules/correspondence/entities/routing-template.entity';
|
||||
|
||||
import { Test, TestingModule } from '@nestjs/testing';
|
||||
|
||||
|
||||
Reference in New Issue
Block a user