690419:1831 feat: update CI/CD to use SSH key authentication #05
CI / CD Pipeline / build (push) Failing after 4m57s
CI / CD Pipeline / deploy (push) Has been skipped

This commit is contained in:
2026-04-19 18:31:30 +07:00
parent 733f3c3987
commit 13745e5874
61 changed files with 6709 additions and 1241 deletions
+57
View File
@@ -0,0 +1,57 @@
# ========================================
# LCBP3 Backend — Environment Variables
# Copy to .env and fill in real values
# หมายเหตุ: ค่า DB_PASSWORD, REDIS_PASSWORD,
# ELASTICSEARCH_PASSWORD ต้องตรงกับที่ตั้งไว้ใน
# services stack (MariaDB/Redis/Elasticsearch
# ดูจาก .env ของ services stack ที่รันอยู่แล้วบน QNAP
# ========================================
# Database
DB_HOST=localhost
DB_PORT=3306
DB_USERNAME=admin
DB_PASSWORD=Center2025
DB_DATABASE=lcbp3_dev
# Redis
REDIS_HOST=localhost
REDIS_PORT=16379
REDIS_PASSWORD=Center2025
# JWT
JWT_SECRET=change-me-in-production
JWT_EXPIRES_IN=7d
# File Storage
UPLOAD_DEST=./uploads
MAX_FILE_SIZE=52428800
# ClamAV
CLAMAV_HOST=localhost
CLAMAV_PORT=3310
# ========================================
# ADR-022 RAG — Retrieval-Augmented Generation
# ========================================
# Qdrant vector store (local docker-compose or QNAP)
QDRANT_URL=http://localhost:6333
# Ollama (Admin Desktop Desk-5439 — ADR-018 AI boundary)
OLLAMA_EMBED_MODEL=nomic-embed-text
OLLAMA_RAG_MODEL=gemma3:12b
OLLAMA_URL=http://192.168.10.100:11434
# Thai preprocessing microservice (PyThaiNLP — Admin Desktop)
THAI_PREPROCESS_URL=http://192.168.10.100:8765
# Typhoon API (cloud LLM — PUBLIC/INTERNAL only, never CONFIDENTIAL)
TYPHOON_API_KEY=your-typhoon-api-key-here
TYPHOON_API_URL=https://api.opentyphoon.ai/v1
# RAG query config
RAG_TOPK=20
RAG_FINAL_K=5
RAG_TIMEOUT_MS=5000
RAG_QUERY_CACHE_TTL=300
+14
View File
@@ -54,10 +54,24 @@ services:
- esdata:/usr/share/elasticsearch/data
networks:
- lcbp3-net
# ADR-022 RAG: Qdrant vector store (tiered multitenancy for project isolation)
qdrant:
image: qdrant/qdrant:v1.16.1
container_name: lcbp3-qdrant-local
restart: always
ports:
- '6333:6333' # REST API
- '6334:6334' # gRPC
volumes:
- qdrant_data:/qdrant/storage
networks:
- lcbp3-net
volumes:
db_data:
redis_data: # เพิ่ม Volume
esdata:
qdrant_data: # ADR-022 RAG vector store
networks:
lcbp3-net:
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large Load Diff
+7 -6
View File
@@ -24,19 +24,19 @@
"seed": "ts-node -r tsconfig-paths/register src/database/seeds/run-seed.ts"
},
"dependencies": {
"@casl/ability": "^6.7.5",
"@casl/ability": "6.8.0",
"@elastic/elasticsearch": "^8.13.0",
"@nestjs-modules/ioredis": "^2.0.2",
"@nestjs/axios": "^4.0.1",
"@nestjs/bullmq": "^11.0.4",
"@nestjs/cache-manager": "^3.0.1",
"@nestjs/common": "^11.0.1",
"@nestjs/common": "^11.1.19",
"@nestjs/config": "^4.0.2",
"@nestjs/core": "^11.0.1",
"@nestjs/core": "^11.1.19",
"@nestjs/elasticsearch": "^11.1.0",
"@nestjs/jwt": "^11.0.1",
"@nestjs/passport": "^11.0.5",
"@nestjs/platform-express": "^11.0.1",
"@nestjs/platform-express": "^11.1.19",
"@nestjs/platform-socket.io": "^11.1.9",
"@nestjs/schedule": "^6.0.1",
"@nestjs/swagger": "^11.2.3",
@@ -44,12 +44,13 @@
"@nestjs/throttler": "^6.4.0",
"@nestjs/typeorm": "^11.0.0",
"@nestjs/websockets": "^11.1.9",
"@qdrant/js-client-rest": "^1.17.0",
"@types/nodemailer": "^7.0.4",
"@willsoto/nestjs-prometheus": "^6.0.2",
"ajv": "^8.17.1",
"ajv-formats": "^3.0.1",
"async-retry": "^1.3.3",
"axios": "^1.13.2",
"axios": "^1.15.0",
"bcrypt": "^6.0.0",
"bullmq": "^5.63.2",
"cache-manager": "^7.2.5",
@@ -81,7 +82,7 @@
"zod": "^4.1.13"
},
"devDependencies": {
"@compodoc/compodoc": "^1.1.32",
"@compodoc/compodoc": "^1.1.23",
"@eslint/eslintrc": "^3.2.0",
"@eslint/js": "^9.18.0",
"@nestjs/cli": "^11.0.0",
+2
View File
@@ -52,6 +52,7 @@ import { SearchModule } from './modules/search/search.module';
import { AuditLogModule } from './modules/audit-log/audit-log.module';
import { MigrationModule } from './modules/migration/migration.module';
import { AiModule } from './modules/ai/ai.module';
import { RagModule } from './modules/rag/rag.module';
@Module({
imports: [
@@ -189,6 +190,7 @@ import { AiModule } from './modules/ai/ai.module';
AuditLogModule,
MigrationModule,
AiModule,
RagModule,
],
controllers: [AppController],
providers: [
@@ -2,7 +2,7 @@ import { Test, TestingModule } from '@nestjs/testing';
import { AbilityFactory, ScopeContext } from './ability.factory';
import { User } from '../../../modules/user/entities/user.entity';
import { UserAssignment } from '../../../modules/user/entities/user-assignment.entity';
import { Role } from '../../../modules/auth/entities/role.entity';
import { Role } from '../../../modules/user/entities/role.entity';
describe('AbilityFactory', () => {
let factory: AbilityFactory;
@@ -14,6 +14,7 @@ export enum ErrorType {
DATABASE_ERROR = 'DATABASE_ERROR',
EXTERNAL_SERVICE = 'EXTERNAL_SERVICE',
INFRASTRUCTURE = 'INFRASTRUCTURE',
SERVICE_UNAVAILABLE = 'SERVICE_UNAVAILABLE', // 503 — ระบบไม่พร้อมให้บริการชั่วคราว (Redlock fail, Redis down)
}
// ระดับความรุนแรงของ Error
@@ -49,6 +50,8 @@ export function getStatusCode(type: ErrorType): number {
case ErrorType.EXTERNAL_SERVICE:
case ErrorType.INFRASTRUCTURE:
return HttpStatus.INTERNAL_SERVER_ERROR;
case ErrorType.SERVICE_UNAVAILABLE:
return HttpStatus.SERVICE_UNAVAILABLE; // 503
default:
return HttpStatus.INTERNAL_SERVER_ERROR;
}
@@ -233,3 +236,27 @@ export class DatabaseException extends BaseException {
);
}
}
// Service Unavailable (503) - ระบบไม่พร้อมให้บริการชั่วคราว
// ADR-021 C1: Redlock Fail-closed — retry ครบแล้ว ยัง acquire ไม่ได้
export class ServiceUnavailableException extends BaseException {
constructor(
code: string,
message: string,
userMessage?: string,
recoveryActions?: string[]
) {
super(
ErrorType.SERVICE_UNAVAILABLE,
code,
message,
userMessage || 'ระบบยุ่งชั่วคราว กรุณาลองใหม่ภายหลัง',
ErrorSeverity.HIGH,
undefined,
recoveryActions || [
'รอสักครู่แล้วลองใหม่',
'แจ้งผู้ดูแลระบบหากยังพบปัญหา',
]
);
}
}
+1
View File
@@ -14,6 +14,7 @@ export {
WorkflowException,
SystemException,
DatabaseException,
ServiceUnavailableException,
} from './base.exception';
export type { ValidationErrorDetail, ErrorPayload } from './base.exception';
@@ -1,5 +1,6 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { BullModule } from '@nestjs/bullmq';
import { ScheduleModule } from '@nestjs/schedule'; // ✅ Import
import { FileStorageService } from './file-storage.service.js';
import { FileStorageController } from './file-storage.controller.js';
@@ -12,6 +13,7 @@ import { UserModule } from '../../modules/user/user.module';
TypeOrmModule.forFeature([Attachment]),
ScheduleModule.forRoot(), // ✅ เปิดใช้งาน Cron Job],
UserModule,
BullModule.registerQueue({ name: 'rag:ocr' }),
],
controllers: [FileStorageController],
providers: [
@@ -4,10 +4,13 @@ import {
NotFoundException,
BadRequestException,
Logger,
Optional,
} from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository, In } from 'typeorm';
import { ConfigService } from '@nestjs/config';
import { InjectQueue } from '@nestjs/bullmq';
import { Queue } from 'bullmq';
import * as fs from 'fs-extra';
import * as path from 'path';
import * as crypto from 'crypto';
@@ -24,7 +27,8 @@ export class FileStorageService {
constructor(
@InjectRepository(Attachment)
private attachmentRepository: Repository<Attachment>,
private configService: ConfigService
private configService: ConfigService,
@Optional() @InjectQueue('rag:ocr') private readonly ragOcrQueue?: Queue
) {
// ใช้ env vars จาก docker-compose สำหรับ Production
// ถ้าไม่ได้กำหนดจะ fallback เป็น ./uploads/temp และ ./uploads/permanent
@@ -90,7 +94,18 @@ export class FileStorageService {
*/
async commit(
tempIds: string[],
options?: { issueDate?: Date; documentType?: string }
options?: {
issueDate?: Date;
documentType?: string;
ragMeta?: {
docType: string;
docNumber: string | null;
revision: string | null;
projectCode: string;
projectPublicId: string;
classification: 'PUBLIC' | 'INTERNAL' | 'CONFIDENTIAL';
};
}
): Promise<Attachment[]> {
if (!tempIds || tempIds.length === 0) {
return [];
@@ -149,7 +164,27 @@ export class FileStorageService {
att.expiresAt = undefined; // เคลียร์วันหมดอายุ
att.referenceDate = effectiveDate; // Save reference date
committedAttachments.push(await this.attachmentRepository.save(att));
const saved = await this.attachmentRepository.save(att);
committedAttachments.push(saved);
if (this.ragOcrQueue && options?.ragMeta) {
await this.ragOcrQueue
.add(
'ocr',
{
attachmentPublicId: saved.publicId,
filePath: saved.filePath,
...options.ragMeta,
},
{ jobId: saved.publicId }
)
.catch((err: unknown) => {
this.logger.error(
`Failed to enqueue rag:ocr for ${saved.publicId}`,
err instanceof Error ? err.stack : String(err)
);
});
}
} else {
this.logger.error(`File missing during commit: ${oldPath}`);
throw new NotFoundException(
@@ -10,10 +10,15 @@ import { of, lastValueFrom } from 'rxjs';
import { Request } from 'express';
import type { Socket } from 'net';
type MockAuditLogRepo = {
create: jest.Mock;
save: jest.Mock;
};
describe('AuditLogInterceptor', () => {
let interceptor: AuditLogInterceptor;
let reflector: Reflector;
let auditLogRepo: jest.Mocked<Partial<typeof AuditLog.prototype.constructor>>;
let auditLogRepo: MockAuditLogRepo;
const createMockUser = (userId: number): User => {
const user = new User();
@@ -55,7 +60,7 @@ describe('AuditLogInterceptor', () => {
});
beforeEach(async () => {
const mockRepository = {
const mockRepository: MockAuditLogRepo = {
create: jest.fn().mockReturnValue({}),
save: jest.fn().mockResolvedValue({}),
};
@@ -78,7 +83,7 @@ describe('AuditLogInterceptor', () => {
interceptor = module.get<AuditLogInterceptor>(AuditLogInterceptor);
reflector = module.get<Reflector>(Reflector);
auditLogRepo = module.get(getRepositoryToken(AuditLog));
auditLogRepo = module.get<MockAuditLogRepo>(getRepositoryToken(AuditLog));
});
afterEach(() => {
@@ -99,7 +99,17 @@ describe('CorrespondenceController', () => {
mockResult
);
const mockReq = { user: { user_id: 1, roles: [] } };
const mockReq = {
user: {
user_id: 1,
username: 'testuser',
password: 'hashedpassword',
email: 'test@example.com',
publicId: '019505a1-7c3e-7000-8000-abc123def456',
createdAt: new Date(),
updatedAt: new Date(),
},
};
(mockCorrespondenceService.findOneByUuid as jest.Mock).mockResolvedValue({
id: 1,
uuid: 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11',
@@ -2,6 +2,7 @@ import { Test, TestingModule } from '@nestjs/testing';
import { MigrationController } from './migration.controller';
import { MigrationService } from './migration.service';
import { ImportCorrespondenceDto } from './dto/import-correspondence.dto';
import { User } from '../user/entities/user.entity';
describe('MigrationController', () => {
let controller: MigrationController;
@@ -32,17 +33,29 @@ describe('MigrationController', () => {
it('should call importCorrespondence on service', async () => {
const dto: ImportCorrespondenceDto = {
document_number: 'DOC-001',
documentNumber: 'DOC-001',
subject: 'Legacy Record',
category: 'Correspondence',
source_file_path: '/staging_ai/test.pdf',
migrated_by: 'SYSTEM_IMPORT',
batch_id: 'batch1',
project_id: 1,
sourceFilePath: '/staging_ai/test.pdf',
migratedBy: 'SYSTEM_IMPORT',
batchId: 'batch1',
projectId: 1,
};
const idempotencyKey = 'key123';
const user = { userId: 5 };
const user: User = {
user_id: 5,
username: 'testuser',
password: 'hashedpassword',
email: 'test@example.com',
publicId: '019505a1-7c3e-7000-8000-abc123def456',
createdAt: new Date(),
updatedAt: new Date(),
isActive: true,
failedAttempts: 0,
primaryOrganizationPublicId: undefined,
generatePublicId: jest.fn(),
};
const result = await controller.importCorrespondence(
dto,
@@ -0,0 +1,86 @@
import { Test, TestingModule } from '@nestjs/testing';
import { IngestionService } from '../ingestion.service';
const QUEUE_TOKEN = 'BullQueue_rag:ocr';
const mockOcrQueue = {
getJob: jest.fn(),
add: jest.fn(),
};
const baseJobData = {
attachmentPublicId: 'att-uuid-001',
filePath: '/uploads/permanent/CORR/2026/04/file.pdf',
docType: 'CORR',
docNumber: 'REF-001',
revision: null,
projectCode: 'PRJ-001',
projectPublicId: 'proj-uuid-001',
classification: 'INTERNAL' as const,
};
describe('IngestionService', () => {
let service: IngestionService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [
IngestionService,
{ provide: QUEUE_TOKEN, useValue: mockOcrQueue },
],
}).compile();
service = module.get<IngestionService>(IngestionService);
jest.clearAllMocks();
});
it('should enqueue rag:ocr job with attachmentPublicId as jobId', async () => {
mockOcrQueue.getJob.mockResolvedValue(null);
mockOcrQueue.add.mockResolvedValue({ id: baseJobData.attachmentPublicId });
await service.enqueue(baseJobData);
expect(mockOcrQueue.add).toHaveBeenCalledWith('ocr', baseJobData, {
jobId: baseJobData.attachmentPublicId,
});
});
it('EC-RAG-001: duplicate enqueue when job is active → second call is no-op (log only)', async () => {
const mockJob = { getState: jest.fn().mockResolvedValue('active') };
mockOcrQueue.getJob.mockResolvedValue(mockJob);
await service.enqueue(baseJobData);
expect(mockOcrQueue.add).not.toHaveBeenCalled();
});
it('EC-RAG-001: duplicate enqueue when job is waiting → second call is no-op', async () => {
const mockJob = { getState: jest.fn().mockResolvedValue('waiting') };
mockOcrQueue.getJob.mockResolvedValue(mockJob);
await service.enqueue(baseJobData);
expect(mockOcrQueue.add).not.toHaveBeenCalled();
});
it('should re-enqueue if job exists but is completed (state=completed)', async () => {
const mockJob = { getState: jest.fn().mockResolvedValue('completed') };
mockOcrQueue.getJob.mockResolvedValue(mockJob);
mockOcrQueue.add.mockResolvedValue({ id: baseJobData.attachmentPublicId });
await service.enqueue(baseJobData);
expect(mockOcrQueue.add).toHaveBeenCalledTimes(1);
});
it('should re-enqueue if job exists but is failed (state=failed)', async () => {
const mockJob = { getState: jest.fn().mockResolvedValue('failed') };
mockOcrQueue.getJob.mockResolvedValue(mockJob);
mockOcrQueue.add.mockResolvedValue({ id: baseJobData.attachmentPublicId });
await service.enqueue(baseJobData);
expect(mockOcrQueue.add).toHaveBeenCalledTimes(1);
});
});
@@ -0,0 +1,206 @@
import { Test, TestingModule } from '@nestjs/testing';
import { ServiceUnavailableException } from '@nestjs/common';
import { getRepositoryToken } from '@nestjs/typeorm';
import { RagService } from '../rag.service';
import { QdrantService } from '../qdrant.service';
import { EmbeddingService } from '../embedding.service';
import { TyphoonService } from '../typhoon.service';
import { IngestionService } from '../ingestion.service';
import { DocumentChunk } from '../entities/document-chunk.entity';
const DEFAULT_REDIS_TOKEN = 'default_IORedisModuleConnectionToken';
const mockQdrant = {
isReady: jest.fn(),
hybridSearch: jest.fn(),
deleteByDocumentId: jest.fn(),
};
const mockEmbedding = {
embed: jest.fn(),
};
const mockTyphoon = {
generate: jest.fn(),
sanitizeInput: jest.fn((t: string) => t),
};
const mockIngestion = { enqueue: jest.fn() };
const mockChunkRepo = {
count: jest.fn(),
delete: jest.fn(),
manager: {
query: jest.fn(),
},
};
const mockRedis = {
get: jest.fn(),
setex: jest.fn(),
};
describe('RagService', () => {
let service: RagService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [
RagService,
{ provide: QdrantService, useValue: mockQdrant },
{ provide: EmbeddingService, useValue: mockEmbedding },
{ provide: TyphoonService, useValue: mockTyphoon },
{ provide: IngestionService, useValue: mockIngestion },
{ provide: getRepositoryToken(DocumentChunk), useValue: mockChunkRepo },
{ provide: DEFAULT_REDIS_TOKEN, useValue: mockRedis },
],
}).compile();
service = module.get<RagService>(RagService);
jest.clearAllMocks();
});
describe('query()', () => {
const dto = {
question: 'เอกสารเกี่ยวกับอะไร?',
projectPublicId: 'proj-uuid-1234',
};
const memberPerms: string[] = [];
const adminPerms = ['system.manage_all'];
it('should return answer with citations on PUBLIC cache miss → write cache', async () => {
mockQdrant.isReady.mockReturnValue(true);
mockRedis.get.mockResolvedValue(null);
mockEmbedding.embed.mockResolvedValue(new Array(768).fill(0.1));
mockQdrant.hybridSearch.mockResolvedValue([
{
chunkId: 'chunk-1',
publicId: 'att-1',
docType: 'CORR',
docNumber: 'REF-001',
revision: null,
projectCode: 'PRJ-001',
contentPreview: 'เนื้อหาเอกสาร',
score: 0.92,
},
]);
mockTyphoon.generate.mockResolvedValue({
answer: 'คำตอบ',
usedFallbackModel: false,
});
const result = await service.query(dto, memberPerms);
expect(result.answer).toBe('คำตอบ');
expect(result.citations).toHaveLength(1);
expect(result.usedFallbackModel).toBe(false);
expect(mockRedis.setex).toHaveBeenCalledTimes(1);
});
it('should return cached result without calling Qdrant on cache hit', async () => {
mockQdrant.isReady.mockReturnValue(true);
const cached = JSON.stringify({
answer: 'cached answer',
citations: [],
confidence: 0.9,
usedFallbackModel: false,
});
mockRedis.get.mockResolvedValue(cached);
const result = await service.query(dto, memberPerms);
expect(result.answer).toBe('cached answer');
expect(mockQdrant.hybridSearch).not.toHaveBeenCalled();
expect(mockEmbedding.embed).not.toHaveBeenCalled();
});
it('CONFIDENTIAL: must use Ollama only, skip cache read and write', async () => {
mockQdrant.isReady.mockReturnValue(true);
mockEmbedding.embed.mockResolvedValue(new Array(768).fill(0.1));
mockQdrant.hybridSearch.mockResolvedValue([]);
mockTyphoon.generate.mockResolvedValue({
answer: 'ลับมาก',
usedFallbackModel: true,
});
const result = await service.query(dto, adminPerms);
expect(mockRedis.get).not.toHaveBeenCalled();
expect(mockRedis.setex).not.toHaveBeenCalled();
expect(mockTyphoon.generate).toHaveBeenCalledWith(
expect.any(String),
true
);
expect(result.usedFallbackModel).toBe(true);
});
it('collectionReady=false → throw ServiceUnavailableException RAG_NOT_READY', async () => {
mockQdrant.isReady.mockReturnValue(false);
await expect(service.query(dto, memberPerms)).rejects.toThrow(
ServiceUnavailableException
);
});
it('cross-project cache isolation: same question different projectPublicId → different cache key', async () => {
mockQdrant.isReady.mockReturnValue(true);
mockRedis.get.mockResolvedValue(null);
mockEmbedding.embed.mockResolvedValue(new Array(768).fill(0.1));
mockQdrant.hybridSearch.mockResolvedValue([]);
mockTyphoon.generate.mockResolvedValue({
answer: 'A',
usedFallbackModel: false,
});
await service.query(
{ question: 'Q?', projectPublicId: 'proj-A' },
memberPerms
);
await service.query(
{ question: 'Q?', projectPublicId: 'proj-B' },
memberPerms
);
const calls = mockRedis.setex.mock.calls as [string, ...unknown[]][];
expect(calls[0][0]).not.toBe(calls[1][0]);
});
it('classification ceiling derived from role, not from request body', async () => {
mockQdrant.isReady.mockReturnValue(true);
mockRedis.get.mockResolvedValue(null);
mockEmbedding.embed.mockResolvedValue(new Array(768).fill(0.1));
mockQdrant.hybridSearch.mockResolvedValue([]);
mockTyphoon.generate.mockResolvedValue({
anwer: 'ok',
usedFallbackModel: false,
});
await service.query(dto, memberPerms);
expect(mockQdrant.hybridSearch).toHaveBeenCalledWith(
expect.any(Array),
dto.projectPublicId,
'INTERNAL',
20
);
jest.clearAllMocks();
mockQdrant.isReady.mockReturnValue(true);
mockRedis.get.mockResolvedValue(null);
mockEmbedding.embed.mockResolvedValue(new Array(768).fill(0.1));
mockQdrant.hybridSearch.mockResolvedValue([]);
mockTyphoon.generate.mockResolvedValue({
answer: 'ok',
usedFallbackModel: true,
});
await service.query(dto, adminPerms);
expect(mockQdrant.hybridSearch).toHaveBeenCalledWith(
expect.any(Array),
dto.projectPublicId,
'CONFIDENTIAL',
20
);
});
});
});
@@ -0,0 +1,11 @@
import { IsNotEmpty, IsString, IsUUID, MaxLength } from 'class-validator';
export class RagQueryDto {
@IsString()
@IsNotEmpty()
@MaxLength(500)
question!: string;
@IsUUID()
projectPublicId!: string;
}
@@ -0,0 +1,16 @@
export interface RagCitation {
chunkId: string;
docNumber: string | null;
docType: string;
revision: string | null;
snippet: string;
score: number;
}
export class RagResponseDto {
answer!: string;
citations!: RagCitation[];
confidence!: number;
usedFallbackModel!: boolean;
cachedAt?: string;
}
@@ -0,0 +1,46 @@
import { Injectable, Logger } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import axios from 'axios';
@Injectable()
export class EmbeddingService {
private readonly logger = new Logger(EmbeddingService.name);
private readonly ollamaUrl: string;
private readonly model: string;
constructor(private readonly configService: ConfigService) {
this.ollamaUrl = this.configService.get<string>(
'OLLAMA_URL',
'http://localhost:11434'
);
this.model = this.configService.get<string>(
'OLLAMA_EMBED_MODEL',
'nomic-embed-text'
);
}
async embed(text: string): Promise<number[]> {
try {
const response = await axios.post<{ embedding: number[] }>(
`${this.ollamaUrl}/api/embeddings`,
{ model: this.model, prompt: text },
{ timeout: 30000 }
);
return response.data.embedding;
} catch (err) {
this.logger.error(
'Embedding failed',
err instanceof Error ? err.stack : String(err)
);
throw err;
}
}
async embedBatch(texts: string[]): Promise<number[][]> {
return Promise.all(texts.map((t) => this.embed(t)));
}
getModelName(): string {
return this.model;
}
}
@@ -0,0 +1,47 @@
import { Column, CreateDateColumn, Entity, PrimaryColumn } from 'typeorm';
@Entity('document_chunks')
export class DocumentChunk {
@PrimaryColumn({ type: 'char', length: 36 })
id!: string;
@Column({ type: 'char', length: 36, name: 'document_id' })
documentId!: string;
@Column({ name: 'chunk_index' })
chunkIndex!: number;
@Column({ type: 'text' })
content!: string;
@Column({ length: 20, name: 'doc_type' })
docType!: string;
@Column({ length: 100, name: 'doc_number', nullable: true })
docNumber!: string | null;
@Column({ length: 20, nullable: true })
revision!: string | null;
@Column({ length: 50, name: 'project_code' })
projectCode!: string;
@Column({ length: 36, name: 'project_public_id' })
projectPublicId!: string;
@Column({
type: 'enum',
enum: ['PUBLIC', 'INTERNAL', 'CONFIDENTIAL'],
default: 'INTERNAL',
})
classification!: 'PUBLIC' | 'INTERNAL' | 'CONFIDENTIAL';
@Column({ length: 20, nullable: true })
version!: string | null;
@Column({ length: 100, name: 'embedding_model', default: 'nomic-embed-text' })
embeddingModel!: string;
@CreateDateColumn({ name: 'created_at', precision: 3 })
createdAt!: Date;
}
@@ -0,0 +1,30 @@
import { Injectable, Logger } from '@nestjs/common';
import { InjectQueue } from '@nestjs/bullmq';
import { Queue } from 'bullmq';
import { OcrJobData } from './processors/ocr.processor';
@Injectable()
export class IngestionService {
private readonly logger = new Logger(IngestionService.name);
constructor(@InjectQueue('rag:ocr') private readonly ocrQueue: Queue) {}
async enqueue(data: OcrJobData): Promise<void> {
const jobId = data.attachmentPublicId;
const existing = await this.ocrQueue.getJob(jobId);
if (existing) {
const state = await existing.getState();
if (state === 'active' || state === 'waiting' || state === 'delayed') {
this.logger.log(
`rag:ocr job already queued for ${jobId} (state: ${state})`
);
return;
}
}
await this.ocrQueue.add('ocr', data, { jobId });
this.logger.log(`Enqueued rag:ocr for attachment ${jobId}`);
}
}
@@ -0,0 +1,110 @@
import { Processor, WorkerHost } from '@nestjs/bullmq';
import { Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { Job } from 'bullmq';
import { v4 as uuidv4 } from 'uuid';
import { EmbeddingService } from '../embedding.service';
import { QdrantService, VectorMetadata } from '../qdrant.service';
import { DocumentChunk } from '../entities/document-chunk.entity';
import { EmbeddingJobData } from './thai-preprocess.processor';
const CHUNK_SIZE = 512;
const CHUNK_OVERLAP = 50;
@Processor('rag:embedding')
export class EmbeddingProcessor extends WorkerHost {
private readonly logger = new Logger(EmbeddingProcessor.name);
constructor(
private readonly embeddingService: EmbeddingService,
private readonly qdrantService: QdrantService,
@InjectRepository(DocumentChunk)
private readonly chunkRepo: Repository<DocumentChunk>
) {
super();
}
async process(job: Job<EmbeddingJobData>): Promise<void> {
const {
attachmentPublicId,
normalizedText,
docType,
docNumber,
revision,
projectCode,
projectPublicId,
classification,
} = job.data;
const chunks = this.chunkText(normalizedText);
const model = this.embeddingService.getModelName();
const upsertPoints: Parameters<QdrantService['upsertBatch']>[0] = [];
const chunkEntities: DocumentChunk[] = [];
for (let i = 0; i < chunks.length; i++) {
const chunkId = uuidv4();
const vector = await this.embeddingService.embed(chunks[i]);
const payload: VectorMetadata = {
chunk_id: chunkId,
public_id: attachmentPublicId,
project_public_id: projectPublicId,
doc_type: docType,
doc_number: docNumber,
revision,
project_code: projectCode,
classification,
content_preview: chunks[i].slice(0, 500),
embedding_model: model,
};
upsertPoints.push({ id: chunkId, vector, payload });
const entity = this.chunkRepo.create({
id: chunkId,
documentId: attachmentPublicId,
chunkIndex: i,
content: chunks[i],
docType,
docNumber,
revision,
projectCode,
projectPublicId,
classification,
embeddingModel: model,
});
chunkEntities.push(entity);
}
if (upsertPoints.length > 0) {
await this.qdrantService.upsertBatch(upsertPoints);
await this.chunkRepo.save(chunkEntities);
}
await this.chunkRepo.manager.query(
`UPDATE attachments SET rag_status = 'INDEXED', rag_last_error = NULL WHERE public_id = ?`,
[attachmentPublicId]
);
this.logger.log(
`Embedded ${chunks.length} chunks for ${attachmentPublicId}`
);
}
private chunkText(text: string): string[] {
const words = text.split(/\s+/);
const chunks: string[] = [];
let start = 0;
while (start < words.length) {
const end = Math.min(start + CHUNK_SIZE, words.length);
chunks.push(words.slice(start, end).join(' '));
start += CHUNK_SIZE - CHUNK_OVERLAP;
}
return chunks.filter((c) => c.trim().length > 0);
}
}
@@ -0,0 +1,68 @@
import { Processor, WorkerHost } from '@nestjs/bullmq';
import { Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { Job } from 'bullmq';
import * as fs from 'fs';
import { InjectQueue } from '@nestjs/bullmq';
import { Queue } from 'bullmq';
import { DocumentChunk } from '../entities/document-chunk.entity';
export interface OcrJobData {
attachmentPublicId: string;
filePath: string;
docType: string;
docNumber: string | null;
revision: string | null;
projectCode: string;
projectPublicId: string;
classification: 'PUBLIC' | 'INTERNAL' | 'CONFIDENTIAL';
}
@Processor('rag:ocr')
export class OcrProcessor extends WorkerHost {
private readonly logger = new Logger(OcrProcessor.name);
constructor(
@InjectQueue('rag:thai-preprocess') private readonly thaiQueue: Queue,
@InjectRepository(DocumentChunk)
private readonly chunkRepo: Repository<DocumentChunk>
) {
super();
}
async process(job: Job<OcrJobData>): Promise<void> {
const { attachmentPublicId, filePath } = job.data;
const existing = await this.chunkRepo.count({
where: { documentId: attachmentPublicId },
});
if (existing > 0) {
this.logger.log(
`rag:ocr job already indexed for ${attachmentPublicId}, skipping`
);
return;
}
await this.chunkRepo.manager.query(
`UPDATE attachments SET rag_status = 'PROCESSING' WHERE public_id = ?`,
[attachmentPublicId]
);
let rawText: string;
try {
rawText = fs.readFileSync(filePath, 'utf-8');
} catch {
rawText = `[binary:${attachmentPublicId}]`;
}
await this.thaiQueue.add(
'preprocess',
{ ...job.data, rawText },
{ jobId: `thai:${attachmentPublicId}` }
);
this.logger.log(`OCR enqueued thai-preprocess for ${attachmentPublicId}`);
}
}
@@ -0,0 +1,56 @@
import { Processor, WorkerHost, InjectQueue } from '@nestjs/bullmq';
import { Logger } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { Queue, Job } from 'bullmq';
import axios from 'axios';
import { OcrJobData } from './ocr.processor';
export interface ThaiPreprocessJobData extends OcrJobData {
rawText: string;
}
export interface EmbeddingJobData extends ThaiPreprocessJobData {
normalizedText: string;
}
@Processor('rag:thai-preprocess')
export class ThaiPreprocessProcessor extends WorkerHost {
private readonly logger = new Logger(ThaiPreprocessProcessor.name);
private readonly thaiUrl: string;
constructor(
private readonly configService: ConfigService,
@InjectQueue('rag:embedding') private readonly embeddingQueue: Queue
) {
super();
this.thaiUrl = this.configService.get<string>(
'THAI_PREPROCESS_URL',
'http://localhost:8765'
);
}
async process(job: Job<ThaiPreprocessJobData>): Promise<void> {
const { rawText, attachmentPublicId } = job.data;
let normalizedText = rawText;
try {
const response = await axios.post<{ normalized: string }>(
`${this.thaiUrl}/normalize`,
{ text: rawText },
{ timeout: 30000 }
);
normalizedText = response.data.normalized ?? rawText;
} catch (err) {
this.logger.warn(
`Thai preprocess failed for ${attachmentPublicId}, using raw text: ${err instanceof Error ? err.message : String(err)}`
);
}
await this.embeddingQueue.add(
'embed',
{ ...job.data, normalizedText } as EmbeddingJobData,
{ jobId: `embed:${attachmentPublicId}` }
);
}
}
+179
View File
@@ -0,0 +1,179 @@
import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { QdrantClient } from '@qdrant/js-client-rest';
export interface VectorMetadata extends Record<string, unknown> {
chunk_id: string;
public_id: string;
project_public_id: string;
doc_type: string;
doc_number: string | null;
revision: string | null;
project_code: string;
classification: 'PUBLIC' | 'INTERNAL' | 'CONFIDENTIAL';
content_preview: string;
embedding_model: string;
}
export interface HybridSearchResult {
chunkId: string;
publicId: string;
docType: string;
docNumber: string | null;
revision: string | null;
projectCode: string;
contentPreview: string;
score: number;
}
const COLLECTION_NAME = 'lcbp3_vectors';
const VECTOR_SIZE = 768;
@Injectable()
export class QdrantService implements OnModuleInit {
private readonly logger = new Logger(QdrantService.name);
private client: QdrantClient;
private collectionReady = false;
constructor(private readonly configService: ConfigService) {
const url = this.configService.get<string>(
'QDRANT_URL',
'http://localhost:6333'
);
this.client = new QdrantClient({ url });
}
async onModuleInit(): Promise<void> {
try {
await this.initCollection();
this.collectionReady = true;
this.logger.log(`Qdrant collection '${COLLECTION_NAME}' ready`);
} catch (err) {
this.logger.error(
'Qdrant collection init failed — RAG queries will return 503',
err instanceof Error ? err.stack : String(err)
);
this.collectionReady = false;
}
}
isReady(): boolean {
return this.collectionReady;
}
private async initCollection(): Promise<void> {
const collections = await this.client.getCollections();
const exists = collections.collections.some(
(c) => c.name === COLLECTION_NAME
);
if (!exists) {
await this.client.createCollection(COLLECTION_NAME, {
vectors: { size: VECTOR_SIZE, distance: 'Cosine' },
hnsw_config: {
payload_m: 16,
m: 0,
},
optimizers_config: { indexing_threshold: 10000 },
});
this.logger.log(`Created Qdrant collection '${COLLECTION_NAME}'`);
await this.client.createPayloadIndex(COLLECTION_NAME, {
field_name: 'project_public_id',
field_schema: { type: 'keyword', is_tenant: true } as Parameters<
QdrantClient['createPayloadIndex']
>[1]['field_schema'],
});
await this.client.createPayloadIndex(COLLECTION_NAME, {
field_name: 'classification',
field_schema: 'keyword',
});
await this.client.createPayloadIndex(COLLECTION_NAME, {
field_name: 'doc_type',
field_schema: 'keyword',
});
await this.client.createPayloadIndex(COLLECTION_NAME, {
field_name: 'doc_number',
field_schema: 'keyword',
});
}
}
async upsertBatch(
points: Array<{ id: string; vector: number[]; payload: VectorMetadata }>
): Promise<void> {
await this.client.upsert(COLLECTION_NAME, {
wait: true,
points: points.map((p) => ({
id: p.id,
vector: p.vector,
payload: p.payload,
})),
});
}
async hybridSearch(
queryVector: number[],
projectPublicId: string,
classificationCeiling: 'PUBLIC' | 'INTERNAL' | 'CONFIDENTIAL',
topK: number
): Promise<HybridSearchResult[]> {
const classificationValues = this.getAllowedClassifications(
classificationCeiling
);
const vectorResults = await this.client.search(COLLECTION_NAME, {
vector: queryVector,
limit: topK,
filter: {
must: [
{ key: 'project_public_id', match: { value: projectPublicId } },
{ key: 'classification', match: { any: classificationValues } },
],
},
with_payload: true,
});
return vectorResults.map((r) => {
const payload = r.payload as unknown as VectorMetadata;
return {
chunkId: payload.chunk_id,
publicId: payload.public_id,
docType: payload.doc_type,
docNumber: payload.doc_number,
revision: payload.revision,
projectCode: payload.project_code,
contentPreview: payload.content_preview,
score: r.score,
};
});
}
async deleteByDocumentId(documentId: string): Promise<void> {
await this.client.delete(COLLECTION_NAME, {
wait: true,
filter: {
must: [{ key: 'public_id', match: { value: documentId } }],
},
});
}
async forceInitCollection(): Promise<void> {
await this.initCollection();
this.collectionReady = true;
this.logger.log(`Qdrant collection force-initialized`);
}
private getAllowedClassifications(
ceiling: 'PUBLIC' | 'INTERNAL' | 'CONFIDENTIAL'
): string[] {
const order: Array<'PUBLIC' | 'INTERNAL' | 'CONFIDENTIAL'> = [
'PUBLIC',
'INTERNAL',
'CONFIDENTIAL',
];
const ceilIdx = order.indexOf(ceiling);
return order.slice(0, ceilIdx + 1);
}
}
+93
View File
@@ -0,0 +1,93 @@
import {
Body,
Controller,
Delete,
Get,
Headers,
HttpCode,
HttpStatus,
Logger,
Param,
Post,
UseGuards,
} from '@nestjs/common';
import { ApiBearerAuth, ApiOperation, ApiTags } from '@nestjs/swagger';
import { Throttle } from '@nestjs/throttler';
import { CurrentUser } from '../../common/decorators/current-user.decorator';
import { RequirePermission } from '../../common/decorators/require-permission.decorator';
import { JwtAuthGuard } from '../../common/guards/jwt-auth.guard';
import { RbacGuard } from '../../common/guards/rbac.guard';
import { ParseUuidPipe } from '../../common/pipes/parse-uuid.pipe';
import { UserService } from '../user/user.service';
import { User } from '../user/entities/user.entity';
import { RagQueryDto } from './dto/rag-query.dto';
import { RagService } from './rag.service';
@ApiTags('RAG')
@ApiBearerAuth()
@UseGuards(JwtAuthGuard, RbacGuard)
@Throttle({ default: { limit: 30, ttl: 60000 } })
@Controller('rag')
export class RagController {
private readonly logger = new Logger(RagController.name);
constructor(
private readonly ragService: RagService,
private readonly userService: UserService
) {}
@Post('query')
@HttpCode(HttpStatus.OK)
@ApiOperation({ summary: 'RAG Q&A — ค้นหาคำตอบจากเอกสารโครงการ' })
@RequirePermission('rag.query')
async query(
@Body() dto: RagQueryDto,
@CurrentUser() user: User,
@Headers('Idempotency-Key') idempotencyKey: string
) {
if (!idempotencyKey) {
this.logger.warn(`Missing Idempotency-Key from user ${user.user_id}`);
}
const permissions = await this.userService.getUserPermissions(user.user_id);
return this.ragService.query(dto, permissions);
}
@Get('status/:attachmentId')
@ApiOperation({ summary: 'ดูสถานะ RAG ingestion ของ attachment' })
@RequirePermission('rag.query')
async getStatus(@Param('attachmentId', ParseUuidPipe) attachmentId: string) {
return this.ragService.getStatus(attachmentId);
}
@Post('ingest/:attachmentId')
@HttpCode(HttpStatus.OK)
@ApiOperation({ summary: 'Re-ingest attachment ที่ FAILED (Admin only)' })
@RequirePermission('rag.manage')
async reIngest(@Param('attachmentId', ParseUuidPipe) attachmentId: string) {
await this.ragService.reIngest(attachmentId);
return { message: 'Re-ingestion queued' };
}
@Delete('vectors/:attachmentId')
@HttpCode(HttpStatus.NO_CONTENT)
@ApiOperation({ summary: 'ลบ vectors ของ attachment ออกจาก Qdrant' })
@RequirePermission('rag.manage')
async deleteVectors(
@Param('attachmentId', ParseUuidPipe) attachmentId: string
) {
await this.ragService.deleteVectors(attachmentId);
}
@Post('admin/init-collection')
@HttpCode(HttpStatus.OK)
@ApiOperation({
summary: 'T038: Init Qdrant collection lcbp3_vectors (admin only)',
})
@RequirePermission('rag.manage')
async initCollection() {
await this.ragService.initCollection();
return { message: 'Qdrant collection initialized' };
}
}
+55
View File
@@ -0,0 +1,55 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { BullModule } from '@nestjs/bullmq';
import { ConfigModule } from '@nestjs/config';
import { DocumentChunk } from './entities/document-chunk.entity';
import { EmbeddingService } from './embedding.service';
import { QdrantService } from './qdrant.service';
import { TyphoonService } from './typhoon.service';
import { RagService } from './rag.service';
import { RagController } from './rag.controller';
import { IngestionService } from './ingestion.service';
import { OcrProcessor } from './processors/ocr.processor';
import { ThaiPreprocessProcessor } from './processors/thai-preprocess.processor';
import { EmbeddingProcessor } from './processors/embedding.processor';
import { UserModule } from '../user/user.module';
const DLQ_DEFAULTS = {
attempts: 3,
backoff: { type: 'exponential' as const, delay: 2000 },
removeOnComplete: 100,
removeOnFail: 200,
};
@Module({
imports: [
ConfigModule,
UserModule,
TypeOrmModule.forFeature([DocumentChunk]),
BullModule.registerQueue(
{ name: 'rag:ocr', defaultJobOptions: DLQ_DEFAULTS },
{ name: 'rag:thai-preprocess', defaultJobOptions: DLQ_DEFAULTS },
{ name: 'rag:embedding', defaultJobOptions: DLQ_DEFAULTS }
),
],
controllers: [RagController],
providers: [
EmbeddingService,
QdrantService,
TyphoonService,
RagService,
IngestionService,
OcrProcessor,
ThaiPreprocessProcessor,
EmbeddingProcessor,
],
exports: [
EmbeddingService,
QdrantService,
TyphoonService,
RagService,
IngestionService,
],
})
export class RagModule {}
+255
View File
@@ -0,0 +1,255 @@
import {
Injectable,
Logger,
ServiceUnavailableException,
BadRequestException,
} from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { InjectRedis } from '@nestjs-modules/ioredis';
import Redis from 'ioredis';
import { createHash } from 'crypto';
import { QdrantService } from './qdrant.service';
import { EmbeddingService } from './embedding.service';
import { TyphoonService } from './typhoon.service';
import { IngestionService } from './ingestion.service';
import { DocumentChunk } from './entities/document-chunk.entity';
import { RagQueryDto } from './dto/rag-query.dto';
import { RagResponseDto, RagCitation } from './dto/rag-response.dto';
const CACHE_TTL_SECONDS = 300;
const PROMPT_CONTEXT_LIMIT = 3000;
@Injectable()
export class RagService {
private readonly logger = new Logger(RagService.name);
constructor(
private readonly qdrant: QdrantService,
private readonly embedding: EmbeddingService,
private readonly typhoon: TyphoonService,
private readonly ingestionService: IngestionService,
@InjectRepository(DocumentChunk)
private readonly chunkRepo: Repository<DocumentChunk>,
@InjectRedis() private readonly redis: Redis
) {}
async query(
dto: RagQueryDto,
userPermissions: string[]
): Promise<RagResponseDto> {
const { question, projectPublicId } = dto;
const classificationCeiling =
this.deriveClassificationCeiling(userPermissions);
const isConfidential = classificationCeiling === 'CONFIDENTIAL';
if (!this.qdrant.isReady()) {
throw new ServiceUnavailableException('RAG_NOT_READY');
}
const cacheKey = this.buildCacheKey(
question,
projectPublicId,
classificationCeiling
);
if (!isConfidential) {
const cached = await this.redis.get(cacheKey);
if (cached) {
const parsed = JSON.parse(cached) as RagResponseDto;
parsed.cachedAt = new Date().toISOString();
return parsed;
}
}
const queryVector = await this.embedding.embed(question);
const topK = 20;
const results = await this.qdrant.hybridSearch(
queryVector,
projectPublicId,
classificationCeiling,
topK
);
const reranked = results.sort((a, b) => b.score - a.score).slice(0, 5);
const context = this.buildContext(reranked);
const safeQuestion = this.typhoon.sanitizeInput(question);
const prompt = this.buildPrompt(safeQuestion, context);
const { answer, usedFallbackModel } = await this.typhoon.generate(
prompt,
isConfidential
);
const citations: RagCitation[] = reranked.map((r) => ({
chunkId: r.chunkId,
docNumber: r.docNumber,
docType: r.docType,
revision: r.revision,
snippet: r.contentPreview.slice(0, 200),
score: r.score,
}));
const confidence = reranked.length > 0 ? reranked[0].score : 0;
const response: RagResponseDto = {
answer,
citations,
confidence,
usedFallbackModel,
};
if (!isConfidential) {
await this.redis.setex(
cacheKey,
CACHE_TTL_SECONDS,
JSON.stringify(response)
);
}
return response;
}
async getStatus(
attachmentPublicId: string
): Promise<{ ragStatus: string; chunkCount: number }> {
const chunkCount = await this.chunkRepo.count({
where: { documentId: attachmentPublicId },
});
const result = await this.chunkRepo.manager.query<{ rag_status: string }[]>(
`SELECT rag_status FROM attachments WHERE public_id = ? LIMIT 1`,
[attachmentPublicId]
);
const ragStatus = result[0]?.rag_status ?? 'PENDING';
return { ragStatus, chunkCount };
}
async reIngest(attachmentPublicId: string): Promise<void> {
const statusResult = await this.chunkRepo.manager.query<
{ rag_status: string; file_path: string }[]
>(
`SELECT rag_status, file_path FROM attachments WHERE public_id = ? LIMIT 1`,
[attachmentPublicId]
);
const current = statusResult[0]?.rag_status;
if (current !== 'FAILED') {
throw new BadRequestException(
`Cannot re-ingest: current status is '${current ?? 'unknown'}', expected 'FAILED'`
);
}
const sample = await this.chunkRepo.findOne({
where: { documentId: attachmentPublicId },
});
await this.chunkRepo.delete({ documentId: attachmentPublicId });
try {
await this.qdrant.deleteByDocumentId(attachmentPublicId);
} catch (err) {
this.logger.error(
`Qdrant delete failed for ${attachmentPublicId} — continuing`,
err instanceof Error ? err.stack : String(err)
);
}
await this.chunkRepo.manager.query(
`UPDATE attachments SET rag_status = 'PENDING', rag_last_error = NULL WHERE public_id = ?`,
[attachmentPublicId]
);
if (sample) {
await this.ingestionService.enqueue({
attachmentPublicId,
filePath: statusResult[0]?.file_path ?? '',
docType: sample.docType,
docNumber: sample.docNumber,
revision: sample.revision,
projectCode: sample.projectCode,
projectPublicId: sample.projectPublicId,
classification: sample.classification,
});
}
}
async initCollection(): Promise<void> {
await this.qdrant.onModuleInit();
}
async deleteVectors(attachmentPublicId: string): Promise<void> {
await this.chunkRepo.delete({ documentId: attachmentPublicId });
try {
await this.qdrant.deleteByDocumentId(attachmentPublicId);
} catch (err) {
this.logger.error(
`Qdrant delete failed for ${attachmentPublicId}`,
err instanceof Error ? err.stack : String(err)
);
}
await this.chunkRepo.manager.query(
`UPDATE attachments SET rag_status = 'PENDING', rag_last_error = NULL WHERE public_id = ?`,
[attachmentPublicId]
);
}
buildContext(
results: Array<{
docType: string;
docNumber: string | null;
revision: string | null;
contentPreview: string;
}>
): string {
let context = '';
for (const r of results) {
const header = `[${r.docType}${r.docNumber ? ` - ${r.docNumber}` : ''}${r.revision ? ` - ${r.revision}` : ''}]`;
const snippet = `${header}\n${r.contentPreview}\n\n`;
if ((context + snippet).length > PROMPT_CONTEXT_LIMIT) break;
context += snippet;
}
return context.trim();
}
private buildPrompt(question: string, context: string): string {
return [
'คุณเป็นผู้ช่วยผู้เชี่ยวชาญด้านเอกสารโครงการก่อสร้าง',
'ตอบคำถามโดยอ้างอิงจากเอกสารที่ให้มาเท่านั้น ห้ามตอบจากความรู้ทั่วไป',
'หากข้อมูลในเอกสารไม่เพียงพอ ให้แจ้งว่า "ไม่พบข้อมูลในเอกสารที่ระบุ"',
'',
'=== เอกสารอ้างอิง ===',
context,
'',
'=== คำถาม ===',
question,
].join('\n');
}
private buildCacheKey(
question: string,
projectPublicId: string,
classificationCeiling: string
): string {
const raw = `${question}|${projectPublicId}|${classificationCeiling}`;
return `rag:query:${createHash('sha256').update(raw).digest('hex')}`;
}
private deriveClassificationCeiling(
permissions: string[]
): 'PUBLIC' | 'INTERNAL' | 'CONFIDENTIAL' {
if (
permissions.includes('system.manage_all') ||
permissions.includes('document.view_confidential')
) {
return 'CONFIDENTIAL';
}
return 'INTERNAL';
}
}
+115
View File
@@ -0,0 +1,115 @@
import { Injectable, Logger } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import axios from 'axios';
export interface LlmGenerateResult {
answer: string;
usedFallbackModel: boolean;
}
interface TyphoonChatResponse {
choices: Array<{ message: { content: string } }>;
}
@Injectable()
export class TyphoonService {
private readonly logger = new Logger(TyphoonService.name);
private readonly typhoonUrl: string;
private readonly typhoonKey: string;
private readonly ollamaUrl: string;
private readonly ollamaModel: string;
private readonly timeoutMs: number;
constructor(private readonly configService: ConfigService) {
this.typhoonUrl = this.configService.get<string>(
'TYPHOON_API_URL',
'https://api.opentyphoon.ai/v1'
);
this.typhoonKey = this.configService.get<string>('TYPHOON_API_KEY', '');
this.ollamaUrl = this.configService.get<string>(
'OLLAMA_URL',
'http://localhost:11434'
);
this.ollamaModel = this.configService.get<string>(
'OLLAMA_RAG_MODEL',
'gemma3:12b'
);
this.timeoutMs = this.configService.get<number>('RAG_TIMEOUT_MS', 5000);
}
async generate(
prompt: string,
forceLocal: boolean
): Promise<LlmGenerateResult> {
if (forceLocal) {
const answer = await this.generateOllama(prompt);
return { answer, usedFallbackModel: true };
}
try {
const answer = await Promise.race([
this.generateTyphoon(prompt),
this.delay(this.timeoutMs).then(() => {
throw new Error('Typhoon timeout');
}),
]);
return { answer, usedFallbackModel: false };
} catch (err) {
this.logger.warn(
`Typhoon failed, falling back to Ollama: ${err instanceof Error ? err.message : String(err)}`
);
const answer = await this.generateOllama(prompt);
return { answer, usedFallbackModel: true };
}
}
sanitizeInput(text: string): string {
return text
.replace(/<CONTEXT_START>|<CONTEXT_END>/gi, '')
.replace(/ignore previous instructions/gi, '')
.replace(/system:/gi, '')
.slice(0, 1000);
}
private async generateTyphoon(prompt: string): Promise<string> {
const response = await axios.post<TyphoonChatResponse>(
`${this.typhoonUrl}/chat/completions`,
{
model: 'typhoon-v2.1-12b-instruct',
messages: [
{
role: 'user',
content: `<CONTEXT_START>\n${prompt}\n<CONTEXT_END>`,
},
],
max_tokens: 1024,
temperature: 0.1,
},
{
headers: {
Authorization: `Bearer ${this.typhoonKey}`,
'Content-Type': 'application/json',
},
timeout: this.timeoutMs,
}
);
return response.data.choices[0]?.message?.content ?? '';
}
private async generateOllama(prompt: string): Promise<string> {
const response = await axios.post<{ response: string }>(
`${this.ollamaUrl}/api/generate`,
{
model: this.ollamaModel,
prompt,
stream: false,
},
{ timeout: 30000 }
);
return response.data.response ?? '';
}
private delay(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
}
@@ -2,6 +2,10 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import {
makeCounterProvider,
makeHistogramProvider,
} from '@willsoto/nestjs-prometheus';
// Entities
import { WorkflowDefinition } from './entities/workflow-definition.entity';
@@ -37,6 +41,17 @@ import { WorkflowEngineController } from './workflow-engine.controller';
WorkflowDslService,
WorkflowEventService,
WorkflowTransitionGuard,
// ADR-021 S1: Redlock observability — Prometheus metrics
makeHistogramProvider({
name: 'workflow_redlock_acquire_duration_ms',
help: 'เวลาที่ใช้ในการ acquire Redlock สำหรับ workflow transition (รวม retry)',
labelNames: ['outcome'], // 'success' | 'failure'
buckets: [50, 100, 250, 500, 1000, 2000, 5000, 10000],
}),
makeCounterProvider({
name: 'workflow_redlock_acquire_failures_total',
help: 'จำนวนครั้งที่ Redlock acquire ล้มเหลวหลัง retry ครบ (Fail-closed HTTP 503)',
}),
],
exports: [WorkflowEngineService], // Export Service ให้ Module อื่น (Correspondence, RFA) เรียกใช้
})
@@ -8,7 +8,6 @@ jest.mock('redlock', () =>
}))
);
import { ConflictException, ServiceUnavailableException } from '@nestjs/common';
import { Test, TestingModule } from '@nestjs/testing';
import { WorkflowEngineService } from './workflow-engine.service';
import { getRepositoryToken } from '@nestjs/typeorm';
@@ -23,7 +22,13 @@ import { Attachment } from '../../common/file-storage/entities/attachment.entity
import { WorkflowDslService } from './workflow-dsl.service';
import { WorkflowEventService } from './workflow-event.service';
import { CACHE_MANAGER } from '@nestjs/cache-manager';
import { NotFoundException, WorkflowException } from '../../common/exceptions';
// ADR-007: \u0e43\u0e0a\u0e49 custom exceptions \u0e17\u0e31\u0e49\u0e07\u0e2b\u0e21\u0e14\u0e08\u0e32\u0e01 common/exceptions (\u0e44\u0e21\u0e48\u0e43\u0e0a\u0e49 @nestjs/common built-in)
import {
NotFoundException,
WorkflowException,
ConflictException,
ServiceUnavailableException,
} from '../../common/exceptions';
import { CreateWorkflowDefinitionDto } from './dto/create-workflow-definition.dto';
// Token ของ @nestjs-modules/ioredis — default Redis connection
@@ -132,6 +137,20 @@ describe('WorkflowEngineService', () => {
// ไม่จำเป็นต้องมี method จริง เพราะ Redlock ถูก mock แล้ว
},
},
// ADR-021 S1: Prometheus metrics mocks
{
provide: 'PROM_METRIC_WORKFLOW_REDLOCK_ACQUIRE_DURATION_MS',
useValue: {
labels: jest.fn().mockReturnThis(),
observe: jest.fn(),
},
},
{
provide: 'PROM_METRIC_WORKFLOW_REDLOCK_ACQUIRE_FAILURES_TOTAL',
useValue: {
inc: jest.fn(),
},
},
],
}).compile();
@@ -496,6 +515,45 @@ describe('WorkflowEngineService', () => {
expect(mockRedlockRelease).toHaveBeenCalled();
});
it('H1: should throw ConflictException when state changes between pre-check and pessimistic lock (TOCTOU)', async () => {
// Arrange: pre-check พบ PENDING_REVIEW (stale read)
(instanceRepo.findOne as jest.Mock).mockResolvedValue({
id: 'inst-1',
currentState: 'PENDING_REVIEW',
});
// แต่ภายใน transaction (pessimistic lock) state เปลี่ยนเป็น APPROVED แล้ว
// (simulate: another request transition ไปก่อนที่ Redlock จะ release)
mockQueryRunner.manager.findOne.mockResolvedValue({
id: 'inst-1',
currentState: 'APPROVED', // ← เปลี่ยนไปแล้ว
status: WorkflowStatus.ACTIVE,
definition: { compiled: mockCompiledWorkflow },
context: {},
});
mockDslService.evaluate.mockReturnValue({
nextState: 'APPROVED',
events: [],
});
await expect(
service.processTransition(
'inst-1',
'APPROVE',
1,
undefined,
{},
attachmentPublicIds
)
).rejects.toThrow(ConflictException);
// ต้อง rollback transaction + release Redlock
expect(mockQueryRunner.rollbackTransaction).toHaveBeenCalled();
expect(mockQueryRunner.commitTransaction).not.toHaveBeenCalled();
expect(mockRedlockRelease).toHaveBeenCalled();
// attachment update ต้องไม่ถูกเรียก
expect(mockQueryRunner.manager.update).not.toHaveBeenCalled();
});
it('C1: should release Redlock even when transition succeeds', async () => {
(instanceRepo.findOne as jest.Mock).mockResolvedValue({
id: 'inst-1',
@@ -1,21 +1,23 @@
// File: src/modules/workflow-engine/workflow-engine.service.ts
import {
Injectable,
Inject,
Logger,
ConflictException,
ServiceUnavailableException,
} from '@nestjs/common';
import { Injectable, Inject, Logger } from '@nestjs/common';
import { CACHE_MANAGER } from '@nestjs/cache-manager';
import type { Cache } from 'cache-manager';
import { NotFoundException, WorkflowException } from '../../common/exceptions';
// ADR-007: ใช้ custom exceptions ที่ extends BaseException เพื่อให้ payload ตรง layered structure
import {
NotFoundException,
WorkflowException,
ConflictException,
ServiceUnavailableException,
} from '../../common/exceptions';
import { InjectRepository } from '@nestjs/typeorm';
import { DataSource, In, Repository } from 'typeorm';
// ADR-021 Clarify Q2: Redis Redlock for transition Fail-closed (Retry 3x → 503)
import { InjectRedis } from '@nestjs-modules/ioredis';
import Redis from 'ioredis';
import Redlock, { Lock } from 'redlock';
import { InjectMetric } from '@willsoto/nestjs-prometheus';
import { Counter, Histogram } from 'prom-client';
// Entities
import { WorkflowDefinition } from './entities/workflow-definition.entity';
import { WorkflowHistory } from './entities/workflow-history.entity';
@@ -30,11 +32,7 @@ import { CreateWorkflowDefinitionDto } from './dto/create-workflow-definition.dt
import { EvaluateWorkflowDto } from './dto/evaluate-workflow.dto';
import { UpdateWorkflowDefinitionDto } from './dto/update-workflow-definition.dto';
import { WorkflowHistoryItemDto } from './dto/workflow-history-item.dto';
import {
CompiledWorkflow,
RawEvent,
WorkflowDslService,
} from './workflow-dsl.service';
import { CompiledWorkflow, WorkflowDslService } from './workflow-dsl.service';
import { WorkflowEventService } from './workflow-event.service'; // [NEW] Import Event Service
// Legacy Interface (Backward Compatibility)
@@ -76,7 +74,12 @@ export class WorkflowEngineService {
private readonly eventService: WorkflowEventService, // [NEW] Inject Service
private readonly dataSource: DataSource, // ใช้สำหรับ Transaction
@Inject(CACHE_MANAGER) private readonly cacheManager: Cache, // ADR-021 T024: History cache
@InjectRedis() private readonly redis: Redis // ADR-021 Clarify Q2: Redlock
@InjectRedis() private readonly redis: Redis, // ADR-021 Clarify Q2: Redlock
// ADR-021 S1: Redlock observability metrics
@InjectMetric('workflow_redlock_acquire_duration_ms')
private readonly redlockAcquireDuration: Histogram<string>,
@InjectMetric('workflow_redlock_acquire_failures_total')
private readonly redlockAcquireFailures: Counter<string>
) {
// ADR-021 Clarify Q2 (C1): Redlock Fail-closed
// Retry 3 ครั้ง × 500ms เพิ่ม jitter → ถ้ายังไม่ได้ throw HTTP 503
@@ -340,6 +343,8 @@ export class WorkflowEngineService {
// อนุญาตให้แนบไฟล์เฉพาะในสถานะ PENDING_REVIEW / PENDING_APPROVAL
// ==============================================================
if (hasAttachments) {
// ADR-021 S2: `id` ใน WorkflowInstance เป็น CHAR(36) UUID direct PK
// (ไม่ใช่ pattern UuidBaseEntity ที่ INT+publicId) — ADR-019 compliant เพราะ UUID ถูก expose โดยตรง
const instancePreCheck = await this.instanceRepo.findOne({
where: { id: instanceId },
select: ['id', 'currentState'],
@@ -352,12 +357,15 @@ export class WorkflowEngineService {
instancePreCheck.currentState
)
) {
throw new ConflictException({
userMessage: 'ไม่สามารถอัปโหลดไฟล์ในสถานะนี้ได้',
recoveryAction:
throw new ConflictException(
'WORKFLOW_STATE_LOCKED',
`Upload rejected: currentState=${instancePreCheck.currentState} not in UPLOAD_ALLOWED_STATES`,
'ไม่สามารถอัปโหลดไฟล์ในสถานะนี้ได้',
[
'อนุญาตเฉพาะสถานะ PENDING_REVIEW หรือ PENDING_APPROVAL เท่านั้น',
currentState: instancePreCheck.currentState,
});
'รีเฟรชหน้าแล้วตรวจสถานะล่าสุด',
]
);
}
}
@@ -367,25 +375,34 @@ export class WorkflowEngineService {
// ==============================================================
const lockKey = `lock:wf:transition:${instanceId}`;
let lock: Lock;
const acquireStart = Date.now();
try {
lock = await this.redlock.acquire([lockKey], 10000); // 10s TTL
// S1: บันทึก duration กรณี acquire สำเร็จ
this.redlockAcquireDuration
.labels({ outcome: 'success' })
.observe(Date.now() - acquireStart);
} catch (err) {
// S1: บันทึก duration + failure counter
this.redlockAcquireDuration
.labels({ outcome: 'failure' })
.observe(Date.now() - acquireStart);
this.redlockAcquireFailures.inc();
this.logger.error(
`Redlock acquire failed after retries for ${instanceId}: ${(err as Error).message}`
);
throw new ServiceUnavailableException({
userMessage: 'ระบบยุ่งชั่วคราว กรุณาลองใหม่ภายหลัง',
recoveryAction: 'รอสักครู่แล้วลองใหม่',
});
throw new ServiceUnavailableException(
'WORKFLOW_LOCK_UNAVAILABLE',
`Redlock acquire failed after 3 retries on lock:wf:transition:${instanceId}`,
'ระบบยุ่งชั่วคราว กรุณาลองใหม่ภายหลัง',
['รอสักครู่แล้วลองใหม่', 'แจ้งผู้ดูแลระบบหากยังพบปัญหา']
);
}
const queryRunner = this.dataSource.createQueryRunner();
await queryRunner.connect();
await queryRunner.startTransaction();
let eventsToDispatch: RawEvent[] = [];
let updatedContext: Record<string, unknown> = {};
try {
// 1. Lock Instance เพื่อป้องกัน Race Condition (Pessimistic Write Lock)
const instance = await queryRunner.manager.findOne(WorkflowInstance, {
@@ -407,6 +424,22 @@ export class WorkflowEngineService {
);
}
// ==============================================================
// ADR-021 (H1): Re-check state ภายใต้ pessimistic lock — ปิด TOCTOU race
// pre-check ด้านหน้าเป็น optimistic fast-fail; เช็กที่นี่เป็น authoritative
// ==============================================================
if (
hasAttachments &&
!WorkflowEngineService.UPLOAD_ALLOWED_STATES.has(instance.currentState)
) {
throw new ConflictException(
'WORKFLOW_STATE_CHANGED',
`TOCTOU: state changed to ${instance.currentState} under pessimistic lock`,
'ไม่สามารถอัปโหลดไฟล์ได้ (สถานะเอกสารได้เปลี่ยนไปก่อนหน้านี้)',
['รีเฟรชหน้าแล้วตรวจสถานะล่าสุดของเอกสาร']
);
}
// 2. Evaluate Logic ผ่าน DSL Service
const compiled = instance.definition
.compiled as unknown as CompiledWorkflow;
@@ -494,27 +527,23 @@ export class WorkflowEngineService {
)
);
// [NEW] เก็บค่าไว้ Dispatch หลัง Commit
eventsToDispatch = evaluation.events;
updatedContext = context;
this.logger.log(
`Transition: ${instanceId} [${fromState}] --${action}--> [${toState}] by User:${userId}`
);
// [NEW] Dispatch Events (Async) ผ่าน WorkflowEventService
if (eventsToDispatch && eventsToDispatch.length > 0) {
// Dispatch Events (Async, Fire-and-forget) ผ่าน WorkflowEventService
if (evaluation.events.length > 0) {
void this.eventService.dispatchEvents(
instance.id,
eventsToDispatch,
updatedContext
evaluation.events,
context
);
}
return {
success: true,
nextState: toState,
events: eventsToDispatch,
events: evaluation.events,
isCompleted: instance.status === WorkflowStatus.COMPLETED,
};
} catch (err) {
+1 -1
View File
@@ -1,6 +1,6 @@
import _request from 'supertest';
import { AppModule } from '../src/app.module';
import { _RoutingTemplate } from '../src/modules/correspondence/entities/routing-template.entity';
import { RoutingTemplate } from '../src/modules/correspondence/entities/routing-template.entity';
import { Test, TestingModule } from '@nestjs/testing';