diff --git a/backend/src/modules/ai/processors/ai-batch.processor.spec.ts b/backend/src/modules/ai/processors/ai-batch.processor.spec.ts index 27f263ec..a85fe681 100644 --- a/backend/src/modules/ai/processors/ai-batch.processor.spec.ts +++ b/backend/src/modules/ai/processors/ai-batch.processor.spec.ts @@ -84,6 +84,7 @@ describe('AiBatchProcessor', () => { }; const mockRedis = { setex: jest.fn().mockResolvedValue('OK'), + get: jest.fn().mockResolvedValue(null), }; const mockAttachmentRepo = { findOne: jest.fn().mockResolvedValue({ @@ -143,6 +144,7 @@ describe('AiBatchProcessor', () => { resolvedPrompt: 'Resolved test prompt with OCR text', versionNumber: 2, }), + findByVersion: jest.fn().mockResolvedValue(null), saveTestResult: jest.fn().mockResolvedValue(undefined), }; beforeEach(async () => { @@ -315,6 +317,68 @@ describe('AiBatchProcessor', () => { expect.stringContaining('completed') ); }); + it('sandbox-ai-extract ควร regenerate response ใหม่เมื่อ parse JSON ครั้งแรกล้มเหลว', async () => { + const cachedOcrPayload = { + ocrText: 'OCR text for retry test\u0002\u0000', + ocrUsed: true, + engineUsed: 'typhoon-np-dms-ocr', + fallbackUsed: false, + timestamp: '2026-06-06T15:00:00.000Z', + }; + mockRedis.get = jest + .fn() + .mockResolvedValueOnce(JSON.stringify(cachedOcrPayload)); + mockAiPromptsService.findByVersion = jest.fn().mockResolvedValue({ + id: 1, + promptType: 'ocr_extraction', + versionNumber: 2, + template: + 'Resolved test prompt with OCR text {{ocr_text}} and context {{master_data_context}}', + isActive: true, + contextConfig: { filter: {} }, + }); + mockOllamaService.generate + .mockResolvedValueOnce('{\u0002\u0000') + .mockResolvedValueOnce( + JSON.stringify({ + subject: 'Recovered after retry', + confidence: 0.91, + tags: ['retry'], + }) + ); + const job = { + id: 'job-ai-extract-retry', + data: { + jobType: 'sandbox-ai-extract', + documentPublicId: 'idem-ai-extract-123', + projectPublicId: 'default', + payload: { promptVersion: 2 }, + idempotencyKey: 'idem-ai-extract-123', + }, + } as unknown as Job; + await processor.process(job); + expect(mockOllamaService.generate).toHaveBeenCalledTimes(2); + expect(mockOllamaService.generate).toHaveBeenNthCalledWith( + 1, + expect.not.stringContaining('\u0002'), + expect.objectContaining({ + timeoutMs: 120000, + }) + ); + expect(mockAiPromptsService.saveTestResult).toHaveBeenCalledWith( + 'ocr_extraction', + 2, + expect.objectContaining({ + subject: 'Recovered after retry', + confidence: 0.91, + }) + ); + expect(mockRedis.setex).toHaveBeenLastCalledWith( + 'ai:rag:result:idem-ai-extract-123', + 3600, + expect.stringContaining('"llmPrompt"') + ); + }); it('EC-001: ควรบันทึก aiIssues เมื่อ AI สกัด Tag ใหม่ที่ไม่มีในระบบ', async () => { mockTagsService.findOrSuggestTags.mockResolvedValueOnce([ { diff --git a/backend/src/modules/ai/processors/ai-batch.processor.ts b/backend/src/modules/ai/processors/ai-batch.processor.ts index c17a2aab..e3e00554 100644 --- a/backend/src/modules/ai/processors/ai-batch.processor.ts +++ b/backend/src/modules/ai/processors/ai-batch.processor.ts @@ -78,6 +78,21 @@ export interface AiBatchJobData { /** OCR text สูงสุดที่ส่งเข้า LLM prompt — ป้องกัน context overflow (num_ctx 8192, Thai ~3 chars/token) */ const MAX_OCR_TEXT_CHARS = 15000; +const MAX_JSON_PARSE_ATTEMPTS = 2; +const removeControlCharacters = ( + value: string, + includeDeleteCharacter = false +): string => + Array.from(value) + .filter((character) => { + const code = character.charCodeAt(0); + const isAsciiControl = + (code >= 0 && code <= 8) || code === 11 || code === 12; + const isAdditionalControl = code >= 14 && code <= 31; + const isDeleteCharacter = includeDeleteCharacter && code === 127; + return !isAsciiControl && !isAdditionalControl && !isDeleteCharacter; + }) + .join(''); const readString = (value: unknown): string | undefined => typeof value === 'string' && value.trim().length > 0 ? value : undefined; @@ -145,6 +160,14 @@ const parseMigrateDocumentMetadata = ( }; }; +const sanitizeLlmJsonResponse = (response: string): string => + removeControlCharacters( + response.replace(/```json/g, '').replace(/```/g, '') + ).trim(); + +const sanitizeOcrText = (text: string): string => + removeControlCharacters(text.replace(/\r\n/g, '\n'), true).trim(); + /** Processor สำหรับงาน AI batch ที่รันทีละงานเพื่อคุม VRAM * lockDuration: 150000ms — รองรับ Ollama sandbox ที่ใช้เวลาสูงสุด 120s (ADR-029 FR-008) * ค่า default ของ BullMQ คือ 30000ms ซึ่งน้อยกว่า timeout → job stall @@ -174,6 +197,51 @@ export class AiBatchProcessor extends WorkerHost { super(); } + /** เรียก LLM แล้ว parse JSON แบบ retry จริงเมื่อได้ผลลัพธ์ไม่สมบูรณ์ */ + private async generateStructuredJson( + prompt: string, + options: { timeoutMs: number; model?: string; system?: string } + ): Promise<{ + extractedMetadata: Record; + rawResponse: string; + cleanedResponse: string; + }> { + let lastRawResponse = ''; + let lastCleanedResponse = ''; + for (let attempt = 1; attempt <= MAX_JSON_PARSE_ATTEMPTS; attempt += 1) { + const rawResponse = await this.ollamaService.generate(prompt, options); + const cleanedResponse = sanitizeLlmJsonResponse(rawResponse); + lastRawResponse = rawResponse; + lastCleanedResponse = cleanedResponse; + this.logger.debug(`Raw LLM response: ${rawResponse}`); + try { + return { + extractedMetadata: JSON.parse(cleanedResponse) as Record< + string, + unknown + >, + rawResponse, + cleanedResponse, + }; + } catch { + if (attempt >= MAX_JSON_PARSE_ATTEMPTS) { + this.logger.error( + `Failed to parse LLM response as JSON after ${MAX_JSON_PARSE_ATTEMPTS} attempts. Raw: ${lastRawResponse}, Cleaned: ${lastCleanedResponse}` + ); + throw new Error( + `Failed to parse LLM response as JSON after ${MAX_JSON_PARSE_ATTEMPTS} attempts. Raw: ${lastRawResponse.substring(0, 200)}, Cleaned: ${lastCleanedResponse.substring(0, 200)}` + ); + } + this.logger.warn( + `JSON parse attempt ${attempt} failed, regenerating response...` + ); + } + } + throw new Error( + `Failed to parse LLM response as JSON after ${MAX_JSON_PARSE_ATTEMPTS} attempts` + ); + } + /** Dispatch งาน batch ตาม jobType */ async process(job: Job): Promise { const isSandbox = @@ -410,6 +478,12 @@ export class AiBatchProcessor extends WorkerHost { pdfPath, engineType ); + const sanitizedOcrText = sanitizeOcrText(ocrResult.text); + if (sanitizedOcrText.length !== ocrResult.text.length) { + this.logger.warn( + `OCR text sanitized before LLM: raw=${ocrResult.text.length} chars, sanitized=${sanitizedOcrText.length} chars` + ); + } const activePrompt = await this.aiPromptsService.getActive('ocr_extraction'); @@ -426,12 +500,12 @@ export class AiBatchProcessor extends WorkerHost { ); const ocrTextSafe = - ocrResult.text.length > MAX_OCR_TEXT_CHARS + sanitizedOcrText.length > MAX_OCR_TEXT_CHARS ? (this.logger.warn( - `OCR text truncated: ${ocrResult.text.length} chars > ${MAX_OCR_TEXT_CHARS} limit (context overflow protection)` + `OCR text truncated: ${sanitizedOcrText.length} chars > ${MAX_OCR_TEXT_CHARS} limit (context overflow protection)` ), - ocrResult.text.substring(0, MAX_OCR_TEXT_CHARS)) - : ocrResult.text; + sanitizedOcrText.substring(0, MAX_OCR_TEXT_CHARS)) + : sanitizedOcrText; const resolvedPrompt = activePrompt.template .replace('{{ocr_text}}', ocrTextSafe) @@ -444,45 +518,12 @@ export class AiBatchProcessor extends WorkerHost { `Prompt stats: OCR=${ocrTextSafe.length} chars, MasterData=${JSON.stringify(masterDataContext, null, 2).length} chars, Total=${resolvedPrompt.length} chars` ); - const response = await this.ollamaService.generate(resolvedPrompt, { - timeoutMs: 120000, - }); - this.logger.debug(`Raw LLM response: ${response}`); - const cleanedResponse = response - .replace(/```json/g, '') - .replace(/```/g, '') - .trim(); - let extractedMetadata: Record | null = null; - let parseAttempts = 0; - const maxParseAttempts = 2; - while (parseAttempts < maxParseAttempts) { - try { - extractedMetadata = JSON.parse(cleanedResponse) as Record< - string, - unknown - >; - break; - } catch { - parseAttempts++; - if (parseAttempts >= maxParseAttempts) { - this.logger.error( - `Failed to parse LLM response as JSON after ${maxParseAttempts} attempts. Raw: ${response}, Cleaned: ${cleanedResponse}` - ); - throw new Error( - `Failed to parse LLM response as JSON after ${maxParseAttempts} attempts. Raw: ${response.substring(0, 200)}, Cleaned: ${cleanedResponse.substring(0, 200)}` - ); - } - this.logger.warn( - `JSON parse attempt ${parseAttempts} failed, retrying...` - ); - await new Promise((resolve) => setTimeout(resolve, 1000)); + const { extractedMetadata } = await this.generateStructuredJson( + resolvedPrompt, + { + timeoutMs: 120000, } - } - if (!extractedMetadata) { - throw new Error( - `Failed to parse LLM response as JSON after ${maxParseAttempts} attempts` - ); - } + ); await this.aiPromptsService.saveTestResult( 'ocr_extraction', activePrompt.versionNumber, @@ -495,7 +536,7 @@ export class AiBatchProcessor extends WorkerHost { requestPublicId: idempotencyKey, status: 'completed', answer: JSON.stringify(extractedMetadata, null, 2), - ocrText: ocrResult.text, + ocrText: sanitizedOcrText, ocrUsed: ocrResult.ocrUsed, engineUsed: ocrResult.engineUsed, fallbackUsed: ocrResult.fallbackUsed, @@ -549,13 +590,19 @@ export class AiBatchProcessor extends WorkerHost { engineType, typhoonOptions ); + const sanitizedOcrText = sanitizeOcrText(ocrResult.text); + if (sanitizedOcrText.length !== ocrResult.text.length) { + this.logger.warn( + `OCR text sanitized before cache: raw=${ocrResult.text.length} chars, sanitized=${sanitizedOcrText.length} chars` + ); + } // Cache OCR text สำหรับ Step 2 await this.redis.setex( `ai:sandbox:ocr:${idempotencyKey}`, 3600, JSON.stringify({ - ocrText: ocrResult.text, + ocrText: sanitizedOcrText, ocrUsed: ocrResult.ocrUsed, engineUsed: ocrResult.engineUsed, fallbackUsed: ocrResult.fallbackUsed, @@ -569,7 +616,7 @@ export class AiBatchProcessor extends WorkerHost { JSON.stringify({ requestPublicId: idempotencyKey, status: 'completed', - ocrText: ocrResult.text, + ocrText: sanitizedOcrText, ocrUsed: ocrResult.ocrUsed, engineUsed: ocrResult.engineUsed, fallbackUsed: ocrResult.fallbackUsed, @@ -624,7 +671,12 @@ export class AiBatchProcessor extends WorkerHost { fallbackUsed?: boolean; timestamp: string; }; - const { ocrText } = parsedOcr; + const ocrText = sanitizeOcrText(parsedOcr.ocrText); + if (ocrText.length !== parsedOcr.ocrText.length) { + this.logger.warn( + `Cached OCR text sanitized before AI extraction: raw=${parsedOcr.ocrText.length} chars, sanitized=${ocrText.length} chars` + ); + } // ดึง prompt version const activePrompt = @@ -667,48 +719,15 @@ export class AiBatchProcessor extends WorkerHost { '{{master_data_context}}', JSON.stringify(masterDataContext, null, 2) ); - - const response = await this.ollamaService.generate(resolvedPrompt, { - timeoutMs: 120000, - }); - - this.logger.debug(`Raw LLM response: ${response}`); - const cleanedResponse = response - .replace(/```json/g, '') - .replace(/```/g, '') - .trim(); - - let extractedMetadata: Record | null = null; - let parseAttempts = 0; - const maxParseAttempts = 2; - while (parseAttempts < maxParseAttempts) { - try { - extractedMetadata = JSON.parse(cleanedResponse) as Record< - string, - unknown - >; - break; - } catch { - parseAttempts++; - if (parseAttempts >= maxParseAttempts) { - this.logger.error( - `Failed to parse LLM response as JSON after ${maxParseAttempts} attempts. Raw: ${response}, Cleaned: ${cleanedResponse}` - ); - throw new Error( - `Failed to parse LLM response as JSON after ${maxParseAttempts} attempts. Raw: ${response.substring(0, 200)}, Cleaned: ${cleanedResponse.substring(0, 200)}` - ); - } - this.logger.warn( - `JSON parse attempt ${parseAttempts} failed, retrying...` - ); - await new Promise((resolve) => setTimeout(resolve, 1000)); + this.logger.debug( + `Prompt stats: OCR=${ocrTextSafe.length} chars, MasterData=${JSON.stringify(masterDataContext, null, 2).length} chars, Total=${resolvedPrompt.length} chars` + ); + const { extractedMetadata } = await this.generateStructuredJson( + resolvedPrompt, + { + timeoutMs: 120000, } - } - if (!extractedMetadata) { - throw new Error( - `Failed to parse LLM response as JSON after ${maxParseAttempts} attempts` - ); - } + ); await this.aiPromptsService.saveTestResult( 'ocr_extraction', @@ -728,6 +747,7 @@ export class AiBatchProcessor extends WorkerHost { engineUsed: parsedOcr.engineUsed, fallbackUsed: parsedOcr.fallbackUsed, promptVersionUsed: targetPrompt.versionNumber, + llmPrompt: resolvedPrompt, completedAt: new Date().toISOString(), }) ); diff --git a/memory/agent-memory.md b/memory/agent-memory.md index 22692b9c..448d2225 100644 --- a/memory/agent-memory.md +++ b/memory/agent-memory.md @@ -20,6 +20,7 @@ - 2026-06-05 (Session 15): Feature 234 RAG Pipeline สมบูรณ์ — implement BGE-M3 embedding (dense 1024 + sparse), BGE-Reranker-Large, Semantic Chunking (typhoon2.5 + `` tags + fallback), Hybrid Qdrant schema (drop+recreate), workflow hook `syncStatus()` → `enqueueRagPrepare()`, processRagPrepare pipeline ใน ai-batch.processor; แก้ CRITICAL 2 ประเด็นจาก speckit-analyze; ผ่าน speckit-tester (19/19 tests), speckit-validate (15/15 FR, ทุก SC); ปิด Gap ทั้ง 2 รายการ (jobId dedup confirmed + integration test 9 tests); สร้าง validation-report.md ใน specs/200-fullstacks/234-rag-pipeline-enhancements/ - 2026-06-06: เพิ่ม MCP MariaDB Tools section ใน memory/agent-memory.md, AGENTS.md, และ rule files (.agents/rules/08-development-flow.md, .devin/rules/08-development-flow.md) — รวม 8 tools (test_connection, show_databases, show_tables, describe_table, query, insert, update, delete), การใช้งานร่วมกับ Development Flow, และข้อควรระวังเรื่อง DDL operations - 2026-06-06: เพิ่ม MCP Memory Tools section ใน memory/agent-memory.md — รวม 9 tools (create_entities, create_relations, add_observations, delete_entities, delete_relations, delete_observations, open_nodes, read_graph, search_nodes), การใช้งานร่วมกับ Development Flow สำหรับจัดการ Knowledge Graph และ Long-term Memory +- 2026-06-06 (Session 15): LLM JSON Parse Failure & VRAM Fix (ADR-035-135) — แก้ปัญหา JSON parse failure ใน Sandbox AI Extraction โดยเพิ่ม retry logic (2 attempts) + enhanced error logging ใน ai-batch.processor.ts, เพิ่ม system prompt support ใน ollama.service.ts, แก้ VRAM contention โดยเปลี่ยน keep_alive=0 ใน ocr-sidecar/app.py (Desk-5439), แก้ ESLint seg fault โดยลด heap size 8192 → 4096 ใน backend/package.json, แก้ Schema mismatch โดยสร้าง delta SQL 2026-06-06-add-ai-prompts-public-id.sql เพิ่ม public_id และ context_config columns ใน ai_prompts table, เพิ่ม prompt stats log + llmPrompt field ใน Sandbox Result, เพิ่ม LLM Prompt UI card (สีม่วง) ใน OcrSandboxPromptManager.tsx --> # 🧠 Agent Long-term Project Memory diff --git a/specs/03-Data-and-Storage/deltas/2026-06-06-add-ai-prompts-public-id.sql b/specs/03-Data-and-Storage/deltas/2026-06-06-add-ai-prompts-public-id.sql new file mode 100644 index 00000000..fdf95e7f --- /dev/null +++ b/specs/03-Data-and-Storage/deltas/2026-06-06-add-ai-prompts-public-id.sql @@ -0,0 +1,23 @@ +-- Delta: เพิ่ม public_id และ context_config columns ใน ai_prompts +-- Date: 2026-06-06 +-- Related ADR: ADR-019 (UUID strategy), ADR-029 (Dynamic Prompts) +-- ------------------------------------------------------------ +-- การเปลี่ยนแปลงโครงสร้างฐานข้อมูล (Schema changes) +-- ------------------------------------------------------------ + +-- เพิ่ม public_id column (UUIDv7) สำหรับ ADR-019 compliance +ALTER TABLE ai_prompts +ADD COLUMN public_id UUID UNIQUE COMMENT 'Public UUID สำหรับ API (ADR-019)'; + +-- เพิ่ม context_config column สำหรับ ADR-029 context filtering +ALTER TABLE ai_prompts +ADD COLUMN context_config JSON NULL COMMENT 'Configuration สำหรับ Master Data context filtering (project/contract scope)'; + +-- สร้าง UUID สำหรับ records ที่มีอยู่แล้ว +UPDATE ai_prompts +SET public_id = UUID() +WHERE public_id IS NULL; + +-- ตั้ง public_id เป็น NOT NULL หลังจาก populate ครบแล้ว +ALTER TABLE ai_prompts +MODIFY COLUMN public_id UUID NOT NULL; diff --git a/specs/88-logs/_backend_logs.md b/specs/88-logs/_backend_logs.md index 49c8af6f..b436337b 100644 Binary files a/specs/88-logs/_backend_logs.md and b/specs/88-logs/_backend_logs.md differ