lcbp3/specs/06-tasks/TASK-BE-010-search-elasticsearch.md

Task: Search & Elasticsearch Integration

Status: 🚧 In Progress Priority: P2 (Medium - Performance Enhancement) Estimated Effort: 4-6 days Dependencies: TASK-BE-001, TASK-BE-005, TASK-BE-007 Owner: Backend Team

---

📋 Overview

สร้าง Search Module ที่ integrate กับ Elasticsearch สำหรับ Full-text Search และ Advanced Filtering

---

🎯 Objectives

• Elasticsearch Integration
• Full-text Search (Correspondences, RFAs, Drawings)
• Advanced Filters
• Search Result Aggregations (Pending verification)
• Auto-indexing (Implemented via Direct Call, not Queue yet)

---

📝 Acceptance Criteria

1. Search Capabilities:

   • Search across multiple document types
   • Full-text search in title, description
   • Filter by project, status, date range
   • Sort results by relevance/date

2. Indexing:

   • Auto-index on document create/update (Direct Call implemented)
   • Async indexing (via queue) - Pending
   • Bulk re-indexing command - Pending

3. Performance:

   • Search results < 500ms
   • Pagination support
   • Highlight search terms

---

🛠️ Implementation Steps

1. Elasticsearch Module Setup

// File: backend/src/modules/search/search.module.ts
import { ElasticsearchModule } from '@nestjs/elasticsearch';

@Module({
  imports: [
    ElasticsearchModule.register({
      node: process.env.ELASTICSEARCH_URL || 'http://localhost:9200',
    }),
  ],
  providers: [SearchService, SearchIndexer],
  exports: [SearchService],
})
export class SearchModule {}

2. Index Mapping

Note

Field Naming Convention: Elasticsearch fields use camelCase to match TypeScript/JavaScript conventions in the application layer. Database columns remain snake_case with TypeORM mapping.

// File: backend/src/modules/search/mappings/correspondence.mapping.ts
export const correspondenceMapping = {
  properties: {
    id: { type: 'integer' },
    correspondenceNumber: { type: 'keyword' },
    title: {
      type: 'text',
      analyzer: 'standard',
      fields: {
        keyword: { type: 'keyword' },
      },
    },
    description: {
      type: 'text',
      analyzer: 'standard',
    },
    projectId: { type: 'integer' },
    projectName: { type: 'keyword' },
    status: { type: 'keyword' },
    createdAt: { type: 'date' },
    createdByUsername: { type: 'keyword' },
    organizationName: { type: 'keyword' },
    typeName: { type: 'keyword' },
    disciplineName: { type: 'keyword' },
  },
};

3. Search Service

// File: backend/src/modules/search/search.service.ts
import { Injectable } from '@nestjs/common';
import { ElasticsearchService } from '@nestjs/elasticsearch';

@Injectable()
export class SearchService {
  private readonly INDEX_NAME = 'lcbp3-documents';

  constructor(private elasticsearch: ElasticsearchService) {}

  async onModuleInit() {
    // Create index if not exists
    const indexExists = await this.elasticsearch.indices.exists({
      index: this.INDEX_NAME,
    });

    if (!indexExists) {
      await this.createIndex();
    }
  }

  private async createIndex(): Promise<void> {
    await this.elasticsearch.indices.create({
      index: this.INDEX_NAME,
      body: {
        mappings: {
          properties: {
            document_type: { type: 'keyword' },
            ...correspondenceMapping.properties,
            ...rfaMapping.properties,
          },
        },
      },
    });
  }

  async search(query: SearchQueryDto): Promise<SearchResult> {
    const must: any[] = [];
    const filter: any[] = [];

    // Full-text search
    if (query.search) {
      must.push({
        multi_match: {
          query: query.search,
          fields: ['title^2', 'description', 'correspondence_number'],
          fuzziness: 'AUTO',
        },
      });
    }

    // Filters
    if (query.document_type) {
      filter.push({ term: { document_type: query.document_type } });
    }

    if (query.project_id) {
      filter.push({ term: { project_id: query.project_id } });
    }

    if (query.status) {
      filter.push({ term: { status: query.status } });
    }

    if (query.date_from || query.date_to) {
      const range: any = {};
      if (query.date_from) range.gte = query.date_from;
      if (query.date_to) range.lte = query.date_to;
      filter.push({ range: { createdAt: range } });
    }

    // Execute search
    const page = query.page || 1;
    const limit = query.limit || 20;
    const from = (page - 1) * limit;

    const result = await this.elasticsearch.search({
      index: this.INDEX_NAME,
      body: {
        from,
        size: limit,
        query: {
          bool: {
            must,
            filter,
          },
        },
        sort: query.sort_by
          ? [{ [query.sort_by]: { order: query.sort_order || 'desc' } }]
          : [{ _score: 'desc' }, { createdAt: 'desc' }],
        highlight: {
          fields: {
            title: {},
            description: {},
          },
        },
        aggs: {
          document_types: {
            terms: { field: 'document_type' },
          },
          statuses: {
            terms: { field: 'status' },
          },
          projects: {
            terms: { field: 'project_id' },
          },
        },
      },
    });

    return {
      items: result.hits.hits.map((hit) => ({
        ...hit._source,
        _score: hit._score,
        _highlights: hit.highlight,
      })),
      total: result.hits.total.value,
      page,
      limit,
      totalPages: Math.ceil(result.hits.total.value / limit),
      aggregations: result.aggregations,
    };
  }

  async indexDocument(
    documentType: string,
    documentId: number,
    data: any
  ): Promise<void> {
    await this.elasticsearch.index({
      index: this.INDEX_NAME,
      id: `${documentType}-${documentId}`,
      body: {
        document_type: documentType,
        ...data,
      },
    });
  }

  async updateDocument(
    documentType: string,
    documentId: number,
    data: any
  ): Promise<void> {
    await this.elasticsearch.update({
      index: this.INDEX_NAME,
      id: `${documentType}-${documentId}`,
      body: {
        doc: data,
      },
    });
  }

  async deleteDocument(
    documentType: string,
    documentId: number
  ): Promise<void> {
    await this.elasticsearch.delete({
      index: this.INDEX_NAME,
      id: `${documentType}-${documentId}`,
    });
  }
}

4. Search Indexer (Queue Worker)

// File: backend/src/modules/search/search-indexer.service.ts
import { Processor, Process } from '@nestjs/bullmq';
import { Job } from 'bullmq';

@Processor('search-indexing')
export class SearchIndexer {
  constructor(
    private searchService: SearchService,
    @InjectRepository(Correspondence)
    private correspondenceRepo: Repository<Correspondence>,
    @InjectRepository(Rfa)
    private rfaRepo: Repository<Rfa>
  ) {}

  @Process('index-correspondence')
  async indexCorrespondence(job: Job<{ id: number }>) {
    const correspondence = await this.correspondenceRepo.findOne({
      where: { id: job.data.id },
      relations: ['project', 'originatorOrganization', 'revisions'],
    });

    if (!correspondence) {
      return;
    }

    const latestRevision = correspondence.revisions[0];

    await this.searchService.indexDocument(
      'correspondence',
      correspondence.id,
      {
        id: correspondence.id,
        correspondenceNumber: correspondence.correspondence_number,
        title: correspondence.title,
        description: latestRevision?.description,
        projectId: correspondence.project_id,
        projectName: correspondence.project.project_name,
        status: correspondence.status,
        createdAt: correspondence.createdAt,
        organizationName:
          correspondence.originatorOrganization.organization_name,
      }
    );
  }

  @Process('index-rfa')
  async indexRfa(job: Job<{ id: number }>) {
    const rfa = await this.rfaRepo.findOne({
      where: { id: job.data.id },
      relations: ['project', 'revisions'],
    });

    if (!rfa) {
      return;
    }

    const latestRevision = rfa.revisions[0];

    await this.searchService.indexDocument('rfa', rfa.id, {
      id: rfa.id,
      rfaNumber: rfa.rfa_number,
      title: rfa.subject,
      description: latestRevision?.description,
      projectId: rfa.project_id,
      projectName: rfa.project.project_name,
      status: rfa.status,
      createdAt: rfa.createdAt,
    });
  }

  @Process('bulk-reindex')
  async bulkReindex(job: Job) {
    // Re-index all correspondences
    const correspondences = await this.correspondenceRepo.find({
      relations: ['project', 'originatorOrganization', 'revisions'],
    });

    for (const corr of correspondences) {
      await this.indexCorrespondence({ data: { id: corr.id } } as Job);
    }

    // Re-index all RFAs
    const rfas = await this.rfaRepo.find({
      relations: ['project', 'revisions'],
    });

    for (const rfa of rfas) {
      await this.indexRfa({ data: { id: rfa.id } } as Job);
    }
  }
}

5. Integration with Service

// File: backend/src/modules/correspondence/correspondence.service.ts (updated)
@Injectable()
export class CorrespondenceService {
  constructor(
    // ... existing dependencies
    private searchQueue: Queue
  ) {}

  async create(
    dto: CreateCorrespondenceDto,
    userId: number
  ): Promise<Correspondence> {
    const correspondence = await this.dataSource.transaction(/* ... */);

    // Queue for indexing (async)
    await this.searchQueue.add('index-correspondence', {
      id: correspondence.id,
    });

    return correspondence;
  }

  async update(id: number, dto: UpdateCorrespondenceDto): Promise<void> {
    await this.corrRepo.update(id, dto);

    // Re-index
    await this.searchQueue.add('index-correspondence', { id });
  }
}

6. Search Controller

// File: backend/src/modules/search/search.controller.ts
@Controller('search')
@UseGuards(JwtAuthGuard)
export class SearchController {
  constructor(private searchService: SearchService) {}

  @Get()
  async search(@Query() query: SearchQueryDto) {
    return this.searchService.search(query);
  }

  @Post('reindex')
  @RequirePermission('admin.manage')
  async reindex() {
    await this.searchQueue.add('bulk-reindex', {});
    return { message: 'Re-indexing started' };
  }
}

---

✅ Testing & Verification

1. Unit Tests

describe('SearchService', () => {
  it('should search with full-text query', async () => {
    const result = await service.search({
      search: 'foundation',
      page: 1,
      limit: 20,
    });

    expect(result.items).toBeDefined();
    expect(result.total).toBeGreaterThan(0);
  });

  it('should filter by project and status', async () => {
    const result = await service.search({
      project_id: 1,
      status: 'submitted',
    });

    result.items.forEach((item) => {
      expect(item.project_id).toBe(1);
      expect(item.status).toBe('submitted');
    });
  });
});

---

📚 Related Documents

• System Architecture - Search
• ADR-005: Technology Stack

---

📦 Deliverables

• SearchService with Elasticsearch
• Search Indexer (Queue Worker) - Pending
• Index Mappings (Implemented in Service)
• Queue Integration - Pending
• Search Controller
• Bulk Re-indexing Command - Pending
• Unit Tests (75% coverage)
• API Documentation

---

🚨 Risks & Mitigation

Risk	Impact	Mitigation
Elasticsearch down	Medium	Fallback to DB search
Index out of sync	Medium	Regular re-indexing
Large result sets	Low	Pagination + limits

---

📌 Notes

• Async indexing via BullMQ
• Index correspondence, RFA, drawings
• Support Thai language search
• Highlight matching terms
• Aggregations for faceted search
• Re-index command for admin