260314:1714 20260314:1700 Refactor Migration #2

backend/src/modules/migration/dto/commit-batch.dto.ts

@@ -0,0 +1,23 @@
+import { IsArray, ValidateNested, IsString, IsNotEmpty } from 'class-validator';
+import { Type } from 'class-transformer';
+import { ImportCorrespondenceDto } from './import-correspondence.dto';
+
+export class CommitBatchItemDto {
+  @IsNotEmpty()
+  queueId!: number;
+
+  @ValidateNested()
+  @Type(() => ImportCorrespondenceDto)
+  dto!: ImportCorrespondenceDto;
+}
+
+export class CommitBatchDto {
+  @IsArray()
+  @ValidateNested({ each: true })
+  @Type(() => CommitBatchItemDto)
+  items!: CommitBatchItemDto[];
+
+  @IsString()
+  @IsNotEmpty()
+  batchId!: string;
+}

backend/src/modules/migration/migration.controller.ts

@@ -2,6 +2,7 @@ import { Controller, Post, Body, Headers, UseGuards, Get, Param, Query, Res, Par
 import { MigrationService } from './migration.service';
 import { ImportCorrespondenceDto } from './dto/import-correspondence.dto';
 import { EnqueueMigrationDto } from './dto/enqueue-migration.dto';
+import { CommitBatchDto } from './dto/commit-batch.dto';
 import { JwtAuthGuard } from '../../common/guards/jwt-auth.guard';
 import { CurrentUser } from '../../common/decorators/current-user.decorator';
 import { ApiTags, ApiOperation, ApiBearerAuth, ApiHeader, ApiQuery, ApiParam } from '@nestjs/swagger';
@@ -31,6 +32,23 @@ export class MigrationController {
     return this.migrationService.importCorrespondence(dto, idempotencyKey, userId);
   }
 
+  @Post('commit_batch')
+  @UseGuards(JwtAuthGuard)
+  @ApiOperation({ summary: 'Batch approve and import migration review queue items' })
+  @ApiHeader({
+    name: 'Idempotency-Key',
+    description: 'Unique key for the entire batch to prevent duplicate execution',
+    required: true,
+  })
+  async commitBatch(
+    @Body() dto: CommitBatchDto,
+    @Headers('idempotency-key') idempotencyKey: string,
+    @CurrentUser() user: any
+  ) {
+    const userId = user?.id || user?.userId || 5;
+    return this.migrationService.commitBatch(dto, idempotencyKey, userId);
+  }
+
   @Post('queue')
   @UseGuards(JwtAuthGuard)
   @ApiOperation({ summary: 'Enqueue a record into the staging migration review queue' })

backend/src/modules/migration/migration.service.ts

@@ -9,6 +9,7 @@ import { InjectRepository } from '@nestjs/typeorm';
 import { Repository, DataSource } from 'typeorm';
 import { ImportCorrespondenceDto } from './dto/import-correspondence.dto';
 import { EnqueueMigrationDto } from './dto/enqueue-migration.dto';
+import { CommitBatchDto } from './dto/commit-batch.dto';
 import { ImportTransaction } from './entities/import-transaction.entity';
 import { Correspondence } from '../correspondence/entities/correspondence.entity';
 import { CorrespondenceRevision } from '../correspondence/entities/correspondence-revision.entity';
@@ -484,11 +485,11 @@ export class MigrationService {
   async approveQueueItem(id: number, dto: ImportCorrespondenceDto, idempotencyKey: string, userId: number) {
     const queueItem = await this.reviewQueueRepo.findOne({ where: { id } });
     if (!queueItem) {
-      throw new BadRequestException('Queue item not found');
+      throw new BadRequestException(`Queue item ${id} not found`);
     }
 
     if (queueItem.status !== MigrationReviewStatus.PENDING) {
-      throw new BadRequestException(`Queue item is already ${queueItem.status}`);
+      throw new BadRequestException(`Queue item ${id} is already ${queueItem.status}`);
     }
 
     // Attempt the import
@@ -503,6 +504,45 @@ export class MigrationService {
     return result;
   }
 
+  async commitBatch(dto: CommitBatchDto, idempotencyKey: string, userId: number) {
+    if (!idempotencyKey) {
+      throw new BadRequestException('Idempotency-Key header is required');
+    }
+
+    const results = [];
+    const errors = [];
+    
+    // We let each import have its own transaction via approveQueueItem 
+    // to avoid one bad record failing the entire batch of valid ones.
+    
+    for (const item of dto.items) {
+       // Create a unique sub-key for each item to avoid idempotency conflicts
+       // when using a batch idempotency key.
+       const subKey = `${idempotencyKey}_${item.queueId}`;
+       
+       // Force batchId on the item dto
+       item.dto.batch_id = dto.batchId;
+       
+       try {
+           const result = await this.approveQueueItem(item.queueId, item.dto, subKey, userId);
+           results.push({ queueId: item.queueId, result });
+       } catch (err: unknown) {
+           const errorMessage = err instanceof Error ? err.message : String(err);
+           errors.push({ queueId: item.queueId, error: errorMessage });
+           this.logger.error(`Batch commit failed for queue ID ${item.queueId}: ${errorMessage}`);
+       }
+    }
+    
+    return {
+        message: 'Batch processing completed',
+        batchId: dto.batchId,
+        processed: results.length,
+        failed: errors.length,
+        results,
+        errors
+    };
+  }
+
   async rejectQueueItem(id: number, userId: number) {
     const queueItem = await this.reviewQueueRepo.findOne({ where: { id } });
     if (!queueItem) {

frontend/app/(dashboard)/admin/migration/page.tsx

@@ -11,18 +11,21 @@ import {
   TableHeader,
   TableRow,
 } from "@/components/ui/table";
+import { Checkbox } from "@/components/ui/checkbox";
 import { Badge } from "@/components/ui/badge";
 import { Button } from "@/components/ui/button";
 import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
 import { format } from "date-fns";
-import { EyeIcon, FileXIcon } from "lucide-react";
+import { EyeIcon, FileXIcon, CheckSquareIcon } from "lucide-react";
 import Link from "next/link";
 import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
 
 export default function MigrationReviewQueuePage() {
   const [items, setItems] = useState<MigrationReviewQueueItem[]>([]);
   const [loading, setLoading] = useState(true);
+  const [submitting, setSubmitting] = useState(false);
   const [statusFilter, setStatusFilter] = useState<string>("PENDING");
+  const [selectedIds, setSelectedIds] = useState<number[]>([]);
 
   useEffect(() => {
     fetchData();
@@ -36,6 +39,7 @@ export default function MigrationReviewQueuePage() {
         limit: 50,
       });
       setItems(res.items);
+      setSelectedIds([]); // reset selection on fetch
     } catch (error) {
       console.error("Failed to fetch queue", error);
     } finally {
@@ -43,6 +47,63 @@ export default function MigrationReviewQueuePage() {
     }
   };
 
+  const handleToggleSelectAll = () => {
+    if (selectedIds.length === items.length) {
+      setSelectedIds([]);
+    } else {
+      setSelectedIds(items.map((i) => i.id));
+    }
+  };
+
+  const handleToggleSelect = (id: number) => {
+    setSelectedIds((prev) => 
+      prev.includes(id) ? prev.filter((i) => i !== id) : [...prev, id]
+    );
+  };
+
+  const handleBatchApprove = async () => {
+    if (selectedIds.length === 0) return;
+    try {
+      setSubmitting(true);
+      
+      const batchItems = items
+        .filter((i) => selectedIds.includes(i.id))
+        .map((item) => ({
+          queueId: item.id,
+          dto: {
+            document_number: item.documentNumber,
+            subject: item.title || item.originalTitle || 'Untitled',
+            category: item.aiSuggestedCategory || 'Correspondence',
+            project_id: item.projectId || 1,
+            migrated_by: 'SYSTEM_IMPORT',
+            temp_attachment_id: item.tempAttachmentId,
+            ai_confidence: item.aiConfidence,
+            ai_issues: item.aiIssues,
+            issued_date: item.issuedDate,
+            received_date: item.receivedDate,
+            sender_id: item.senderOrganizationId,
+            receiver_id: item.receiverOrganizationId,
+            details: {
+              tags: item.extractedTags
+            }
+          }
+        }));
+
+      const batchId = `BATCH_UI_${Date.now()}`;
+      await migrationService.commitBatch(
+        { items: batchItems, batchId },
+        batchId
+      );
+      
+      fetchData();
+    } catch (error) {
+      console.error("Batch commit failed", error);
+      alert("Batch commit failed. See console for details.");
+    } finally {
+      setSubmitting(false);
+    }
+  };
+
   return (
     <div className="space-y-6">
       <div className="flex justify-between flex-wrap gap-4 items-center">
@@ -53,6 +114,16 @@ export default function MigrationReviewQueuePage() {
           </p>
         </div>
         <div className="flex items-center gap-4">
+          {selectedIds.length > 0 && (
+            <Button 
+              variant="default" 
+              onClick={handleBatchApprove} 
+              disabled={submitting}
+            >
+              <CheckSquareIcon className="mr-2 h-4 w-4" /> 
+              {submitting ? "Processing..." : `Batch Approve (${selectedIds.length})`}
+            </Button>
+          )}
           <Link href="/admin/migration/errors">
             <Button variant="outline">
               <FileXIcon className="mr-2 h-4 w-4" /> View Error Logs
@@ -86,6 +157,13 @@ export default function MigrationReviewQueuePage() {
               <Table>
                 <TableHeader>
                   <TableRow>
+                    <TableHead className="w-[50px]">
+                      <Checkbox 
+                        checked={items.length > 0 && selectedIds.length === items.length}
+                        onCheckedChange={handleToggleSelectAll}
+                        aria-label="Select all"
+                      />
+                    </TableHead>
                     <TableHead>Document No.</TableHead>
                     <TableHead>Suggested Category</TableHead>
                     <TableHead>Confidence</TableHead>
@@ -97,6 +175,13 @@ export default function MigrationReviewQueuePage() {
                 <TableBody>
                   {items.map((item) => (
                     <TableRow key={item.id}>
+                      <TableCell>
+                        <Checkbox 
+                          checked={selectedIds.includes(item.id)}
+                          onCheckedChange={() => handleToggleSelect(item.id)}
+                          aria-label={`Select item ${item.id}`}
+                        />
+                      </TableCell>
                       <TableCell className="font-medium">{item.documentNumber}</TableCell>
                       <TableCell>{item.aiSuggestedCategory || "Unknown"}</TableCell>
                       <TableCell>

frontend/lib/services/migration.service.ts

@@ -4,6 +4,7 @@ import {
   MigrationErrorItem,
   PaginatedResponse,
   MigrationReviewStatus,
+  CommitBatchDto,
 } from '@/types/migration';
 
 export const migrationService = {
@@ -43,6 +44,15 @@ export const migrationService = {
     return data?.data || data;
   },
 
+  commitBatch: async (payload: CommitBatchDto, idempotencyKey: string) => {
+    const { data } = await api.post(`/migration/commit_batch`, payload, {
+      headers: {
+        'idempotency-key': idempotencyKey,
+      },
+    });
+    return data?.data || data;
+  },
+
   getStagingFileUrl: (filePath: string) => {
     // Generate the URL directly since it returns a file stream.
     // Ensure we encode the file path correctly.

frontend/types/migration.ts

@@ -17,6 +17,25 @@ export interface MigrationReviewQueueItem {
   reviewedBy?: string;
   reviewedAt?: string;
   createdAt: string;
+  projectId?: number;
+  senderOrganizationId?: number;
+  receiverOrganizationId?: number;
+  receivedDate?: string;
+  issuedDate?: string;
+  remarks?: string;
+  aiSummary?: string;
+  extractedTags?: any;
+  tempAttachmentId?: number;
+}
+
+export interface CommitBatchItemDto {
+  queueId: number;
+  dto: Record<string, any>;
+}
+
+export interface CommitBatchDto {
+  items: CommitBatchItemDto[];
+  batchId: string;
 }
 
 export enum MigrationErrorType {