feat: data import module and payment fixes

- Add import module with CSV/Excel template download and bulk import endpoints
- Support client+subscription and outstanding invoice imports with per-row error handling
- Templates include Field Guide sheet with valid plan/area names
- Add manager role to unremitted payments endpoint
- Add createdAt field to remittance history response

package.json

@@ -1,7 +1,9 @@
 {
   "name": "fiberops-api",
   "private": true,
-  "workspaces": ["packages/*"],
+  "workspaces": [
+    "packages/*"
+  ],
   "scripts": {
     "dev": "nest start --watch",
     "build": "nest build",
@@ -32,6 +34,7 @@
     "passport-jwt": "^4.0.1",
     "reflect-metadata": "^0.2.0",
     "rxjs": "^7.8.1",
+    "xlsx": "^0.18.5",
     "zod": "^3.24.0"
   },
   "devDependencies": {

src/app.module.ts

@@ -29,6 +29,7 @@ import { AccountingModule } from './accounting/accounting.module';
 import { PayrollModule } from './payroll/payroll.module';
 import { RoleModule } from './role/role.module';
 import { CommentModule } from './comment/comment.module';
+import { ImportModule } from './import/import.module';
 import { GlobalExceptionFilter } from './common/filters/http-exception.filter';
 import { ResponseInterceptor } from './common/interceptors/response.interceptor';
 import { PermissionsGuard } from './common/guards/permissions.guard';
@@ -71,6 +72,7 @@ import { AccessGuard } from './common/guards/access.guard';
     PayrollModule,
     RoleModule,
     CommentModule,
+    ImportModule,
   ],
   providers: [
     { provide: APP_FILTER, useClass: GlobalExceptionFilter },

src/import/import.controller.ts

@@ -0,0 +1,91 @@
+import {
+  Controller,
+  Post,
+  Get,
+  Param,
+  UseGuards,
+  UseInterceptors,
+  UploadedFile,
+  Res,
+} from '@nestjs/common';
+import { AuthGuard } from '@nestjs/passport';
+import { FileInterceptor } from '@nestjs/platform-express';
+import { Response } from 'express';
+import { ImportService } from './import.service';
+import { Roles } from '../common/decorators/roles.decorator';
+import { RolesGuard } from '../common/guards/roles.guard';
+import { TenantGuard } from '../common/guards/tenant.guard';
+import { CurrentUser, CurrentUserPayload } from '../common/decorators/current-user.decorator';
+
+@Controller('import')
+@UseGuards(AuthGuard('jwt'), TenantGuard, RolesGuard)
+export class ImportController {
+  constructor(private readonly importService: ImportService) {}
+
+  @Get('template/:type')
+  @Roles('tenant_admin')
+  async downloadTemplate(
+    @Param('type') type: string,
+    @CurrentUser() user: CurrentUserPayload,
+    @Res() res: Response,
+  ) {
+    const { buffer, filename, contentType } =
+      await this.importService.generateTemplate(user.tenantId, type);
+    res.setHeader('Content-Type', contentType);
+    res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
+    res.send(buffer);
+  }
+
+  @Post('clients')
+  @Roles('tenant_admin')
+  @UseInterceptors(
+    FileInterceptor('file', {
+      limits: { fileSize: 10 * 1024 * 1024 },
+      fileFilter: (_req, file, cb) => {
+        const ext = file.originalname.split('.').pop()?.toLowerCase();
+        if (['csv', 'xlsx', 'xls'].includes(ext || '')) {
+          cb(null, true);
+        } else {
+          cb(new Error('Only CSV and Excel files are allowed'), false);
+        }
+      },
+    }),
+  )
+  async importClients(
+    @CurrentUser() user: CurrentUserPayload,
+    @UploadedFile() file: Express.Multer.File,
+  ) {
+    return this.importService.importClients(
+      user.tenantId,
+      user.sub,
+      file.buffer,
+      file.originalname,
+    );
+  }
+
+  @Post('invoices')
+  @Roles('tenant_admin')
+  @UseInterceptors(
+    FileInterceptor('file', {
+      limits: { fileSize: 10 * 1024 * 1024 },
+      fileFilter: (_req, file, cb) => {
+        const ext = file.originalname.split('.').pop()?.toLowerCase();
+        if (['csv', 'xlsx', 'xls'].includes(ext || '')) {
+          cb(null, true);
+        } else {
+          cb(new Error('Only CSV and Excel files are allowed'), false);
+        }
+      },
+    }),
+  )
+  async importInvoices(
+    @CurrentUser() user: CurrentUserPayload,
+    @UploadedFile() file: Express.Multer.File,
+  ) {
+    return this.importService.importInvoices(
+      user.tenantId,
+      file.buffer,
+      file.originalname,
+    );
+  }
+}

src/import/import.module.ts

@@ -0,0 +1,11 @@
+import { Module } from '@nestjs/common';
+import { ImportController } from './import.controller';
+import { ImportService } from './import.service';
+import { PrismaModule } from '../prisma/prisma.module';
+
+@Module({
+  imports: [PrismaModule],
+  controllers: [ImportController],
+  providers: [ImportService],
+})
+export class ImportModule {}

src/import/import.service.ts

@@ -0,0 +1,427 @@
+import {
+  Injectable,
+  BadRequestException,
+} from '@nestjs/common';
+import { PrismaService } from '../prisma/prisma.service';
+import * as XLSX from 'xlsx';
+
+interface RowResult {
+  row: number;
+  status: 'ok' | 'error';
+  message?: string;
+}
+
+interface ImportResult {
+  total: number;
+  imported: number;
+  errors: number;
+  details: RowResult[];
+}
+
+@Injectable()
+export class ImportService {
+  constructor(private readonly prisma: PrismaService) {}
+
+  /* ------------------------------------------------------------------ */
+  /*  Template generation                                                */
+  /* ------------------------------------------------------------------ */
+
+  async generateTemplate(
+    tenantId: string,
+    type: string,
+  ): Promise<{ buffer: Buffer; filename: string; contentType: string }> {
+    if (type === 'clients') {
+      return this.generateClientsTemplate(tenantId);
+    }
+    if (type === 'invoices') {
+      return this.generateInvoicesTemplate(tenantId);
+    }
+    throw new BadRequestException(
+      `Unknown template type "${type}". Use "clients" or "invoices".`,
+    );
+  }
+
+  private async generateClientsTemplate(tenantId: string) {
+    const [plans, areas] = await Promise.all([
+      this.prisma.plan.findMany({
+        where: { tenantId, isActive: true, deletedAt: null },
+        select: { name: true },
+        orderBy: { name: 'asc' },
+      }),
+      this.prisma.area.findMany({
+        where: { tenantId, deletedAt: null },
+        select: { name: true },
+        orderBy: { name: 'asc' },
+      }),
+    ]);
+
+    const headers = [
+      'firstName*',
+      'lastName*',
+      'email',
+      'phone',
+      'address*',
+      'accountNumber',
+      'planName*',
+      'subscriptionType*',
+      'areaName',
+    ];
+
+    const exampleRow = [
+      'Juan',
+      'Dela Cruz',
+      'juan@example.com',
+      '+639171234567',
+      '123 Main Street, Barangay 1',
+      'C-000001',
+      plans[0]?.name || 'Plan 1',
+      'postpaid',
+      areas[0]?.name || '',
+    ];
+
+    const ws = XLSX.utils.aoa_to_sheet([headers, exampleRow]);
+
+    // Add data-validation notes in a second sheet
+    const notesData = [
+      ['Field', 'Required', 'Description', 'Valid Values'],
+      ['firstName', 'Yes', 'Client first name', ''],
+      ['lastName', 'Yes', 'Client last name', ''],
+      ['email', 'No', 'Email address', ''],
+      ['phone', 'No', 'Phone number', ''],
+      ['address', 'Yes', 'Full address', ''],
+      ['accountNumber', 'No', 'Existing account number (auto-generated if empty)', ''],
+      ['planName', 'Yes', 'Must match an existing plan name', plans.map((p) => p.name).join(', ') || '(no plans created yet)'],
+      ['subscriptionType', 'Yes', 'Subscription type', 'prepaid, postpaid'],
+      ['areaName', 'No', 'Must match an existing area name', areas.map((a) => a.name).join(', ') || '(no areas created yet)'],
+    ];
+    const notesWs = XLSX.utils.aoa_to_sheet(notesData);
+
+    // Set column widths
+    ws['!cols'] = headers.map(() => ({ wch: 25 }));
+    notesWs['!cols'] = [
+      { wch: 20 },
+      { wch: 10 },
+      { wch: 45 },
+      { wch: 50 },
+    ];
+
+    const wb = XLSX.utils.book_new();
+    XLSX.utils.book_append_sheet(wb, ws, 'Clients');
+    XLSX.utils.book_append_sheet(wb, notesWs, 'Field Guide');
+
+    const buffer = XLSX.write(wb, { type: 'buffer', bookType: 'xlsx' });
+    return {
+      buffer,
+      filename: 'fiberops_clients_template.xlsx',
+      contentType: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+    };
+  }
+
+  private async generateInvoicesTemplate(tenantId: string) {
+    const headers = [
+      'accountNumber*',
+      'invoiceNumber',
+      'amount*',
+      'balance*',
+      'dueDate*',
+      'status*',
+    ];
+
+    const exampleRow = [
+      'C-000001',
+      '',
+      '1500.00',
+      '1500.00',
+      '2026-06-01',
+      'unpaid',
+    ];
+
+    const ws = XLSX.utils.aoa_to_sheet([headers, exampleRow]);
+
+    const notesData = [
+      ['Field', 'Required', 'Description', 'Valid Values'],
+      ['accountNumber', 'Yes', 'Must match an existing client account number', ''],
+      ['invoiceNumber', 'No', 'Invoice number (auto-generated if empty)', ''],
+      ['amount', 'Yes', 'Invoice total amount', ''],
+      ['balance', 'Yes', 'Outstanding balance', ''],
+      ['dueDate', 'Yes', 'Due date in YYYY-MM-DD format', ''],
+      ['status', 'Yes', 'Invoice status', 'unpaid, partial'],
+    ];
+    const notesWs = XLSX.utils.aoa_to_sheet(notesData);
+
+    ws['!cols'] = headers.map(() => ({ wch: 25 }));
+    notesWs['!cols'] = [
+      { wch: 20 },
+      { wch: 10 },
+      { wch: 50 },
+      { wch: 30 },
+    ];
+
+    const wb = XLSX.utils.book_new();
+    XLSX.utils.book_append_sheet(wb, ws, 'Invoices');
+    XLSX.utils.book_append_sheet(wb, notesWs, 'Field Guide');
+
+    const buffer = XLSX.write(wb, { type: 'buffer', bookType: 'xlsx' });
+    return {
+      buffer,
+      filename: 'fiberops_invoices_template.xlsx',
+      contentType: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+    };
+  }
+
+  /* ------------------------------------------------------------------ */
+  /*  Client + Subscription import                                       */
+  /* ------------------------------------------------------------------ */
+
+  async importClients(
+    tenantId: string,
+    userId: string,
+    fileBuffer: Buffer,
+    filename: string,
+  ): Promise<ImportResult> {
+    const rows = this.parseFile(fileBuffer, filename);
+    if (rows.length === 0) {
+      throw new BadRequestException('File is empty');
+    }
+
+    // Prefetch plans and areas for lookup
+    const [plans, areas] = await Promise.all([
+      this.prisma.plan.findMany({
+        where: { tenantId, isActive: true, deletedAt: null },
+        select: { id: true, name: true },
+      }),
+      this.prisma.area.findMany({
+        where: { tenantId, deletedAt: null },
+        select: { id: true, name: true },
+      }),
+    ]);
+
+    const planMap = new Map(plans.map((p) => [p.name.toLowerCase(), p.id]));
+    const areaMap = new Map(areas.map((a) => [a.name.toLowerCase(), a.id]));
+
+    const details: RowResult[] = [];
+
+    for (let i = 0; i < rows.length; i++) {
+      const row = rows[i];
+      const rowNumber = i + 2; // Excel is 1-indexed, +1 for header
+
+      try {
+        const firstName = this.req(row, 'firstName', rowNumber);
+        const lastName = this.req(row, 'lastName', rowNumber);
+        const address = this.req(row, 'address', rowNumber);
+        const planName = this.req(row, 'planName', rowNumber);
+        const subscriptionType = this.req(row, 'subscriptionType', rowNumber);
+
+        if (!['prepaid', 'postpaid'].includes(subscriptionType.toLowerCase())) {
+          throw new Error(`Invalid subscriptionType "${subscriptionType}". Must be "prepaid" or "postpaid".`);
+        }
+
+        const planId = planMap.get(planName.toLowerCase());
+        if (!planId) {
+          throw new Error(`Plan "${planName}" not found. Available plans: ${plans.map((p) => p.name).join(', ') || '(none)'}`);
+        }
+
+        const areaName = this.opt(row, 'areaName');
+        const areaId = areaName ? areaMap.get(areaName.toLowerCase()) : undefined;
+        if (areaName && !areaId) {
+          throw new Error(`Area "${areaName}" not found. Available areas: ${areas.map((a) => a.name).join(', ') || '(none)'}`);
+        }
+
+        const accountNumber = this.opt(row, 'accountNumber') || undefined;
+
+        // Check account number uniqueness if provided
+        if (accountNumber) {
+          const existing = await this.prisma.client.findFirst({
+            where: { tenantId, accountNumber },
+          });
+          if (existing) {
+            throw new Error(`Account number "${accountNumber}" already exists for client ${existing.firstName} ${existing.lastName}`);
+          }
+        }
+
+        // Generate account number if not provided
+        const finalAccountNumber =
+          accountNumber ||
+          (await this.generateAccountNumber(tenantId));
+
+        const client = await this.prisma.client.create({
+          data: {
+            tenantId,
+            accountNumber: finalAccountNumber,
+            firstName,
+            lastName,
+            email: this.opt(row, 'email'),
+            phone: this.opt(row, 'phone'),
+            address,
+            areaId,
+          },
+        });
+
+        // Create subscription
+        await this.prisma.subscription.create({
+          data: {
+            tenantId,
+            clientId: client.id,
+            planId,
+            type: subscriptionType.toLowerCase(),
+            status: 'pending',
+          },
+        });
+
+        details.push({ row: rowNumber, status: 'ok', message: `Created ${firstName} ${lastName} (${finalAccountNumber})` });
+      } catch (err: any) {
+        details.push({ row: rowNumber, status: 'error', message: err.message });
+      }
+    }
+
+    const imported = details.filter((d) => d.status === 'ok').length;
+    const errors = details.filter((d) => d.status === 'error').length;
+
+    // Log audit
+    if (imported > 0) {
+      await this.prisma.auditLog.create({
+        data: {
+          tenantId,
+          userId,
+          action: 'IMPORT_CLIENTS',
+          entity: 'client',
+          entityId: 'bulk',
+          details: { imported, errors, total: rows.length },
+        },
+      });
+    }
+
+    return { total: rows.length, imported, errors, details };
+  }
+
+  /* ------------------------------------------------------------------ */
+  /*  Invoice import                                                     */
+  /* ------------------------------------------------------------------ */
+
+  async importInvoices(
+    tenantId: string,
+    fileBuffer: Buffer,
+    filename: string,
+  ): Promise<ImportResult> {
+    const rows = this.parseFile(fileBuffer, filename);
+    if (rows.length === 0) {
+      throw new BadRequestException('File is empty');
+    }
+
+    const details: RowResult[] = [];
+
+    for (let i = 0; i < rows.length; i++) {
+      const row = rows[i];
+      const rowNumber = i + 2;
+
+      try {
+        const accountNumber = this.req(row, 'accountNumber', rowNumber);
+        const amountStr = this.req(row, 'amount', rowNumber);
+        const balanceStr = this.req(row, 'balance', rowNumber);
+        const dueDateStr = this.req(row, 'dueDate', rowNumber);
+        const status = this.req(row, 'status', rowNumber);
+
+        // Find client by account number
+        const client = await this.prisma.client.findFirst({
+          where: { tenantId, accountNumber },
+        });
+        if (!client) {
+          throw new Error(`Client with account number "${accountNumber}" not found`);
+        }
+
+        const amount = parseFloat(amountStr);
+        const balance = parseFloat(balanceStr);
+        if (isNaN(amount) || amount <= 0) {
+          throw new Error(`Invalid amount "${amountStr}"`);
+        }
+        if (isNaN(balance) || balance < 0) {
+          throw new Error(`Invalid balance "${balanceStr}"`);
+        }
+
+        const dueDate = new Date(dueDateStr);
+        if (isNaN(dueDate.getTime())) {
+          throw new Error(`Invalid dueDate "${dueDateStr}". Use YYYY-MM-DD format.`);
+        }
+
+        if (!['unpaid', 'partial'].includes(status.toLowerCase())) {
+          throw new Error(`Invalid status "${status}". Must be "unpaid" or "partial".`);
+        }
+
+        // Generate invoice number
+        const invoiceNumber =
+          this.opt(row, 'invoiceNumber') ||
+          (await this.generateInvoiceNumber(tenantId));
+
+        await this.prisma.invoice.create({
+          data: {
+            tenantId,
+            clientId: client.id,
+            number: invoiceNumber,
+            amount,
+            balance,
+            status: status.toLowerCase(),
+            dueDate,
+          },
+        });
+
+        details.push({
+          row: rowNumber,
+          status: 'ok',
+          message: `Invoice ${invoiceNumber} for ${client.firstName} ${client.lastName} (${accountNumber})`,
+        });
+      } catch (err: any) {
+        details.push({ row: rowNumber, status: 'error', message: err.message });
+      }
+    }
+
+    const imported = details.filter((d) => d.status === 'ok').length;
+    const errors = details.filter((d) => d.status === 'error').length;
+
+    return { total: rows.length, imported, errors, details };
+  }
+
+  /* ------------------------------------------------------------------ */
+  /*  Helpers                                                            */
+  /* ------------------------------------------------------------------ */
+
+  private parseFile(buffer: Buffer, filename: string): Record<string, string>[] {
+    const ext = filename.split('.').pop()?.toLowerCase();
+    const wb = XLSX.read(buffer, { type: 'buffer' });
+    const ws = wb.Sheets[wb.SheetNames[0]];
+    const rows: Record<string, string>[] = XLSX.utils.sheet_to_json(ws, {
+      defval: '',
+    });
+
+    // Normalize headers: strip asterisks and whitespace
+    return rows.map((row) => {
+      const normalized: Record<string, string> = {};
+      for (const [key, value] of Object.entries(row)) {
+        normalized[key.replace(/\*/g, '').trim()] = String(value).trim();
+      }
+      return normalized;
+    });
+  }
+
+  private req(row: Record<string, string>, field: string, rowNumber: number): string {
+    const value = row[field];
+    if (!value) {
+      throw new Error(`Missing required field "${field}"`);
+    }
+    return value;
+  }
+
+  private opt(row: Record<string, string>, field: string): string | undefined {
+    const value = row[field];
+    return value || undefined;
+  }
+
+  private async generateAccountNumber(tenantId: string): Promise<string> {
+    const count = await this.prisma.client.count({ where: { tenantId } });
+    return `C-${String(count + 1).padStart(6, '0')}`;
+  }
+
+  private async generateInvoiceNumber(tenantId: string): Promise<string> {
+    const count = await this.prisma.invoice.count({ where: { tenantId } });
+    return `INV-${String(count + 1).padStart(6, '0')}`;
+  }
+}

src/payment/payment.controller.ts

@@ -41,9 +41,10 @@ export class PaymentController {
   }
 
   @Get('unremitted')
-  @Roles('technician', 'collector')
+  @Roles('technician', 'collector', 'manager')
   async getUnremitted(@CurrentUser() user: CurrentUserPayload) {
-    return this.paymentService.getUnremittedPayments(user.tenantId, user.sub);
+    const isManager = user.roles?.some((r: string) => ['manager', 'tenant_admin', 'super_admin'].includes(r));
+    return this.paymentService.getUnremittedPayments(user.tenantId, isManager ? null : user.sub);
   }
 
   @Get('remittances')

src/payment/payment.service.ts

@@ -151,6 +151,7 @@ export class PaymentService {
     // Attach full payment details to each remittance's join records
     return remittances.map((r) => ({
       ...r,
+      createdAt: r.submittedAt.toISOString(),
       payments: r.payments.map((rp) => ({
         ...rp,
         payment: paymentMap.get(rp.paymentId) ?? null,