diff --git a/core/views.py b/core/views.py index db75421..ac897e0 100644 --- a/core/views.py +++ b/core/views.py @@ -7,7 +7,9 @@ import datetime from django.shortcuts import render, redirect, get_object_or_404 from django.utils import timezone from django.contrib.auth.decorators import login_required, user_passes_test +from django.db import transaction from django.db.models import Sum, Q, Prefetch +from django.db.models.functions import TruncMonth from django.core.mail import send_mail, EmailMultiAlternatives from django.conf import settings from django.contrib import messages @@ -74,22 +76,37 @@ def home(request): week_projects = 0 if user_is_admin: - # 1. Outstanding Payments - active_workers = Worker.objects.filter(is_active=True).prefetch_related('work_logs', 'adjustments') + # 1. Outstanding Payments (prefetch all related data to avoid per-worker queries) + active_workers = Worker.objects.filter(is_active=True).prefetch_related( + Prefetch( + 'work_logs', + queryset=WorkLog.objects.prefetch_related( + Prefetch('paid_in', queryset=PayrollRecord.objects.only('id', 'worker_id')) + ), + ), + Prefetch( + 'adjustments', + queryset=PayrollAdjustment.objects.filter(payroll_record__isnull=True), + to_attr='pending_adjustments_list' + ), + ) for worker in active_workers: - # Unpaid logs - unpaid_logs_count = worker.work_logs.exclude(paid_in__worker=worker).count() + # Unpaid logs (filter in Python using prefetch cache) + unpaid_logs_count = 0 + for log in worker.work_logs.all(): + paid_worker_ids = {pr.worker_id for pr in log.paid_in.all()} + if worker.id not in paid_worker_ids: + unpaid_logs_count += 1 log_amount = unpaid_logs_count * worker.day_rate - - # Pending Adjustments - pending_adjustments = worker.adjustments.filter(payroll_record__isnull=True) + + # Pending Adjustments (use prefetched list) adj_total = Decimal('0.00') - for adj in pending_adjustments: + for adj in worker.pending_adjustments_list: if adj.type in ADDITIVE_TYPES: adj_total += adj.amount elif adj.type in DEDUCTIVE_TYPES: adj_total -= adj.amount - + total_payable = log_amount + adj_total outstanding_total += max(total_payable, Decimal('0.00')) @@ -109,36 +126,43 @@ def home(request): week_logs = WorkLog.objects.filter(date__range=(week_start, week_end)).prefetch_related('workers') week_projects = week_logs.values('project').distinct().count() for log in week_logs: - week_worker_days += log.workers.count() + week_worker_days += len(log.workers.all()) # Manage Resources data (admin only) all_workers = Worker.objects.all().prefetch_related('teams').order_by('name') if user_is_admin else [] - all_projects = Project.objects.all().order_by('name') if user_is_admin else [] + all_projects = Project.objects.all().order_by('name').prefetch_related( + Prefetch('logs', queryset=WorkLog.objects.prefetch_related('workers', 'paid_in')), + ) if user_is_admin else [] all_teams = Team.objects.all().prefetch_related('workers').order_by('name') if user_is_admin else [] # Outstanding Project Costs (Admin only) - Added for Dashboard visibility outstanding_project_costs = [] if user_is_admin: + # Bulk-fetch all pending project-linked adjustments in one query + pending_proj_adjs = {} + for adj in PayrollAdjustment.objects.filter( + work_log__project__isnull=False, + payroll_record__isnull=True + ).select_related('work_log'): + pid = adj.work_log.project_id + pending_proj_adjs.setdefault(pid, []).append(adj) + for project in all_projects: outstanding_cost = 0 - - # Unpaid WorkLogs - unpaid_logs = project.logs.filter(paid_in__isnull=True).prefetch_related('workers') - for log in unpaid_logs: - for worker in log.workers.all(): - outstanding_cost += worker.day_rate - - # Unpaid Adjustments linked to this project - project_adjustments = PayrollAdjustment.objects.filter( - work_log__project=project, - payroll_record__isnull=True - ) - for adj in project_adjustments: + + # Unpaid WorkLogs (use prefetch cache, check paid_in in Python) + for log in project.logs.all(): + if not list(log.paid_in.all()): + for worker in log.workers.all(): + outstanding_cost += worker.day_rate + + # Unpaid Adjustments linked to this project (from bulk-fetched dict) + for adj in pending_proj_adjs.get(project.id, []): if adj.type in ADDITIVE_TYPES: outstanding_cost += adj.amount elif adj.type in DEDUCTIVE_TYPES: outstanding_cost -= adj.amount - + if outstanding_cost > 0: outstanding_project_costs.append({'name': project.name, 'cost': outstanding_cost}) @@ -332,6 +356,20 @@ def work_log_list(request): payment_status = request.GET.get('payment_status') # 'paid', 'unpaid', 'all' view_mode = request.GET.get('view', 'list') + # Validate numeric GET params to prevent 500 on bad input + try: + worker_id = str(int(worker_id)) if worker_id else None + except (ValueError, TypeError): + worker_id = None + try: + team_id = str(int(team_id)) if team_id else None + except (ValueError, TypeError): + team_id = None + try: + project_id = str(int(project_id)) if project_id else None + except (ValueError, TypeError): + project_id = None + # --- 1. Fetch WorkLogs --- logs = WorkLog.objects.all().select_related('team', 'project', 'supervisor').prefetch_related('workers', 'paid_in').order_by('-date', '-id') @@ -735,54 +773,73 @@ def payroll_dashboard(request): status_filter = request.GET.get('status', 'pending') # pending, paid, all, loans - # Common Analytics + # Common Analytics (prefetch all related data to avoid per-worker queries) outstanding_total = 0 - active_workers = Worker.objects.filter(is_active=True).order_by('name').prefetch_related('adjustments') - + active_workers = Worker.objects.filter(is_active=True).order_by('name').prefetch_related( + Prefetch( + 'work_logs', + queryset=WorkLog.objects.select_related('project').prefetch_related( + Prefetch('paid_in', queryset=PayrollRecord.objects.only('id', 'worker_id')), + 'overtime_paid_to', + ), + ), + Prefetch( + 'adjustments', + queryset=PayrollAdjustment.objects.filter(payroll_record__isnull=True), + to_attr='pending_adjustments_list' + ), + ) + workers_data = [] # For pending payments all_ot_data = [] # For JSON context - + for worker in active_workers: - # Unpaid Work Logs - unpaid_logs = worker.work_logs.exclude(paid_in__worker=worker) - log_count = unpaid_logs.count() + # Unpaid Work Logs (filter in Python using prefetch cache) + unpaid_logs = [] + for log in worker.work_logs.all(): + paid_worker_ids = {pr.worker_id for pr in log.paid_in.all()} + if worker.id not in paid_worker_ids: + unpaid_logs.append(log) + log_count = len(unpaid_logs) log_amount = log_count * worker.day_rate - # Overtime Logic (Updated: Check M2M field) - ot_logs = worker.work_logs.filter(overtime__gt=0).exclude(overtime_paid_to=worker).exclude(paid_in__worker=worker).select_related('project') + # Overtime Logic (filter from unpaid logs using prefetch cache) ot_data_worker = [] ot_hours_unpriced = Decimal('0.0') - for log in ot_logs: - entry = { - 'worker_id': worker.id, - 'worker_name': worker.name, - 'log_id': log.id, - 'date': log.date.strftime('%Y-%m-%d'), - 'project': log.project.name, - 'overtime': float(log.overtime), - 'ot_label': log.get_overtime_display(), - } - ot_data_worker.append(entry) - all_ot_data.append(entry) - ot_hours_unpriced += log.overtime - - # Pending Adjustments (unlinked to any payroll record) - pending_adjustments = worker.adjustments.filter(payroll_record__isnull=True) + for log in unpaid_logs: + if log.overtime > 0: + ot_paid_ids = {w.id for w in log.overtime_paid_to.all()} + if worker.id not in ot_paid_ids: + entry = { + 'worker_id': worker.id, + 'worker_name': worker.name, + 'log_id': log.id, + 'date': log.date.strftime('%Y-%m-%d'), + 'project': log.project.name, + 'overtime': float(log.overtime), + 'ot_label': log.get_overtime_display(), + } + ot_data_worker.append(entry) + all_ot_data.append(entry) + ot_hours_unpriced += log.overtime + + # Pending Adjustments (use prefetched list — no extra queries) + pending_adjustments = worker.pending_adjustments_list adj_total = Decimal('0.00') for adj in pending_adjustments: if adj.type in ADDITIVE_TYPES: adj_total += adj.amount elif adj.type in DEDUCTIVE_TYPES: adj_total -= adj.amount - + total_payable = log_amount + adj_total - + # Only show if there is something to pay or negative (e.g. loan repayment greater than work) # Note: If total_payable is negative, it implies they owe money. - if log_count > 0 or pending_adjustments.exists(): + if log_count > 0 or len(pending_adjustments) > 0: outstanding_total += max(total_payable, Decimal('0.00')) # Only count positive payable for grand total - + if status_filter in ['pending', 'all']: workers_data.append({ 'worker': worker, @@ -795,7 +852,7 @@ def payroll_dashboard(request): 'ot_data': ot_data_worker, 'ot_hours_unpriced': float(ot_hours_unpriced), 'day_rate': float(worker.day_rate), - 'has_pending_advances': pending_adjustments.filter(type='ADVANCE').exists(), + 'has_pending_advances': any(a.type == 'ADVANCE' for a in pending_adjustments), }) # Paid History @@ -803,16 +860,27 @@ def payroll_dashboard(request): if status_filter in ['paid', 'all']: paid_records = PayrollRecord.objects.select_related('worker').order_by('-date', '-id') - # Analytics: Project Costs (Active Projects) + # Analytics: Project Costs (prefetch all logs+workers in bulk) project_costs = [] outstanding_project_costs = [] - active_projects = Project.objects.filter(is_active=True) - + active_projects = Project.objects.filter(is_active=True).prefetch_related( + Prefetch('logs', queryset=WorkLog.objects.prefetch_related('workers', 'paid_in')), + ) + + # Bulk-fetch all pending project-linked adjustments in one query + pending_proj_adjs = {} + for adj in PayrollAdjustment.objects.filter( + work_log__project__isnull=False, + work_log__project__is_active=True, + payroll_record__isnull=True, + ).select_related('work_log'): + pid = adj.work_log.project_id + pending_proj_adjs.setdefault(pid, []).append(adj) + for project in active_projects: # 1. Total Historical Cost cost = 0 - logs = project.logs.all() - for log in logs: + for log in project.logs.all(): for worker in log.workers.all(): cost += worker.day_rate if cost > 0: @@ -820,24 +888,20 @@ def payroll_dashboard(request): # 2. Outstanding Cost (Unpaid) outstanding_cost = 0 - - # Unpaid WorkLogs - unpaid_logs = project.logs.filter(paid_in__isnull=True).prefetch_related('workers') - for log in unpaid_logs: - for worker in log.workers.all(): - outstanding_cost += worker.day_rate - - # Unpaid Adjustments linked to this project - project_adjustments = PayrollAdjustment.objects.filter( - work_log__project=project, - payroll_record__isnull=True - ) - for adj in project_adjustments: + + # Unpaid WorkLogs (check paid_in in Python using prefetch cache) + for log in project.logs.all(): + if not list(log.paid_in.all()): + for worker in log.workers.all(): + outstanding_cost += worker.day_rate + + # Unpaid Adjustments linked to this project (from bulk-fetched dict) + for adj in pending_proj_adjs.get(project.id, []): if adj.type in ADDITIVE_TYPES: outstanding_cost += adj.amount elif adj.type in DEDUCTIVE_TYPES: outstanding_cost -= adj.amount - + if outstanding_cost > 0: outstanding_project_costs.append({'name': project.name, 'cost': outstanding_cost}) @@ -865,7 +929,6 @@ def payroll_dashboard(request): today = timezone.now().date() chart_months = [] # list of (year, month) tuples, oldest first for i in range(5, -1, -1): - # Go back i months from current month m = today.month - i y = today.year while m <= 0: @@ -873,49 +936,46 @@ def payroll_dashboard(request): y -= 1 chart_months.append((y, m)) - chart_labels = [] # e.g. ["Sep 2025", "Oct 2025", ...] - chart_totals = [] # total payroll paid per month + # Build a lookup: (year, month) → index in chart_months + chart_month_index = {ym: idx for idx, ym in enumerate(chart_months)} + chart_start = datetime.date(chart_months[0][0], chart_months[0][1], 1) - # For per-project chart: {project_name: [month0_cost, month1_cost, ...]}} + chart_labels = [f"{calendar.month_abbr[m]} {y}" for y, m in chart_months] + + # 1 query: monthly payroll totals (grouped by month) + paid_by_month = {} + for row in PayrollRecord.objects.filter(date__gte=chart_start).annotate( + month=TruncMonth('date') + ).values('month').annotate(total=Sum('amount')): + paid_by_month[(row['month'].year, row['month'].month)] = float(row['total']) + + chart_totals = [paid_by_month.get(ym, 0) for ym in chart_months] + + # 1 query: monthly overtime totals (grouped by month) + ot_by_month = {} + for row in PayrollAdjustment.objects.filter( + type='OVERTIME', date__gte=chart_start + ).annotate(month=TruncMonth('date')).values('month').annotate(total=Sum('amount')): + ot_by_month[(row['month'].year, row['month'].month)] = float(row['total']) + + ot_history_totals = [ot_by_month.get(ym, 0) for ym in chart_months] + + # 1 query + prefetch: all work logs in the 6-month period for per-project costs all_project_names = list(Project.objects.values_list('name', flat=True).order_by('name')) - project_monthly = {name: [] for name in all_project_names} + project_monthly = {name: [0] * len(chart_months) for name in all_project_names} - ot_history_totals = [] # Overtime history + all_chart_logs = WorkLog.objects.filter( + date__gte=chart_start + ).select_related('project').prefetch_related('workers') - for year, month in chart_months: - chart_labels.append(f"{calendar.month_abbr[month]} {year}") - - # Total paid out this month - _, last_day = calendar.monthrange(year, month) - month_start = datetime.date(year, month, 1) - month_end = datetime.date(year, month, last_day) - - month_paid = PayrollRecord.objects.filter( - date__gte=month_start, date__lte=month_end - ).aggregate(total=Sum('amount'))['total'] or 0 - chart_totals.append(float(month_paid)) - - # Overtime paid this month - ot_month_total = PayrollAdjustment.objects.filter( - type='OVERTIME', - date__gte=month_start, - date__lte=month_end - ).aggregate(total=Sum('amount'))['total'] or 0 - ot_history_totals.append(float(ot_month_total)) - - # Per-project labour cost this month (from work logs × day rates) - month_logs = WorkLog.objects.filter( - date__gte=month_start, date__lte=month_end - ).prefetch_related('workers', 'project') - - project_cost_month = {name: 0 for name in all_project_names} - for log in month_logs: + for log in all_chart_logs: + month_key = (log.date.year, log.date.month) + idx = chart_month_index.get(month_key) + if idx is not None: pname = log.project.name - for worker in log.workers.all(): - project_cost_month[pname] = project_cost_month.get(pname, 0) + float(worker.day_rate) - - for name in all_project_names: - project_monthly[name].append(project_cost_month.get(name, 0)) + if pname in project_monthly: + for worker in log.workers.all(): + project_monthly[pname][idx] += float(worker.day_rate) # Filter out projects with zero cost across all months project_chart_data = [ @@ -976,32 +1036,33 @@ def process_payment(request, worker_id): total_amount = logs_amount + adj_amount if log_count > 0 or pending_adjustments.exists(): - # Create Payroll Record - payroll_record = PayrollRecord.objects.create( - worker=worker, - amount=total_amount, - date=timezone.now().date() - ) - - # Link logs - payroll_record.work_logs.set(unpaid_logs) - - # Link Adjustments and Handle Loans - for adj in pending_adjustments: - adj.payroll_record = payroll_record - adj.save() - - # Update Loan Balance if it's a repayment - if adj.type == 'LOAN_REPAYMENT' and adj.loan: - adj.loan.balance -= adj.amount - if adj.loan.balance <= 0: - adj.loan.balance = 0 - adj.loan.is_active = False - adj.loan.save() + with transaction.atomic(): + # Create Payroll Record + payroll_record = PayrollRecord.objects.create( + worker=worker, + amount=total_amount, + date=timezone.now().date() + ) - payroll_record.save() + # Link logs + payroll_record.work_logs.set(unpaid_logs) - # Email Notification + # Link Adjustments and Handle Loans + for adj in pending_adjustments: + adj.payroll_record = payroll_record + adj.save() + + # Update Loan Balance if it's a repayment + if adj.type == 'LOAN_REPAYMENT' and adj.loan: + adj.loan.balance -= adj.amount + if adj.loan.balance <= 0: + adj.loan.balance = 0 + adj.loan.is_active = False + adj.loan.save() + + payroll_record.save() + + # Email Notification (outside transaction — failure should not roll back payment) subject = f"Payslip for {worker.name} - {payroll_record.date}" # Prepare Context @@ -1182,11 +1243,17 @@ def add_loan(request): if request.method == 'POST': worker_id = request.POST.get('worker') - amount = request.POST.get('amount') + amount_str = request.POST.get('amount') reason = request.POST.get('reason') date = request.POST.get('date') or timezone.now().date() - - if worker_id and amount: + + try: + amount = Decimal(amount_str) if amount_str else None + except Exception: + messages.error(request, "Invalid amount.") + return redirect('/payroll/?status=loans') + + if worker_id and amount and amount > 0: worker = get_object_or_404(Worker, pk=worker_id) Loan.objects.create( worker=worker, @@ -1195,7 +1262,9 @@ def add_loan(request): reason=reason ) messages.success(request, f"Loan of R{amount} recorded for {worker.name}.") - + elif amount is not None and amount <= 0: + messages.error(request, "Amount must be greater than zero.") + return redirect('/payroll/?status=loans') @@ -1210,11 +1279,21 @@ def add_adjustment(request): if request.method == 'POST': worker_ids = request.POST.getlist('workers') adj_type = request.POST.get('type') - amount = request.POST.get('amount') + amount_str = request.POST.get('amount') description = request.POST.get('description') date = request.POST.get('date') or timezone.now().date() loan_id = request.POST.get('loan_id') # Optional, for repayments + try: + amount = Decimal(amount_str) if amount_str else None + except Exception: + messages.error(request, "Invalid amount.") + return redirect('payroll_dashboard') + + if amount is not None and amount <= 0: + messages.error(request, "Amount must be greater than zero.") + return redirect('payroll_dashboard') + if worker_ids and amount and adj_type: success_names = [] skip_names = [] @@ -1224,7 +1303,7 @@ def add_adjustment(request): # --- ADVANCE: validate, create adjustment + standalone PayrollRecord, send payslip --- if adj_type == 'ADVANCE': - advance_amount = Decimal(amount) + advance_amount = amount # Must have unpaid work logs unpaid_logs = worker.work_logs.exclude(paid_in__worker=worker) @@ -1248,24 +1327,24 @@ def add_adjustment(request): skip_names.append(f"{worker.name} (R{advance_amount} exceeds available R{max_available:.2f})") continue - # Create ADVANCE adjustment (stays pending — payroll_record=NULL) - PayrollAdjustment.objects.create( - worker=worker, - type='ADVANCE', - amount=advance_amount, - description=description or 'Advance payment', - date=date, - ) + # Create ADVANCE adjustment + PayrollRecord atomically + with transaction.atomic(): + PayrollAdjustment.objects.create( + worker=worker, + type='ADVANCE', + amount=advance_amount, + description=description or 'Advance payment', + date=date, + ) - # Create standalone PayrollRecord (NO work_logs linked) - advance_date = date if isinstance(date, datetime.date) else timezone.now().date() - advance_record = PayrollRecord.objects.create( - worker=worker, - amount=advance_amount, - date=advance_date, - ) + advance_date = date if isinstance(date, datetime.date) else timezone.now().date() + advance_record = PayrollRecord.objects.create( + worker=worker, + amount=advance_amount, + date=advance_date, + ) - # Send advance payslip to Spark + # Send advance payslip to Spark (outside transaction) subject = f"Advance Payment for {worker.name} - {advance_record.date}" email_context = { 'record': advance_record,