admin/38156-vm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optimize dashboard queries — reduce from 200+ to ~20 queries

Browse Source 
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Konrad du Plessis 2026-02-20 00:35:26 +02:00
parent 9a28c99860
commit 6a55fe8098
1 changed files with 238 additions and 159 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

397
core/views.py
View File

@ -7,7 +7,9 @@ import datetime
from django.shortcuts import render, redirect, get_object_or_404
from django.utils import timezone
from django.contrib.auth.decorators import login_required, user_passes_test
from django.db import transaction
from django.db.models import Sum, Q, Prefetch
from django.db.models.functions import TruncMonth
from django.core.mail import send_mail, EmailMultiAlternatives
from django.conf import settings
from django.contrib import messages
@ -74,22 +76,37 @@ def home(request):
week_projects = 0
if user_is_admin:
# 1. Outstanding Payments
active_workers = Worker.objects.filter(is_active=True).prefetch_related('work_logs', 'adjustments')
# 1. Outstanding Payments (prefetch all related data to avoid per-worker queries)
active_workers = Worker.objects.filter(is_active=True).prefetch_related(
Prefetch(
'work_logs',
queryset=WorkLog.objects.prefetch_related(
Prefetch('paid_in', queryset=PayrollRecord.objects.only('id', 'worker_id'))
),
),
Prefetch(
'adjustments',
queryset=PayrollAdjustment.objects.filter(payroll_record__isnull=True),
to_attr='pending_adjustments_list'
),
)
for worker in active_workers:
# Unpaid logs
unpaid_logs_count = worker.work_logs.exclude(paid_in__worker=worker).count()
# Unpaid logs (filter in Python using prefetch cache)
unpaid_logs_count = 0
for log in worker.work_logs.all():
paid_worker_ids = {pr.worker_id for pr in log.paid_in.all()}
if worker.id not in paid_worker_ids:
unpaid_logs_count += 1
log_amount = unpaid_logs_count * worker.day_rate
# Pending Adjustments
pending_adjustments = worker.adjustments.filter(payroll_record__isnull=True)
# Pending Adjustments (use prefetched list)
adj_total = Decimal('0.00')
for adj in pending_adjustments:
for adj in worker.pending_adjustments_list:
if adj.type in ADDITIVE_TYPES:
adj_total += adj.amount
elif adj.type in DEDUCTIVE_TYPES:
adj_total -= adj.amount
total_payable = log_amount + adj_total
outstanding_total += max(total_payable, Decimal('0.00'))
@ -109,36 +126,43 @@ def home(request):
week_logs = WorkLog.objects.filter(date__range=(week_start, week_end)).prefetch_related('workers')
week_projects = week_logs.values('project').distinct().count()
for log in week_logs:
week_worker_days += log.workers.count()
week_worker_days += len(log.workers.all())
# Manage Resources data (admin only)
all_workers = Worker.objects.all().prefetch_related('teams').order_by('name') if user_is_admin else []
all_projects = Project.objects.all().order_by('name') if user_is_admin else []
all_projects = Project.objects.all().order_by('name').prefetch_related(
Prefetch('logs', queryset=WorkLog.objects.prefetch_related('workers', 'paid_in')),
) if user_is_admin else []
all_teams = Team.objects.all().prefetch_related('workers').order_by('name') if user_is_admin else []
# Outstanding Project Costs (Admin only) - Added for Dashboard visibility
outstanding_project_costs = []
if user_is_admin:
# Bulk-fetch all pending project-linked adjustments in one query
pending_proj_adjs = {}
for adj in PayrollAdjustment.objects.filter(
work_log__project__isnull=False,
payroll_record__isnull=True
).select_related('work_log'):
pid = adj.work_log.project_id
pending_proj_adjs.setdefault(pid, []).append(adj)
for project in all_projects:
outstanding_cost = 0
# Unpaid WorkLogs
unpaid_logs = project.logs.filter(paid_in__isnull=True).prefetch_related('workers')
for log in unpaid_logs:
for worker in log.workers.all():
outstanding_cost += worker.day_rate
# Unpaid Adjustments linked to this project
project_adjustments = PayrollAdjustment.objects.filter(
work_log__project=project,
payroll_record__isnull=True
)
for adj in project_adjustments:
# Unpaid WorkLogs (use prefetch cache, check paid_in in Python)
for log in project.logs.all():
if not list(log.paid_in.all()):
for worker in log.workers.all():
outstanding_cost += worker.day_rate
# Unpaid Adjustments linked to this project (from bulk-fetched dict)
for adj in pending_proj_adjs.get(project.id, []):
if adj.type in ADDITIVE_TYPES:
outstanding_cost += adj.amount
elif adj.type in DEDUCTIVE_TYPES:
outstanding_cost -= adj.amount
if outstanding_cost > 0:
outstanding_project_costs.append({'name': project.name, 'cost': outstanding_cost})
@ -332,6 +356,20 @@ def work_log_list(request):
payment_status = request.GET.get('payment_status') # 'paid', 'unpaid', 'all'
view_mode = request.GET.get('view', 'list')
# Validate numeric GET params to prevent 500 on bad input
try:
worker_id = str(int(worker_id)) if worker_id else None
except (ValueError, TypeError):
worker_id = None
try:
team_id = str(int(team_id)) if team_id else None
except (ValueError, TypeError):
team_id = None
try:
project_id = str(int(project_id)) if project_id else None
except (ValueError, TypeError):
project_id = None
# --- 1. Fetch WorkLogs ---
logs = WorkLog.objects.all().select_related('team', 'project', 'supervisor').prefetch_related('workers', 'paid_in').order_by('-date', '-id')
@ -735,54 +773,73 @@ def payroll_dashboard(request):
status_filter = request.GET.get('status', 'pending') # pending, paid, all, loans
# Common Analytics
# Common Analytics (prefetch all related data to avoid per-worker queries)
outstanding_total = 0
active_workers = Worker.objects.filter(is_active=True).order_by('name').prefetch_related('adjustments')
active_workers = Worker.objects.filter(is_active=True).order_by('name').prefetch_related(
Prefetch(
'work_logs',
queryset=WorkLog.objects.select_related('project').prefetch_related(
Prefetch('paid_in', queryset=PayrollRecord.objects.only('id', 'worker_id')),
'overtime_paid_to',
),
),
Prefetch(
'adjustments',
queryset=PayrollAdjustment.objects.filter(payroll_record__isnull=True),
to_attr='pending_adjustments_list'
),
)
workers_data = [] # For pending payments
all_ot_data = [] # For JSON context
for worker in active_workers:
# Unpaid Work Logs
unpaid_logs = worker.work_logs.exclude(paid_in__worker=worker)
log_count = unpaid_logs.count()
# Unpaid Work Logs (filter in Python using prefetch cache)
unpaid_logs = []
for log in worker.work_logs.all():
paid_worker_ids = {pr.worker_id for pr in log.paid_in.all()}
if worker.id not in paid_worker_ids:
unpaid_logs.append(log)
log_count = len(unpaid_logs)
log_amount = log_count * worker.day_rate
# Overtime Logic (Updated: Check M2M field)
ot_logs = worker.work_logs.filter(overtime__gt=0).exclude(overtime_paid_to=worker).exclude(paid_in__worker=worker).select_related('project')
# Overtime Logic (filter from unpaid logs using prefetch cache)
ot_data_worker = []
ot_hours_unpriced = Decimal('0.0')
for log in ot_logs:
entry = {
'worker_id': worker.id,
'worker_name': worker.name,
'log_id': log.id,
'date': log.date.strftime('%Y-%m-%d'),
'project': log.project.name,
'overtime': float(log.overtime),
'ot_label': log.get_overtime_display(),
}
ot_data_worker.append(entry)
all_ot_data.append(entry)
ot_hours_unpriced += log.overtime
# Pending Adjustments (unlinked to any payroll record)
pending_adjustments = worker.adjustments.filter(payroll_record__isnull=True)
for log in unpaid_logs:
if log.overtime > 0:
ot_paid_ids = {w.id for w in log.overtime_paid_to.all()}
if worker.id not in ot_paid_ids:
entry = {
'worker_id': worker.id,
'worker_name': worker.name,
'log_id': log.id,
'date': log.date.strftime('%Y-%m-%d'),
'project': log.project.name,
'overtime': float(log.overtime),
'ot_label': log.get_overtime_display(),
}
ot_data_worker.append(entry)
all_ot_data.append(entry)
ot_hours_unpriced += log.overtime
# Pending Adjustments (use prefetched list — no extra queries)
pending_adjustments = worker.pending_adjustments_list
adj_total = Decimal('0.00')
for adj in pending_adjustments:
if adj.type in ADDITIVE_TYPES:
adj_total += adj.amount
elif adj.type in DEDUCTIVE_TYPES:
adj_total -= adj.amount
total_payable = log_amount + adj_total
# Only show if there is something to pay or negative (e.g. loan repayment greater than work)
# Note: If total_payable is negative, it implies they owe money.
if log_count > 0 or pending_adjustments.exists():
if log_count > 0 or len(pending_adjustments) > 0:
outstanding_total += max(total_payable, Decimal('0.00')) # Only count positive payable for grand total
if status_filter in ['pending', 'all']:
workers_data.append({
'worker': worker,
@ -795,7 +852,7 @@ def payroll_dashboard(request):
'ot_data': ot_data_worker,
'ot_hours_unpriced': float(ot_hours_unpriced),
'day_rate': float(worker.day_rate),
'has_pending_advances': pending_adjustments.filter(type='ADVANCE').exists(),
'has_pending_advances': any(a.type == 'ADVANCE' for a in pending_adjustments),
})
# Paid History
@ -803,16 +860,27 @@ def payroll_dashboard(request):
if status_filter in ['paid', 'all']:
paid_records = PayrollRecord.objects.select_related('worker').order_by('-date', '-id')
# Analytics: Project Costs (Active Projects)
# Analytics: Project Costs (prefetch all logs+workers in bulk)
project_costs = []
outstanding_project_costs = []
active_projects = Project.objects.filter(is_active=True)
active_projects = Project.objects.filter(is_active=True).prefetch_related(
Prefetch('logs', queryset=WorkLog.objects.prefetch_related('workers', 'paid_in')),
)
# Bulk-fetch all pending project-linked adjustments in one query
pending_proj_adjs = {}
for adj in PayrollAdjustment.objects.filter(
work_log__project__isnull=False,
work_log__project__is_active=True,
payroll_record__isnull=True,
).select_related('work_log'):
pid = adj.work_log.project_id
pending_proj_adjs.setdefault(pid, []).append(adj)
for project in active_projects:
# 1. Total Historical Cost
cost = 0
logs = project.logs.all()
for log in logs:
for log in project.logs.all():
for worker in log.workers.all():
cost += worker.day_rate
if cost > 0:
@ -820,24 +888,20 @@ def payroll_dashboard(request):
# 2. Outstanding Cost (Unpaid)
outstanding_cost = 0
# Unpaid WorkLogs
unpaid_logs = project.logs.filter(paid_in__isnull=True).prefetch_related('workers')
for log in unpaid_logs:
for worker in log.workers.all():
outstanding_cost += worker.day_rate
# Unpaid Adjustments linked to this project
project_adjustments = PayrollAdjustment.objects.filter(
work_log__project=project,
payroll_record__isnull=True
)
for adj in project_adjustments:
# Unpaid WorkLogs (check paid_in in Python using prefetch cache)
for log in project.logs.all():
if not list(log.paid_in.all()):
for worker in log.workers.all():
outstanding_cost += worker.day_rate
# Unpaid Adjustments linked to this project (from bulk-fetched dict)
for adj in pending_proj_adjs.get(project.id, []):
if adj.type in ADDITIVE_TYPES:
outstanding_cost += adj.amount
elif adj.type in DEDUCTIVE_TYPES:
outstanding_cost -= adj.amount
if outstanding_cost > 0:
outstanding_project_costs.append({'name': project.name, 'cost': outstanding_cost})
@ -865,7 +929,6 @@ def payroll_dashboard(request):
today = timezone.now().date()
chart_months = [] # list of (year, month) tuples, oldest first
for i in range(5, -1, -1):
# Go back i months from current month
m = today.month - i
y = today.year
while m <= 0:
@ -873,49 +936,46 @@ def payroll_dashboard(request):
y -= 1
chart_months.append((y, m))
chart_labels = [] # e.g. ["Sep 2025", "Oct 2025", ...]
chart_totals = [] # total payroll paid per month
# Build a lookup: (year, month) → index in chart_months
chart_month_index = {ym: idx for idx, ym in enumerate(chart_months)}
chart_start = datetime.date(chart_months[0][0], chart_months[0][1], 1)
# For per-project chart: {project_name: [month0_cost, month1_cost, ...]}}
chart_labels = [f"{calendar.month_abbr[m]} {y}" for y, m in chart_months]
# 1 query: monthly payroll totals (grouped by month)
paid_by_month = {}
for row in PayrollRecord.objects.filter(date__gte=chart_start).annotate(
month=TruncMonth('date')
).values('month').annotate(total=Sum('amount')):
paid_by_month[(row['month'].year, row['month'].month)] = float(row['total'])
chart_totals = [paid_by_month.get(ym, 0) for ym in chart_months]
# 1 query: monthly overtime totals (grouped by month)
ot_by_month = {}
for row in PayrollAdjustment.objects.filter(
type='OVERTIME', date__gte=chart_start
).annotate(month=TruncMonth('date')).values('month').annotate(total=Sum('amount')):
ot_by_month[(row['month'].year, row['month'].month)] = float(row['total'])
ot_history_totals = [ot_by_month.get(ym, 0) for ym in chart_months]
# 1 query + prefetch: all work logs in the 6-month period for per-project costs
all_project_names = list(Project.objects.values_list('name', flat=True).order_by('name'))
project_monthly = {name: [] for name in all_project_names}
project_monthly = {name: [0] * len(chart_months) for name in all_project_names}
ot_history_totals = [] # Overtime history
all_chart_logs = WorkLog.objects.filter(
date__gte=chart_start
).select_related('project').prefetch_related('workers')
for year, month in chart_months:
chart_labels.append(f"{calendar.month_abbr[month]} {year}")
# Total paid out this month
_, last_day = calendar.monthrange(year, month)
month_start = datetime.date(year, month, 1)
month_end = datetime.date(year, month, last_day)
month_paid = PayrollRecord.objects.filter(
date__gte=month_start, date__lte=month_end
).aggregate(total=Sum('amount'))['total'] or 0
chart_totals.append(float(month_paid))
# Overtime paid this month
ot_month_total = PayrollAdjustment.objects.filter(
type='OVERTIME',
date__gte=month_start,
date__lte=month_end
).aggregate(total=Sum('amount'))['total'] or 0
ot_history_totals.append(float(ot_month_total))
# Per-project labour cost this month (from work logs × day rates)
month_logs = WorkLog.objects.filter(
date__gte=month_start, date__lte=month_end
).prefetch_related('workers', 'project')
project_cost_month = {name: 0 for name in all_project_names}
for log in month_logs:
for log in all_chart_logs:
month_key = (log.date.year, log.date.month)
idx = chart_month_index.get(month_key)
if idx is not None:
pname = log.project.name
for worker in log.workers.all():
project_cost_month[pname] = project_cost_month.get(pname, 0) + float(worker.day_rate)
for name in all_project_names:
project_monthly[name].append(project_cost_month.get(name, 0))
if pname in project_monthly:
for worker in log.workers.all():
project_monthly[pname][idx] += float(worker.day_rate)
# Filter out projects with zero cost across all months
project_chart_data = [
@ -976,32 +1036,33 @@ def process_payment(request, worker_id):
total_amount = logs_amount + adj_amount
if log_count > 0 or pending_adjustments.exists():
# Create Payroll Record
payroll_record = PayrollRecord.objects.create(
worker=worker,
amount=total_amount,
date=timezone.now().date()
)
# Link logs
payroll_record.work_logs.set(unpaid_logs)
# Link Adjustments and Handle Loans
for adj in pending_adjustments:
adj.payroll_record = payroll_record
adj.save()
# Update Loan Balance if it's a repayment
if adj.type == 'LOAN_REPAYMENT' and adj.loan:
adj.loan.balance -= adj.amount
if adj.loan.balance <= 0:
adj.loan.balance = 0
adj.loan.is_active = False
adj.loan.save()
with transaction.atomic():
# Create Payroll Record
payroll_record = PayrollRecord.objects.create(
worker=worker,
amount=total_amount,
date=timezone.now().date()
)
payroll_record.save()
# Link logs
payroll_record.work_logs.set(unpaid_logs)
# Email Notification
# Link Adjustments and Handle Loans
for adj in pending_adjustments:
adj.payroll_record = payroll_record
adj.save()
# Update Loan Balance if it's a repayment
if adj.type == 'LOAN_REPAYMENT' and adj.loan:
adj.loan.balance -= adj.amount
if adj.loan.balance <= 0:
adj.loan.balance = 0
adj.loan.is_active = False
adj.loan.save()
payroll_record.save()
# Email Notification (outside transaction — failure should not roll back payment)
subject = f"Payslip for {worker.name} - {payroll_record.date}"
# Prepare Context
@ -1182,11 +1243,17 @@ def add_loan(request):
if request.method == 'POST':
worker_id = request.POST.get('worker')
amount = request.POST.get('amount')
amount_str = request.POST.get('amount')
reason = request.POST.get('reason')
date = request.POST.get('date') or timezone.now().date()
if worker_id and amount:
try:
amount = Decimal(amount_str) if amount_str else None
except Exception:
messages.error(request, "Invalid amount.")
return redirect('/payroll/?status=loans')
if worker_id and amount and amount > 0:
worker = get_object_or_404(Worker, pk=worker_id)
Loan.objects.create(
worker=worker,
@ -1195,7 +1262,9 @@ def add_loan(request):
reason=reason
)
messages.success(request, f"Loan of R{amount} recorded for {worker.name}.")
elif amount is not None and amount <= 0:
messages.error(request, "Amount must be greater than zero.")
return redirect('/payroll/?status=loans')
@ -1210,11 +1279,21 @@ def add_adjustment(request):
if request.method == 'POST':
worker_ids = request.POST.getlist('workers')
adj_type = request.POST.get('type')
amount = request.POST.get('amount')
amount_str = request.POST.get('amount')
description = request.POST.get('description')
date = request.POST.get('date') or timezone.now().date()
loan_id = request.POST.get('loan_id') # Optional, for repayments
try:
amount = Decimal(amount_str) if amount_str else None
except Exception:
messages.error(request, "Invalid amount.")
return redirect('payroll_dashboard')
if amount is not None and amount <= 0:
messages.error(request, "Amount must be greater than zero.")
return redirect('payroll_dashboard')
if worker_ids and amount and adj_type:
success_names = []
skip_names = []
@ -1224,7 +1303,7 @@ def add_adjustment(request):
# --- ADVANCE: validate, create adjustment + standalone PayrollRecord, send payslip ---
if adj_type == 'ADVANCE':
advance_amount = Decimal(amount)
advance_amount = amount
# Must have unpaid work logs
unpaid_logs = worker.work_logs.exclude(paid_in__worker=worker)
@ -1248,24 +1327,24 @@ def add_adjustment(request):
skip_names.append(f"{worker.name} (R{advance_amount} exceeds available R{max_available:.2f})")
continue
# Create ADVANCE adjustment (stays pending — payroll_record=NULL)
PayrollAdjustment.objects.create(
worker=worker,
type='ADVANCE',
amount=advance_amount,
description=description or 'Advance payment',
date=date,
)
# Create ADVANCE adjustment + PayrollRecord atomically
with transaction.atomic():
PayrollAdjustment.objects.create(
worker=worker,
type='ADVANCE',
amount=advance_amount,
description=description or 'Advance payment',
date=date,
)
# Create standalone PayrollRecord (NO work_logs linked)
advance_date = date if isinstance(date, datetime.date) else timezone.now().date()
advance_record = PayrollRecord.objects.create(
worker=worker,
amount=advance_amount,
date=advance_date,
)
advance_date = date if isinstance(date, datetime.date) else timezone.now().date()
advance_record = PayrollRecord.objects.create(
worker=worker,
amount=advance_amount,
date=advance_date,
)
# Send advance payslip to Spark
# Send advance payslip to Spark (outside transaction)
subject = f"Advance Payment for {worker.name} - {advance_record.date}"
email_context = {
'record': advance_record,