from django.contrib.auth.decorators import login_required
from django.contrib.auth.forms import PasswordChangeForm
from django.utils.dateparse import parse_date
from datetime import datetime, time, timedelta
import base64
import re
import urllib.parse
import urllib.request
import csv
import io
import json
from django.http import JsonResponse, HttpResponse
from django.urls import reverse
from django.shortcuts import render, redirect, get_object_or_404
from django.db.models import Q, Sum, Value, DecimalField
from django.contrib import messages
from django.core.paginator import Paginator
from django.conf import settings
from django.db.models.functions import Coalesce
from .models import Voter, Tenant, Interaction, Donation, VoterLikelihood, EventParticipation, Event, EventType, InteractionType, DonationMethod, ElectionType, CampaignSettings, Volunteer, ParticipationStatus, VolunteerEvent, Interest, VolunteerRole, ScheduledCall
from .forms import VoterForm, InteractionForm, DonationForm, VoterLikelihoodForm, EventParticipationForm, VoterImportForm, AdvancedVoterSearchForm, EventParticipantAddForm, EventForm, VolunteerForm, VolunteerEventForm, VolunteerEventAddForm, DoorVisitLogForm, ScheduledCallForm, UserUpdateForm, EventParticipationImportForm, ParticipantMappingForm
import logging
import zoneinfo
from django.utils import timezone

from .permissions import role_required, can_view_donations, can_edit_voter, can_view_volunteers, can_edit_volunteer, can_view_voters, get_user_role
logger = logging.getLogger(__name__)

def _handle_uploaded_file(uploaded_file):
    """
    Handles uploaded CSV or Excel files, reads content, and extracts headers.
    Returns (headers, data_rows) or (None, None) if file type is unsupported or an error occurs.
    """
    # For simplicity, assuming CSV for now. Extend with openpyxl for Excel if needed.
    try:
        file_content = uploaded_file.read()
        decoded_file = file_content.decode('utf-8')
        io_string = io.StringIO(decoded_file)
        
        # Try to sniff CSV dialect
        try:
            dialect = csv.Sniffer().sniff(io_string.read(1024))
            io_string.seek(0)  # Rewind after sniffing
            reader = csv.reader(io_string, dialect)
        except csv.Error:
            # Not a CSV or sniffing failed, assume comma-separated
            io_string.seek(0)
            reader = csv.reader(io_string)

        headers = [header.strip() for header in next(reader)]
        data_rows = []
        for row in reader:
            if len(row) == len(headers):
                data_rows.append([item.strip() for item in row])
            else:
                logger.warning(f"Skipping malformed row in uploaded file: {row}")
                continue
        
        return headers, data_rows
    except Exception as e:
        logger.error(f"Error processing uploaded file: {e}")
        return None, None

def index(request):
    """
    Main landing page for Grassroots Campaign Manager.
    Displays a list of campaigns if the user is logged in but hasn't selected one.
    """
    tenants = Tenant.objects.all()
    selected_tenant_id = request.session.get('tenant_id')
    selected_tenant = None
    metrics = {}
    recent_interactions = []
    upcoming_events = []
    
    if selected_tenant_id:
        selected_tenant = Tenant.objects.filter(id=selected_tenant_id).first()
        if selected_tenant:
            voters = selected_tenant.voters.all()
            total_donations = Donation.objects.filter(voter__tenant=selected_tenant).aggregate(total=Sum('amount'))['total'] or 0
            
            # Get or create settings for the tenant
            campaign_settings, _ = CampaignSettings.objects.get_or_create(tenant=selected_tenant)
            donation_goal = campaign_settings.donation_goal
            
            donation_percentage = 0
            if donation_goal > 0:
                donation_percentage = float(round((total_donations / donation_goal) * 100, 1))
            
            metrics = {
                'total_registered_voters': voters.count(),
                'total_target_voters': voters.filter(is_targeted=True).count(),
                'total_supporting': voters.filter(candidate_support='supporting').count(),
                'total_target_households': voters.filter(is_targeted=True).exclude(address='').values('address').distinct().count(),
                'total_door_visits': voters.filter(door_visit=True).exclude(address='').values('address').distinct().count(),
                'total_signs': voters.filter(Q(yard_sign='wants') | Q(yard_sign='has')).exclude(address='').values('address').distinct().count(),
                'total_window_stickers': voters.filter(Q(window_sticker='wants') | Q(window_sticker='has')).count(),
                'total_donations': float(total_donations),
                'donation_goal': float(donation_goal),
                'donation_percentage': donation_percentage,
                'volunteers_count': Volunteer.objects.filter(tenant=selected_tenant).count(),
                'interactions_count': Interaction.objects.filter(voter__tenant=selected_tenant).count(),
                'events_count': Event.objects.filter(tenant=selected_tenant).count(),
                'pending_calls_count': ScheduledCall.objects.filter(tenant=selected_tenant, status='pending').count(),
            }
            
            recent_interactions = Interaction.objects.filter(voter__tenant=selected_tenant).order_by('-date')[:5]
            upcoming_events = Event.objects.filter(tenant=selected_tenant, date__gte=timezone.now().date()).order_by('date')[:5]


    context = {
        'tenants': tenants,
        'selected_tenant': selected_tenant,
        'metrics': metrics,
        'recent_interactions': recent_interactions,
        'upcoming_events': upcoming_events,
    }
    return render(request, 'core/index.html', context)

def select_campaign(request, tenant_id):
    """
    Sets the selected campaign in the session.
    """
    tenant = get_object_or_404(Tenant, id=tenant_id)
    request.session['tenant_id'] = tenant.id
    messages.success(request, f"You are now managing: {tenant.name}")
    return redirect('index')

@role_required(['admin', 'campaign_manager', 'campaign_staff', 'system_admin', 'campaign_admin'], permission='core.view_voter')
def voter_list(request):
    """
    List and search voters. Restricted to selected tenant.
    """
    selected_tenant_id = request.session.get("tenant_id")
    if not selected_tenant_id:
        messages.warning(request, "Please select a campaign first.")
        return redirect("index")

    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    query = request.GET.get("q")
    voters = Voter.objects.filter(tenant=tenant).order_by("last_name", "first_name")

    # Filtering based on dashboard metrics
    if request.GET.get("is_targeted") == "true":
        voters = voters.filter(is_targeted=True)
    if request.GET.get("support") == "supporting":
        voters = voters.filter(candidate_support="supporting")
    if request.GET.get("has_address") == "true":
        voters = voters.exclude(address__isnull=True).exclude(address="")
    if request.GET.get("visited") == "true":
        voters = voters.filter(door_visit=True)
    if request.GET.get("yard_sign") == "true":
        voters = voters.filter(Q(yard_sign="wants") | Q(yard_sign="has"))
    if request.GET.get("window_sticker") == "true":
        voters = voters.filter(Q(window_sticker="wants") | Q(window_sticker="has"))
    if request.GET.get("has_donations") == "true":
        voters = voters.filter(donations__isnull=False).distinct()

    if query:
        query = query.strip()
        search_filter = Q(first_name__icontains=query) | Q(last_name__icontains=query) | Q(voter_id__iexact=query)

        if "," in query:
            parts = [p.strip() for p in query.split(",")]
            if len(parts) >= 2:
                last_part = parts[0]
                first_part = parts[1]
                search_filter |= Q(last_name__icontains=last_part, first_name__icontains=first_part)
        elif " " in query:
            parts = query.split()
            if len(parts) >= 2:
                first_part = parts[0]
                last_part = " ".join(parts[1:])
                search_filter |= Q(first_name__icontains=first_part, last_name__icontains=last_part)

        voters = voters.filter(search_filter).order_by("last_name", "first_name")
        
    paginator = Paginator(voters, 50)
    page_number = request.GET.get('page')
    voters_page = paginator.get_page(page_number)
    

    context = {
        "voters": voters_page,
        "query": query,
        "selected_tenant": tenant,
        "call_form": ScheduledCallForm(tenant=tenant),
    }
    return render(request, "core/voter_list.html", context)


@role_required(['admin', 'campaign_manager', 'campaign_staff', 'system_admin', 'campaign_admin'], permission='core.view_voter')
def voter_detail(request, voter_id):
    """
    360-degree view of a voter.
    """
    selected_tenant_id = request.session.get('tenant_id')
    if not selected_tenant_id:
        messages.warning(request, "Please select a campaign first.")
        return redirect('index')

    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    voter = get_object_or_404(Voter, id=voter_id, tenant=tenant)


    context = {
        'voter': voter,
        'selected_tenant': tenant,
        'voting_records': voter.voting_records.all().order_by('-election_date'),
        'donations': voter.donations.all().order_by('-date'),
        'interactions': voter.interactions.all().order_by('-date'),
        'event_participations': voter.event_participations.all().order_by('-event__date'),
        'likelihoods': voter.likelihoods.all(),
        'voter_form': VoterForm(instance=voter, user=request.user, tenant=tenant),
        'interaction_form': InteractionForm(tenant=tenant),
        'donation_form': DonationForm(tenant=tenant),
        'likelihood_form': VoterLikelihoodForm(tenant=tenant),
        'event_participation_form': EventParticipationForm(tenant=tenant),
        'call_form': ScheduledCallForm(tenant=tenant),
    }
    return render(request, 'core/voter_detail.html', context)

@role_required(["admin", "campaign_manager", "campaign_staff", "system_admin", "campaign_admin"], permission="core.change_voter")
def voter_edit(request, voter_id):
    """
    Update voter core demographics.
    """
    selected_tenant_id = request.session.get('tenant_id')
    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    voter = get_object_or_404(Voter, id=voter_id, tenant=tenant)

    if request.method == 'POST':
        # Log incoming coordinate data for debugging
        lat_raw = request.POST.get('latitude')
        lon_raw = request.POST.get('longitude')
        logger.info(f"Voter Edit POST: lat={lat_raw}, lon={lon_raw}")

        form = VoterForm(request.POST, instance=voter, user=request.user, tenant=tenant)
        if form.is_valid():
            # If coordinates were provided in POST, ensure they are applied to the instance
            # This handles cases where readonly or other widget settings might interfere
            voter = form.save(commit=False);
            if lat_raw:
                try:
                    voter.latitude = lat_raw
                except: pass
            if lon_raw:
                try:
                    voter.longitude = lon_raw
                except: pass
            
            voter.save()
            messages.success(request, "Voter profile updated successfully.")
        else:
            logger.warning(f"Voter Edit Form Invalid: {form.errors}")
            for field, errors in form.errors.items():
                for error in errors:
                    messages.error(request, f"Error in {field}: {error}")
    return redirect('voter_detail', voter_id=voter.id)

def add_interaction(request, voter_id):
    selected_tenant_id = request.session.get('tenant_id')
    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    voter = get_object_or_404(Voter, id=voter_id, tenant=tenant)

    if request.method == 'POST':
        form = InteractionForm(request.POST, tenant=tenant)
        if form.is_valid():
            interaction = form.save(commit=False)
            interaction.voter = voter
            interaction.save()
            messages.success(request, "Interaction added.")
    return redirect(reverse('voter_detail', kwargs={'voter_id': voter.id}) + '?active_tab=interactions')

def edit_interaction(request, interaction_id):
    selected_tenant_id = request.session.get('tenant_id')
    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    interaction = get_object_or_404(Interaction, id=interaction_id, voter__tenant=tenant)
    
    if request.method == 'POST':
        form = InteractionForm(request.POST, instance=interaction, tenant=tenant)
        if form.is_valid():
            form.save()
            messages.success(request, "Interaction updated.")
    return redirect(reverse('voter_detail', kwargs={'voter_id': interaction.voter.id}) + '?active_tab=interactions')

def delete_interaction(request, interaction_id):
    selected_tenant_id = request.session.get('tenant_id')
    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    interaction = get_object_or_404(Interaction, id=interaction_id, voter__tenant=tenant)
    voter_id = interaction.voter.id
    
    if request.method == 'POST':
        interaction.delete()
        messages.success(request, "Interaction deleted.")
    return redirect(reverse('voter_detail', kwargs={'voter_id': voter_id}) + '?active_tab=interactions')

@role_required(['admin', 'campaign_manager', 'campaign_staff', 'system_admin', 'campaign_admin'], permission='core.add_donation')
def add_donation(request, voter_id):
    selected_tenant_id = request.session.get('tenant_id')
    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    voter = get_object_or_404(Voter, id=voter_id, tenant=tenant)

    if request.method == 'POST':
        form = DonationForm(request.POST, tenant=tenant)
        if form.is_valid():
            donation = form.save(commit=False)
            donation.voter = voter
            donation.save()
            messages.success(request, "Donation recorded.")
    return redirect(reverse('voter_detail', kwargs={'voter_id': voter.id}) + '?active_tab=donations')

@role_required(['admin', 'campaign_manager', 'campaign_staff', 'system_admin', 'campaign_admin'], permission='core.change_donation')
def edit_donation(request, donation_id):
    selected_tenant_id = request.session.get('tenant_id')
    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    donation = get_object_or_404(Donation, id=donation_id, voter__tenant=tenant)
    
    if request.method == 'POST':
        form = DonationForm(request.POST, instance=donation, tenant=tenant)
        if form.is_valid():
            form.save()
            messages.success(request, "Donation updated.")
    return redirect(reverse('voter_detail', kwargs={'voter_id': donation.voter.id}) + '?active_tab=donations')

@role_required(['admin', 'campaign_manager', 'campaign_staff', 'system_admin', 'campaign_admin'], permission='core.delete_donation')
def delete_donation(request, donation_id):
    selected_tenant_id = request.session.get('tenant_id')
    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    donation = get_object_or_404(Donation, id=donation_id, voter__tenant=tenant)
    voter_id = donation.voter.id
    
    if request.method == 'POST':
        donation.delete()
        messages.success(request, "Donation deleted.")
    return redirect(reverse('voter_detail', kwargs={'voter_id': voter_id}) + '?active_tab=donations')

def add_likelihood(request, voter_id):
    selected_tenant_id = request.session.get('tenant_id')
    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    voter = get_object_or_404(Voter, id=voter_id, tenant=tenant)

    if request.method == 'POST':
        form = VoterLikelihoodForm(request.POST, tenant=tenant)
        if form.is_valid():
            likelihood = form.save(commit=False)
            likelihood.voter = voter
            # Handle potential duplicate election_type
            VoterLikelihood.objects.filter(voter=voter, election_type=likelihood.election_type).delete()
            likelihood.save()
            messages.success(request, "Likelihood updated.")
    return redirect('voter_detail', voter_id=voter.id)

def edit_likelihood(request, likelihood_id):
    selected_tenant_id = request.session.get('tenant_id')
    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    likelihood = get_object_or_404(VoterLikelihood, id=likelihood_id, voter__tenant=tenant)
    
    if request.method == 'POST':
        form = VoterLikelihoodForm(request.POST, instance=likelihood, tenant=tenant)
        if form.is_valid():
            election_type = form.cleaned_data['election_type']
            if VoterLikelihood.objects.filter(voter=likelihood.voter, election_type=election_type).exclude(id=likelihood.id).exists():
                VoterLikelihood.objects.filter(voter=likelihood.voter, election_type=election_type).exclude(id=likelihood.id).delete()
            form.save()
            messages.success(request, "Likelihood updated.")
    return redirect('voter_detail', voter_id=likelihood.voter.id)

def delete_likelihood(request, likelihood_id):
    selected_tenant_id = request.session.get('tenant_id')
    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    likelihood = get_object_or_404(VoterLikelihood, id=likelihood_id, voter__tenant=tenant)
    voter_id = likelihood.voter.id
    
    if request.method == 'POST':
        likelihood.delete()
        messages.success(request, "Likelihood record deleted.")
    return redirect('voter_detail', voter_id=voter_id)

def add_event_participation(request, voter_id):
    selected_tenant_id = request.session.get('tenant_id')
    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    voter = get_object_or_404(Voter, id=voter_id, tenant=tenant)

    if request.method == 'POST':
        form = EventParticipationForm(request.POST, tenant=tenant)
        if form.is_valid():
            participation = form.save(commit=False)
            participation.voter = voter
            # Avoid duplicate participation
            if not EventParticipation.objects.filter(voter=voter, event=participation.event).exists():
                participation.save()
                messages.success(request, "Event participation added.")
            else:
                messages.warning(request, "Voter is already participating in this event.")
    return redirect(reverse('voter_detail', kwargs={'voter_id': voter.id}) + '?active_tab=events')

def edit_event_participation(request, participation_id):
    selected_tenant_id = request.session.get('tenant_id')
    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    participation = get_object_or_404(EventParticipation, id=participation_id, event__tenant=tenant)
    
    if request.method == 'POST':
        form = EventParticipationForm(request.POST, instance=participation, tenant=tenant)
        if form.is_valid():
            event = form.cleaned_data['event']
            if EventParticipation.objects.filter(voter=participation.voter, event=event).exclude(id=participation.id).exists():
                messages.warning(request, "Voter is already participating in that event.")
            else:
                form.save()
                messages.success(request, "Event participation updated.")
    return redirect(reverse('voter_detail', kwargs={'voter_id': participation.voter.id}) + '?active_tab=events')

def delete_event_participation(request, participation_id):
    selected_tenant_id = request.session.get('tenant_id')
    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    participation = get_object_or_404(EventParticipation, id=participation_id, event__tenant=tenant)
    voter_id = participation.voter.id
    
    if request.method == 'POST':
        participation.delete()
        messages.success(request, "Event participation removed.")
    return redirect(reverse('voter_detail', kwargs={'voter_id': voter_id}) + '?active_tab=events')

def voter_geocode(request, voter_id):
    """
    Manually trigger geocoding for a voter, potentially using values from the request.
    """
    selected_tenant_id = request.session.get('tenant_id')
    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    voter = get_object_or_404(Voter, id=voter_id, tenant=tenant)

    if request.method == 'POST':
        street = request.POST.get('address_street', voter.address_street)
        city = request.POST.get('city', voter.city)
        state = request.POST.get('state', voter.state)
        zip_code = request.POST.get('zip_code', voter.zip_code)
        
        parts = [street, city, state, zip_code]
        full_address = ", ".join([p for p in parts if p])
        
        # Use a temporary instance to avoid saving until the user clicks "Save" in the modal
        temp_voter = Voter(
            address_street=street,
            city=city,
            state=state,
            zip_code=zip_code,
            address=full_address
        )
        success, error_msg = temp_voter.geocode_address()
        
        if success:
            return JsonResponse({
                'success': True,
                'latitude': str(temp_voter.latitude),
                'longitude': str(temp_voter.longitude),
                'address': full_address
            })
        else:
            return JsonResponse({
                'success': False,
                'error': f"Geocoding failed: {error_msg or 'No results found.'}"
            })

    return JsonResponse({'success': False, 'error': 'Invalid request method.'})

@role_required(['admin', 'campaign_manager', 'campaign_staff', 'system_admin', 'campaign_admin'], permission='core.view_voter')
def voter_advanced_search(request):
    """
    Advanced search for voters with multiple filters.
    """
    selected_tenant_id = request.session.get("tenant_id")
    if not selected_tenant_id:
        messages.warning(request, "Please select a campaign first.")
        return redirect("index")

    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    voters = Voter.objects.filter(tenant=tenant).order_by("last_name", "first_name")
    
    form = AdvancedVoterSearchForm(request.GET)
    if form.is_valid():
        data = form.cleaned_data
        if data.get('first_name'):
            voters = voters.filter(first_name__icontains=data['first_name'])
        if data.get('last_name'):
            voters = voters.filter(last_name__icontains=data['last_name'])
        if data.get('address'):
            voters = voters.filter(Q(address__icontains=data['address']) | Q(address_street__icontains=data['address']))
        if data.get('voter_id'):
            voters = voters.filter(voter_id__iexact=data['voter_id'])
        if data.get('birth_month'):
            voters = voters.filter(birthdate__month=data['birth_month'])
        if data.get('city'):
            voters = voters.filter(city__icontains=data['city'])
        if data.get('zip_code'):
            voters = voters.filter(zip_code__icontains=data['zip_code'])
        if data.get('district'):
            voters = voters.filter(district=data['district'])
        if data.get('precinct'):
            voters = voters.filter(precinct=data['precinct'])
        if data.get('email'):
            voters = voters.filter(email__icontains=data['email'])
        if data.get('phone_type'):
            voters = voters.filter(phone_type=data['phone_type'])
        if data.get('is_targeted'):
            voters = voters.filter(is_targeted=True)
        if data.get('candidate_support'):
            voters = voters.filter(candidate_support=data['candidate_support'])
        if data.get('yard_sign'):
            voters = voters.filter(yard_sign=data['yard_sign'])
        if data.get('window_sticker'):
            voters = voters.filter(window_sticker=data['window_sticker'])

        # Add donation amount filters
        min_total_donation = data.get('min_total_donation')
        max_total_donation = data.get('max_total_donation')

        if min_total_donation is not None or max_total_donation is not None:
            # Annotate each voter with their total donation amount, treating no donations as 0
            voters = voters.annotate(total_donation_amount=Coalesce(Sum('donations__amount'), Value(0), output_field=DecimalField()))

            if min_total_donation is not None:
                voters = voters.filter(total_donation_amount__gte=min_total_donation)
            if max_total_donation is not None:
                voters = voters.filter(total_donation_amount__lte=max_total_donation)

    paginator = Paginator(voters, 50)
    page_number = request.GET.get('page')
    voters_page = paginator.get_page(page_number)


    context = {
        'form': form,
        'voters': voters_page,
        'selected_tenant': tenant,
        'call_form': ScheduledCallForm(tenant=tenant),
    }
    return render(request, "core/voter_advanced_search.html", context)

def export_voters_csv(request):
    """
    Exports selected or filtered voters to a CSV file.
    """
    selected_tenant_id = request.session.get("tenant_id")
    if not selected_tenant_id:
        messages.warning(request, "Please select a campaign first.")
        return redirect("index")

    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    
    if request.method != 'POST':
        return redirect('voter_advanced_search')
        
    action = request.POST.get('action')
    voters = Voter.objects.filter(tenant=tenant)
    
    if action == 'export_selected':
        voter_ids = request.POST.getlist('selected_voters')
        voters = voters.filter(id__in=voter_ids)
    else: # export_all
        # Re-apply filters from hidden inputs
        # These are passed as filter_fieldname
        filters = {}
        for key, value in request.POST.items():
            if key.startswith('filter_') and value:
                field_name = key.replace('filter_', '')
                filters[field_name] = value
        
        # We can use the AdvancedVoterSearchForm to validate and apply filters
        # but we need to pass data without the prefix
        form = AdvancedVoterSearchForm(filters)
        if form.is_valid():
            data = form.cleaned_data
            if data.get('first_name'):
                voters = voters.filter(first_name__icontains=data['first_name'])
            if data.get('last_name'):
                voters = voters.filter(last_name__icontains=data['last_name'])
            if data.get('address'):
                voters = voters.filter(Q(address__icontains=data['address']) | Q(address_street__icontains=data['address']))
            if data.get('voter_id'):
                voters = voters.filter(voter_id__iexact=data['voter_id'])
            if data.get('birth_month'):
                voters = voters.filter(birthdate__month=data['birth_month'])
            if data.get('city'):
                voters = voters.filter(city__icontains=data['city'])
            if data.get('zip_code'):
                voters = voters.filter(zip_code__icontains=data['zip_code'])
            if data.get('district'):
                voters = voters.filter(district=data['district'])
            if data.get('precinct'):
                voters = voters.filter(precinct=data['precinct'])
            if data.get('email'):
                voters = voters.filter(email__icontains=data['email'])
            if data.get('phone_type'):
                voters = voters.filter(phone_type=data['phone_type'])
            if data.get('is_targeted'):
                voters = voters.filter(is_targeted=True)
            if data.get('candidate_support'):
                voters = voters.filter(candidate_support=data['candidate_support'])
            if data.get('yard_sign'):
                voters = voters.filter(yard_sign=data['yard_sign'])
            if data.get('window_sticker'):
                voters = voters.filter(window_sticker=data['window_sticker'])

            # Add donation amount filters for export
            min_total_donation = data.get('min_total_donation')
            max_total_donation = data.get('max_total_donation')

            if min_total_donation is not None or max_total_donation is not None:
                voters = voters.annotate(total_donation_amount=Coalesce(Sum('donations__amount'), Value(0), output_field=DecimalField()))

                if min_total_donation is not None:
                    voters = voters.filter(total_donation_amount__gte=min_total_donation)
                if max_total_donation is not None:
                    voters = voters.filter(total_donation_amount__lte=max_total_donation)

    voters = voters.order_by('last_name', 'first_name')
    
    response = HttpResponse(content_type='text/csv')
    response['Content-Disposition'] = f'attachment; filename="voters_export_{timezone.now().strftime("%Y%m%d_%H%M%S")}.csv"'
    
    writer = csv.writer(response)
    writer.writerow([
        'Voter ID', 'First Name', 'Last Name', 'Nickname', 'Birthdate', 
        'Address', 'City', 'State', 'Zip Code', 'Phone', 'Phone Type', 'Secondary Phone', 'Secondary Phone Type', 'Email', 
        'District', 'Precinct', 'Is Targeted', 'Support', 'Yard Sign', 'Window Sticker', 'Notes'
    ])
    
    for voter in voters:
        writer.writerow([
            voter.voter_id, voter.first_name, voter.last_name, voter.nickname, voter.birthdate,
            voter.address, voter.city, voter.state, voter.zip_code, voter.phone, voter.get_phone_type_display(), voter.secondary_phone, voter.get_secondary_phone_type_display(), voter.email,
            voter.district, voter.precinct, 'Yes' if voter.is_targeted else 'No',
            voter.get_candidate_support_display(), voter.get_yard_sign_display(), voter.get_window_sticker_display(), voter.notes
        ])
        
    return response

def voter_delete(request, voter_id):
    """
    Delete a voter profile.
    """
    selected_tenant_id = request.session.get('tenant_id')
    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    voter = get_object_or_404(Voter, id=voter_id, tenant=tenant)

    if request.method == 'POST':
        voter.delete()
        messages.success(request, "Voter profile deleted successfully.")
        return redirect('voter_list')
    
    return redirect('voter_detail', voter_id=voter.id)

@role_required(['admin', 'campaign_manager', 'campaign_staff', 'system_admin', 'campaign_admin'], permission='core.view_voter')
def bulk_send_sms(request):
    """
    Sends bulk SMS to selected voters using Twilio API.
    """
    if request.method != 'POST':
        return redirect('voter_advanced_search')
        
    selected_tenant_id = request.session.get("tenant_id")
    if not selected_tenant_id:
        messages.warning(request, "Please select a campaign first.")
        return redirect("index")
    
    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    settings = getattr(tenant, 'settings', None)
    if not settings:
        messages.error(request, "Campaign settings not found.")
        return redirect('voter_advanced_search')
    
    account_sid = settings.twilio_account_sid
    auth_token = settings.twilio_auth_token
    from_number = settings.twilio_from_number
    
    if not account_sid or not auth_token or not from_number:
        messages.error(request, "Twilio configuration is incomplete in Campaign Settings.")
        return redirect('voter_advanced_search')
        
    voter_ids = request.POST.getlist('selected_voters')
    message_body = request.POST.get('message_body')

    # client_time_str is not defined, removed to avoid error.
    # interaction_date = timezone.now()
    # if client_time_str:
    #     try:
    #         interaction_date = datetime.fromisoformat(client_time_str)
    #         if timezone.is_naive(interaction_date):
    #             interaction_date = timezone.make_aware(interaction_date)
    #     except Exception as e:
    #         logger.warning(f'Failed to parse client_time {client_time_str}: {e}')
    
    if not message_body:
        messages.error(request, "Message body cannot be empty.")
        return redirect('voter_advanced_search')
        
    voters = Voter.objects.filter(tenant=tenant, id__in=voter_ids, phone_type='cell').exclude(phone='')
    
    if not voters.exists():
        messages.warning(request, "No voters with a valid cell phone number were selected.")
        return redirect('voter_advanced_search')

    success_count = 0
    fail_count = 0
    
    auth_str = f"{account_sid}:{auth_token}"
    auth_header = base64.b64encode(auth_str.encode()).decode()
    url = f"https://api.twilio.com/2010-04-01/Accounts/{account_sid}/Messages.json"
    
    # Get or create interaction type for SMS
    interaction_type, _ = InteractionType.objects.get_or_create(tenant=tenant, name="SMS Text")

    for voter in voters:
        # Format phone to E.164 (assume US +1)
        digits = re.sub(r'\\D', '', str(voter.phone))
        if len(digits) == 10:
            to_number = f"+1{digits}"
        elif len(digits) == 11 and digits.startswith('1'):
            to_number = f"+{digits}"
        else:
            # Skip invalid phone numbers
            fail_count += 1
            continue
            
        data_dict = {
            'To': to_number,
            'From': from_number,
            'Body': message_body
        }
        data = urllib.parse.urlencode(data_dict).encode()
        
        req = urllib.request.Request(url, data=data, method='POST')
        req.add_header("Authorization", f"Basic {auth_header}")
        
        try:
            with urllib.request.urlopen(req, timeout=10) as response:
                if response.status in [200, 201]:
                    success_count += 1
                    # Log interaction
                    Interaction.objects.create(
                        voter=voter,
                        # volunteer=volunteer, # volunteer is not defined here
                        type=interaction_type,
                        # date=interaction_date, # interaction_date removed
                        description='Mass SMS Text',
                        notes=message_body
                    )
                else:
                    fail_count += 1
        except Exception as e:
            logger.error(f"Error sending SMS to {voter.phone}: {e}")
            fail_count += 1
            
    messages.success(request, f"Bulk SMS process completed: {success_count} successful, {fail_count} failed/skipped.")
    return redirect('voter_advanced_search')

@role_required(['admin', 'campaign_manager', 'campaign_staff', 'system_admin', 'campaign_admin'], permission='core.view_event')
def event_list(request):
    selected_tenant_id = request.session.get("tenant_id")
    if not selected_tenant_id:
        messages.warning(request, "Please select a campaign first.")
        return redirect("index")

    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    events = Event.objects.filter(tenant=tenant).order_by('-date')
    

    context = {
        'tenant': tenant,
        'events': events,
        'selected_tenant': tenant,
    }
    return render(request, 'core/event_list.html', context)

@role_required(['admin', 'campaign_manager', 'campaign_staff', 'system_admin', 'campaign_admin'], permission='core.view_event')
def event_detail(request, event_id):
    selected_tenant_id = request.session.get("tenant_id")
    if not selected_tenant_id:
        messages.warning(request, "Please select a campaign first.")
        return redirect("index")

    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    event = get_object_or_404(Event, id=event_id, tenant=tenant)
    participations = event.participations.all().select_related('voter', 'participation_status').order_by('voter__last_name', 'voter__first_name')
    
    # Get assigned volunteers
    volunteers = event.volunteer_assignments.all().select_related('volunteer').order_by('volunteer__last_name', 'volunteer__first_name')
    
    # Form for adding a new participant
    add_form = EventParticipantAddForm(tenant=tenant)
    # Form for adding a new volunteer
    default_role = event.default_volunteer_role
    if not default_role and event.event_type:
        default_role = event.event_type.default_volunteer_role
    add_volunteer_form = VolunteerEventAddForm(tenant=tenant, initial={'role_type': default_role})
    
    participation_statuses = ParticipationStatus.objects.filter(tenant=tenant, is_active=True)
    

    context = {
        'tenant': tenant,
        'selected_tenant': tenant,
        'event': event,
        'participations': participations,
        'volunteers': volunteers,
        'add_form': add_form,
        'add_volunteer_form': add_volunteer_form,
        'participation_statuses': participation_statuses,
    }
    return render(request, 'core/event_detail.html', context)

def event_add_participant(request, event_id):
    tenant_id = request.session.get("tenant_id")
    tenant = get_object_or_404(Tenant, id=tenant_id)
    event = get_object_or_404(Event, id=event_id, tenant=tenant)
    
    if request.method == 'POST':
        form = EventParticipantAddForm(request.POST, tenant=tenant)
        if form.is_valid():
            participation = form.save(commit=False)
            participation.event = event
            if not EventParticipation.objects.filter(event=event, voter=participation.voter).exists():
                participation.save()
                messages.success(request, f"{participation.voter} added to event.")
            else:
                messages.warning(request, "Voter is already a participant.")
        else:
            messages.error(request, "Error adding participant.")
            
    return redirect('event_detail', event_id=event.id)

def event_edit_participant(request, participation_id):
    tenant_id = request.session.get("tenant_id")
    tenant = get_object_or_404(Tenant, id=tenant_id)
    participation = get_object_or_404(EventParticipation, id=participation_id, event__tenant=tenant)
    
    if request.method == 'POST':
        status_id = request.POST.get('participation_status')
        if status_id:
            status = get_object_or_404(ParticipationStatus, id=status_id, tenant=tenant)
            participation.participation_status = status
            participation.save()
            messages.success(request, f"Participation updated for {participation.voter}.")
        else:
            messages.error(request, "Invalid status.")
            
    return redirect('event_detail', event_id=participation.event.id)

def event_delete_participant(request, participation_id):
    tenant_id = request.session.get("tenant_id")
    tenant = get_object_or_404(Tenant, id=tenant_id)
    participation = get_object_or_404(EventParticipation, id=participation_id, event__tenant=tenant)
    event_id = participation.event.id
    voter_name = str(participation.voter)
    participation.delete()
    messages.success(request, f"{voter_name} removed from event.")
    return redirect('event_detail', event_id=event_id)

def voter_search_json(request):
    """
    JSON endpoint for voter search, used by autocomplete/search UI.
    """
    selected_tenant_id = request.session.get("tenant_id")
    if not selected_tenant_id:
        return JsonResponse({"results": []})
        
    query = request.GET.get("q", "").strip()
    if len(query) < 2:
        return JsonResponse({"results": []})
        
    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    voters = Voter.objects.filter(tenant=tenant)
    
    search_filter = Q(first_name__icontains=query) | Q(last_name__icontains=query) | Q(voter_id__iexact=query)
    
    if "," in query:
        parts = [p.strip() for p in query.split(",") ]
        if len(parts) >= 2:
            search_filter |= Q(last_name__icontains=parts[0], first_name__icontains=parts[1])
            
    results = voters.filter(search_filter).order_by("last_name", "first_name")[:20]
    
    data = []
    for v in results:
        data.append({
            "id": v.id,
            "text": f"{v.last_name}, {v.first_name} ({v.voter_id})",
            "address": v.address,
            "phone": v.phone
        })
        
    return JsonResponse({"results": data})

@role_required(['admin', 'campaign_manager', 'campaign_staff', 'system_admin', 'campaign_admin'], permission='core.view_volunteer')
def volunteer_list(request):
    selected_tenant_id = request.session.get("tenant_id")
    if not selected_tenant_id:
        messages.warning(request, "Please select a campaign first.")
        return redirect("index")

    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    volunteers = Volunteer.objects.filter(tenant=tenant).order_by('last_name', 'first_name')
    
    # Simple search
    query = request.GET.get("q")
    if query:
        volunteers = volunteers.filter(
            Q(first_name__icontains=query) | Q(last_name__icontains=query) | Q(email__icontains=query)
        )

    # Interest filter
    interest_id = request.GET.get("interest")
    if interest_id:
        volunteers = volunteers.filter(interests__id=interest_id).distinct()

    interests = Interest.objects.filter(tenant=tenant).order_by('name')
    
    paginator = Paginator(volunteers, 50)
    page_number = request.GET.get('page')
    volunteers_page = paginator.get_page(page_number)

    context = {
        'tenant': tenant,
        'selected_tenant': tenant,
        'volunteers': volunteers_page,
        'query': query,
        'interests': interests,
        'selected_interest': interest_id,
    }
    return render(request, 'core/volunteer_list.html', context)

@role_required(['admin', 'campaign_manager', 'campaign_staff', 'system_admin', 'campaign_admin'], permission='core.add_volunteer')
def volunteer_add(request):
    selected_tenant_id = request.session.get("tenant_id")
    if not selected_tenant_id:
        messages.warning(request, "Please select a campaign first.")
        return redirect("index")
    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    
    if request.method == 'POST':
        form = VolunteerForm(request.POST, tenant=tenant)
        if form.is_valid():
            volunteer = form.save(commit=False)
            volunteer.tenant = tenant
            volunteer.save()
            form.save_m2m() # Save interests
            messages.success(request, f"Volunteer {volunteer} added successfully.")
            return redirect('volunteer_detail', volunteer_id=volunteer.id)
    else:
        form = VolunteerForm(tenant=tenant)
        

    context = {
        'form': form,
        'tenant': tenant,
        'selected_tenant': tenant,
        'is_create': True,
    }
    return render(request, 'core/volunteer_detail.html', context)

@role_required(['admin', 'campaign_manager', 'campaign_staff', 'system_admin', 'campaign_admin'], permission='core.view_volunteer')
def volunteer_detail(request, volunteer_id):
    selected_tenant_id = request.session.get("tenant_id")
    if not selected_tenant_id:
        messages.warning(request, "Please select a campaign first.")
        return redirect("index")
    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    volunteer = get_object_or_404(Volunteer, id=volunteer_id, tenant=tenant)
    
    if request.method == 'POST':
        form = VolunteerForm(request.POST, instance=volunteer, tenant=tenant)
        if form.is_valid():
            form.save()
            messages.success(request, f"Volunteer {volunteer} updated successfully.")
            return redirect('volunteer_detail', volunteer_id=volunteer.id)
    else:
        form = VolunteerForm(instance=volunteer, tenant=tenant)
        
    assignments = volunteer.event_assignments.all().select_related('event')
    assign_form = VolunteerEventForm(tenant=tenant)
    

    context = {
        'volunteer': volunteer,
        'form': form,
        'assignments': assignments,
        'assign_form': assign_form,
        'tenant': tenant,
        'selected_tenant': tenant,
    }
    return render(request, 'core/volunteer_detail.html', context)

@role_required(['admin', 'campaign_manager', 'campaign_staff', 'system_admin', 'campaign_admin'], permission='core.delete_volunteer')
def volunteer_delete(request, volunteer_id):
    selected_tenant_id = request.session.get("tenant_id")
    if not selected_tenant_id:
        messages.warning(request, "Please select a campaign first.")
        return redirect("index")
    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    volunteer = get_object_or_404(Volunteer, id=volunteer_id, tenant=tenant)
    
    if request.method == 'POST':
        volunteer.delete()
        messages.success(request, "Volunteer deleted.")
        return redirect('volunteer_list')
    return redirect('volunteer_detail', volunteer_id=volunteer.id)

@role_required(['admin', 'campaign_manager', 'campaign_staff', 'system_admin', 'campaign_admin'], permission='core.change_volunteer')
def volunteer_assign_event(request, volunteer_id):
    selected_tenant_id = request.session.get("tenant_id")
    if not selected_tenant_id:
        messages.warning(request, "Please select a campaign first.")
        return redirect("index")
    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    volunteer = get_object_or_404(Volunteer, id=volunteer_id, tenant=tenant)
    
    if request.method == 'POST':
        form = VolunteerEventForm(request.POST, tenant=tenant)
        if form.is_valid():
            assignment = form.save(commit=False)
            assignment.volunteer = volunteer
            assignment.save()
            messages.success(request, f"Assigned to {assignment.event}.")
        else:
            messages.error(request, "Error assigning to event.")
            
    return redirect('volunteer_detail', volunteer_id=volunteer.id)

@role_required(['admin', 'campaign_manager', 'campaign_staff', 'system_admin', 'campaign_admin'], permission='core.change_volunteer')
def volunteer_remove_event(request, assignment_id):
    selected_tenant_id = request.session.get("tenant_id")
    if not selected_tenant_id:
        messages.warning(request, "Please select a campaign first.")
        return redirect("index")
    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    assignment = get_object_or_404(VolunteerEvent, id=assignment_id, volunteer__tenant=tenant)
    volunteer_id = assignment.volunteer.id
    assignment.delete()
    messages.success(request, "Assignment removed.")
    return redirect('volunteer_detail', volunteer_id=volunteer_id)

def interest_add(request):
    selected_tenant_id = request.session.get("tenant_id")
    if not selected_tenant_id:
        return JsonResponse({'success': False, 'error': 'No campaign selected.'})
    
    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    if request.method == 'POST':
        name = request.POST.get('name', '').strip()
        if name:
            interest, created = Interest.objects.get_or_create(tenant=tenant, name=name)
            if created:
                return JsonResponse({'success': True, 'id': interest.id, 'name': interest.name})
            else:
                return JsonResponse({'success': False, 'error': 'Interest already exists.'})
    return JsonResponse({'success': False, 'error': 'Invalid request.'})

def interest_delete(request, interest_id):
    selected_tenant_id = request.session.get("tenant_id")
    if not selected_tenant_id:
        return JsonResponse({'success': False, 'error': 'No campaign selected.'})
    
    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    interest = get_object_or_404(Interest, id=interest_id, tenant=tenant)
    
    if request.method == 'POST':
        interest.delete()
        return JsonResponse({'success': True})
    return JsonResponse({'success': False, 'error': 'Invalid request.'})

@role_required(['admin', 'campaign_manager', 'campaign_staff', 'system_admin', 'campaign_admin'], permission='core.add_event')
def event_create(request):
    selected_tenant_id = request.session.get("tenant_id")
    if not selected_tenant_id:
        messages.warning(request, "Please select a campaign first.")
        return redirect("index")

    tenant = get_object_or_404(Tenant, id=selected_tenant_id)

    if request.method == "POST":
        form = EventForm(request.POST, tenant=tenant)
        if form.is_valid():
            event = form.save(commit=False)
            event.tenant = tenant
            event.save()
            messages.success(request, "Event created successfully.")
            return redirect("event_detail", event_id=event.id)
    else:
        form = EventForm(tenant=tenant)


    context = {
        "form": form,
        "tenant": tenant,
        "selected_tenant": tenant,
        "is_create": True,
    }
    return render(request, "core/event_edit.html", context)

@role_required(['admin', 'campaign_manager', 'campaign_staff', 'system_admin', 'campaign_admin'], permission='core.change_event')
def event_edit(request, event_id):
    selected_tenant_id = request.session.get("tenant_id")
    if not selected_tenant_id:
        messages.warning(request, "Please select a campaign first.")
        return redirect("index")

    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    event = get_object_or_404(Event, id=event_id, tenant=tenant)

    if request.method == 'POST':
        form = EventForm(request.POST, instance=event, tenant=tenant)
        if form.is_valid():
            form.save()
            messages.success(request, "Event updated successfully.")
            return redirect('event_detail', event_id=event.id)
    else:
        form = EventForm(instance=event, tenant=tenant)


    context = {
        'form': form,
        'event': event,
        'tenant': tenant,
        'selected_tenant': tenant,
    }
    return render(request, 'core/event_edit.html', context)

@role_required(['admin', 'campaign_manager', 'system_admin', 'campaign_admin'], permission='core.change_event')
def import_participants(request, event_id):
    selected_tenant_id = request.session.get("tenant_id")
    if not selected_tenant_id:
        messages.warning(request, "Please select a campaign first.")
        return redirect("index")

    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    event = get_object_or_404(Event, id=event_id, tenant=tenant)

    if request.method == 'POST':
        form = EventParticipationImportForm(request.POST, request.FILES, event=event)
        if form.is_valid():
            uploaded_file = form.cleaned_data['file']
            
            headers, data_rows = _handle_uploaded_file(uploaded_file)
            
            if headers and data_rows:
                # Store data in session for the mapping step
                request.session['imported_participants_data'] = {
                    'event_id': event.id,
                    'headers': headers,
                    'data_rows': data_rows,
                    'file_name': uploaded_file.name
                }
                messages.info(request, f"File '{uploaded_file.name}' uploaded successfully. Now map the fields.")
                return redirect('import_participants_map_fields', event_id=event.id)
            else:
                messages.error(request, "Could not read data from the uploaded file. Please ensure it's a valid CSV/Excel.")
        else:
            messages.error(request, "No file was uploaded or an error occurred with the form.")
            # For debugging, you might want to log form.errors
            logger.error(f"EventParticipationImportForm errors: {form.errors}")
    
    return redirect('event_detail', event_id=event.id)

@role_required(['admin', 'campaign_manager', 'system_admin', 'campaign_admin'], permission='core.change_event')
def import_participants_map_fields(request, event_id):
    selected_tenant_id = request.session.get("tenant_id")
    if not selected_tenant_id:
        messages.warning(request, "Please select a campaign first.")
        return redirect("index")

    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    event = get_object_or_404(Event, id=event_id, tenant=tenant)

    imported_data = request.session.get('imported_participants_data')
    if not imported_data or imported_data['event_id'] != event.id:
        messages.error(request, "No data found to map. Please upload a file first.")
        return redirect('event_detail', event_id=event.id)

    headers = imported_data['headers']
    file_name = imported_data['file_name']

    if request.method == 'POST':
        form = ParticipantMappingForm(request.POST, headers=headers, tenant=tenant)
        if form.is_valid():
            email_column = form.cleaned_data['email_column']
            name_column = form.cleaned_data['name_column'] # Retrieve name column
            phone_column = form.cleaned_data['phone_column'] # Retrieve phone column
            participation_status_column = form.cleaned_data['participation_status_column']
            default_participation_status = form.cleaned_data['default_participation_status'] # Retrieve default status

            # Store mapping in session and proceed to processing
            request.session['imported_participants_data']['email_column'] = email_column
            request.session['imported_participants_data']['name_column'] = name_column # Store name column
            request.session['imported_participants_data']['phone_column'] = phone_column # Store phone column
            request.session['imported_participants_data']['participation_status_column'] = participation_status_column
            request.session['imported_participants_data']['default_participation_status_id'] = default_participation_status.id if default_participation_status else None
            request.session.modified = True # Ensure session is saved
            logger.debug(f"Session after mapping: {request.session.get('imported_participants_data')}") # Added debug logging
            
            return redirect('process_participants_import', event_id=event.id)
        else:
            logger.error(f"ParticipantMappingForm errors: {form.errors}") # Added logging
            messages.error(request, "Please correct the mapping errors.")
    else:
        form = ParticipantMappingForm(headers=headers, tenant=tenant)

    context = {
        'event': event,
        'form': form,
        'file_name': file_name,
        'headers': headers,
    }
    return render(request, 'core/event_participant_map_fields.html', context)

@role_required(['admin', 'campaign_manager', 'system_admin', 'campaign_admin'], permission='core.change_event')
def process_participants_import(request, event_id):
    logger.debug(f"Session at start of process_participants_import: {request.session.get('imported_participants_data')}") # Added debug logging
    selected_tenant_id = request.session.get("tenant_id")
    if not selected_tenant_id:
        messages.warning(request, "Please select a campaign first.")
        return redirect("index")

    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    event = get_object_or_404(Event, id=event_id, tenant=tenant)

    imported_data = request.session.get('imported_participants_data')
    if not imported_data or imported_data['event_id'] != event.id:
        messages.error(request, "No data found to process. Please upload and map a file first.")
        return redirect('event_detail', event_id=event.id)

    headers = imported_data['headers']
    data_rows = imported_data['data_rows']
    
    # Safely get column names from session, handle cases where they might be missing
    email_column = imported_data.get('email_column')
    name_column = imported_data.get('name_column') # Retrieve name column
    phone_column = imported_data.get('phone_column') # Retrieve phone column
    participation_status_column = imported_data.get('participation_status_column')
    default_participation_status_id = imported_data.get('default_participation_status_id')

    logger.debug(f"process_participants_import - name_column from session: {name_column}") # DEBUG LOGGING
    logger.debug(f"process_participants_import - phone_column from session: {phone_column}") # DEBUG LOGGING

    # Validate that required columns are present
    if not email_column:
        messages.error(request, "Email column mapping is missing. Please go back and map the fields.")
        return redirect('import_participants_map_fields', event_id=event.id)

    matched_count = 0
    unmatched_participants = []
    
    # Get all active participation statuses for the tenant
    participation_statuses_map = {status.name.lower(): status for status in ParticipationStatus.objects.filter(tenant=tenant, is_active=True)}
    default_status_obj = None
    if default_participation_status_id:
        default_status_obj = get_object_or_404(ParticipationStatus, id=default_participation_status_id, tenant=tenant)

    for row_index, row in enumerate(data_rows):
        row_dict = dict(zip(headers, row))
        email = row_dict.get(email_column)
        phone = row_dict.get(phone_column) if phone_column else None
        
        # DEBUG LOGGING: Log the value of the name column for each row
        if name_column:
            logger.debug(f"process_participants_import - Row {row_index}: name_column='{name_column}', name_value='{row_dict.get(name_column)}'")
        if phone_column:
            logger.debug(f"process_participants_import - Row {row_index}: phone_column='{phone_column}', phone_value='{phone}'")

        participation_status_name = row_dict.get(participation_status_column)
        
        if not email:
            logger.warning(f"Row {row_index+2}: Skipping due to missing email.")
            continue

        voter = Voter.objects.filter(tenant=tenant, email__iexact=email).first()
        
        if voter:
            # If phone is mapped and present, and not already associated with voter, update it
            if phone and voter.phone != phone and voter.secondary_phone != phone:
                voter.phone = phone
                voter.phone_type = 'cell'
                voter.save()

            # Match found, add as participant if not already existing
            status = participation_statuses_map.get(participation_status_name.lower()) if participation_status_name else default_status_obj
            if not status:
                status, _ = ParticipationStatus.objects.get_or_create(tenant=tenant, name='Unknown', is_active=True) # Fallback to unknown if no default and no match

            if not EventParticipation.objects.filter(event=event, voter=voter).exists():
                EventParticipation.objects.create(
                    event=event,
                    voter=voter,
                    participation_status=status
                )
                matched_count += 1
            else:
                logger.info(f"Voter {voter.email} is already a participant in event {event.name}. Skipping.")
        else:
            # No match found, add to unmatched list
            unmatched_participants.append({
                'row_data': row_dict,
                'original_row_index': row_index, # Keep original index for reference if needed
            })
    
    if unmatched_participants:
        # Store unmatched data in session for manual matching
        request.session['unmatched_participants_data'] = {
            'event_id': event.id,
            'unmatched_rows': unmatched_participants,
            'file_name': imported_data['file_name'],
            'email_column': email_column,
            'name_column': name_column, # Pass name column to unmatched data
            'phone_column': phone_column, # Pass phone column to unmatched data
            'participation_status_column': participation_status_column,
            'default_participation_status_id': default_participation_status_id,
        }
        messages.warning(request, f"{len(unmatched_participants)} participants could not be automatically matched. Please match them manually.")
        return redirect('match_participants', event_id=event.id)
    else:
        messages.success(request, f"Successfully imported {matched_count} participants for event {event.name}.")
        del request.session['imported_participants_data'] # Clean up session
    
    return redirect('event_detail', event_id=event.id)

@role_required(['admin', 'campaign_manager', 'system_admin', 'campaign_admin'], permission='core.change_event')
def match_participants(request, event_id):
    selected_tenant_id = request.session.get("tenant_id")
    if not selected_tenant_id:
        messages.warning(request, "Please select a campaign first.")
        return redirect("index")

    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    event = get_object_or_404(Event, id=event_id, tenant=tenant)

    unmatched_data = request.session.get('unmatched_participants_data')
    if not unmatched_data or unmatched_data['event_id'] != event.id:
        messages.error(request, "No unmatched participant data found. Please try importing again.")
        return redirect('event_detail', event_id=event.id)

    unmatched_rows = unmatched_data['unmatched_rows']
    file_name = unmatched_data['file_name']
    email_column = unmatched_data['email_column']
    name_column = unmatched_data['name_column'] # Retrieve name column
    phone_column = unmatched_data.get('phone_column') # Retrieve phone column
    participation_status_column = unmatched_data['participation_status_column']
    default_participation_status_id = unmatched_data.get('default_participation_status_id')

    logger.debug(f"match_participants context: email_column={email_column}, name_column={name_column}, phone_column={phone_column}, participation_status_column={participation_status_column}") # DEBUG LOGGING

    # DEBUG LOGGING: Log the value of the name column for each unmatched row
    for index, row_data in enumerate(unmatched_rows):
        name_value = row_data.get('row_data', {}).get(name_column)
        phone_value = row_data.get('row_data', {}).get(phone_column)
        logger.debug(f"match_participants - Unmatched row {index}: name_column='{name_column}', name_value='{name_value}', phone_column='{phone_column}', phone_value='{phone_value}'")


    if request.method == 'POST':
        matched_count = 0
        current_unmatched_rows = [] # To store rows that are still unmatched after POST
        
        default_status_obj = None
        if default_participation_status_id:
            default_status_obj = get_object_or_404(ParticipationStatus, id=default_participation_status_id, tenant=tenant)

        for index, row_data in enumerate(unmatched_rows):
            original_row_index = row_data['original_row_index']
            posted_voter_id = request.POST.get(f'voter_match_{original_row_index}')
            
            if posted_voter_id:
                # Manual match provided
                voter = get_object_or_404(Voter, id=posted_voter_id, tenant=tenant)
                
                # Update voter's email
                voter_email_from_file = row_data['row_data'].get(email_column)
                if voter_email_from_file and voter.email != voter_email_from_file:
                    voter.email = voter_email_from_file
                    voter.save()
                
                # Update voter's phone if mapped and different
                voter_phone_from_file = row_data['row_data'].get(phone_column)
                if voter_phone_from_file and voter.phone != voter_phone_from_file and voter.secondary_phone != voter_phone_from_file:
                    voter.phone = voter_phone_from_file
                    voter.phone_type = 'cell'
                    voter.save()

                # Add as participant if not already existing
                participation_status_name = row_data['row_data'].get(participation_status_column)
                status = None
                if participation_status_name:
                    status = ParticipationStatus.objects.filter(tenant=tenant, name__iexact=participation_status_name).first()
                
                if not status:
                    status = default_status_obj
                
                if not status:
                    status, _ = ParticipationStatus.objects.get_or_create(tenant=tenant, name='Unknown', is_active=True)

                if not EventParticipation.objects.filter(event=event, voter=voter).exists():
                    EventParticipation.objects.create(
                        event=event,
                        voter=voter,
                        participation_status=status
                    )
                    matched_count += 1
                else:
                    messages.warning(request, f"Voter {voter.email} is already a participant in event {event.name}. Skipping manual match for this voter.")
            else:
                # Still unmatched, keep for re-display
                current_unmatched_rows.append(row_data)
        
        if matched_count > 0:
            messages.success(request, f"Successfully matched {matched_count} participants.")
            
        if current_unmatched_rows:
            request.session['unmatched_participants_data']['unmatched_rows'] = current_unmatched_rows
            messages.warning(request, f"{len(current_unmatched_rows)} participants still need manual matching.")
            return redirect('match_participants', event_id=event.id)
        else:
            messages.success(request, "All participants have been matched.")
            del request.session['unmatched_participants_data'] # Clean up session
            del request.session['imported_participants_data'] # Also clean up this
        
        return redirect('event_detail', event_id=event.id)

    context = {
        'event': event,
        'unmatched_rows': unmatched_rows,
        'file_name': file_name,
        'email_column': email_column,
        'name_column': name_column, # Pass name column to template
        'phone_column': phone_column, # Pass phone column to template
        'participation_status_column': participation_status_column,
    }
    return render(request, 'core/event_participant_matching.html', context)


def volunteer_search_json(request):
    """
    JSON endpoint for volunteer search, used by autocomplete/search UI.
    """
    selected_tenant_id = request.session.get("tenant_id")
    if not selected_tenant_id:
        return JsonResponse({"results": []})
        
    query = request.GET.get("q", "").strip()
    if len(query) < 2:
        return JsonResponse({"results": []})
        
    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    volunteers = Volunteer.objects.filter(tenant=tenant)
    
    search_filter = Q(first_name__icontains=query) | Q(last_name__icontains=query) | Q(email__icontains=query)
    
    results = volunteers.filter(search_filter).order_by("last_name", "first_name")[:20]
    
    data = []
    for v in results:
        data.append({
            "id": v.id,
            "text": f"{v.first_name} {v.last_name} ({v.email})",
            "phone": v.phone
        })
        
    return JsonResponse({"results": data})

def event_add_volunteer(request, event_id):
    tenant_id = request.session.get("tenant_id")
    tenant = get_object_or_404(Tenant, id=tenant_id)
    event = get_object_or_404(Event, id=event_id, tenant=tenant)
    
    if request.method == 'POST':
        form = VolunteerEventAddForm(request.POST, tenant=tenant)
        if form.is_valid():
            assignment = form.save(commit=False)
            assignment.event = event
            if not VolunteerEvent.objects.filter(event=event, volunteer=assignment.volunteer).exists():
                assignment.save()
                messages.success(request, f"{assignment.volunteer} added as volunteer.")
            else:
                messages.warning(request, "Volunteer is already assigned to this event.")
        else:
            messages.error(request, "Error adding volunteer.")
            
    return redirect('event_detail', event_id=event.id)

def event_remove_volunteer(request, assignment_id):
    tenant_id = request.session.get("tenant_id")
    tenant = get_object_or_404(Tenant, id=tenant_id)
    assignment = get_object_or_404(VolunteerEvent, id=assignment_id, event__tenant=tenant)
    event_id = assignment.event.id
    volunteer_name = str(assignment.volunteer)
    assignment.delete()
    messages.success(request, f"{volunteer_name} removed from event volunteers.")
    return redirect('event_detail', event_id=event_id)

@role_required(['admin', 'campaign_manager', 'campaign_staff', 'system_admin', 'campaign_admin'], permission='core.view_volunteer')
def volunteer_bulk_send_sms(request):
    """
    Sends bulk SMS to selected volunteers using Twilio API.
    """
    if request.method != 'POST':
        return redirect('volunteer_list')
        
    selected_tenant_id = request.session.get("tenant_id")
    if not selected_tenant_id:
        messages.warning(request, "Please select a campaign first.")
        return redirect("index")
    
    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    settings = getattr(tenant, 'settings', None)
    if not settings:
        messages.error(request, "Campaign settings not found.")
        return redirect('volunteer_list')
    
    account_sid = settings.twilio_account_sid
    auth_token = settings.twilio_auth_token
    from_number = settings.twilio_from_number
    
    if not account_sid or not auth_token or not from_number:
        messages.error(request, "Twilio configuration is incomplete in Campaign Settings.")
        return redirect('volunteer_list')
        
    volunteer_ids = request.POST.getlist('selected_volunteers')
    message_body = request.POST.get('message_body')

    if not message_body:
        messages.error(request, "Message body cannot be empty.")
        return redirect('volunteer_list')
        
    volunteers = Volunteer.objects.filter(tenant=tenant, id__in=volunteer_ids).exclude(phone='')
    
    if not volunteers.exists():
        messages.warning(request, "No volunteers with a valid phone number were selected.")
        return redirect('volunteer_list')

    success_count = 0
    fail_count = 0
    
    auth_str = f"{account_sid}:{auth_token}"
    auth_header = base64.b64encode(auth_str.encode()).decode()
    url = f"https://api.twilio.com/2010-04-01/Accounts/{account_sid}/Messages.json"
    
    for volunteer in volunteers:
        # Format phone to E.164 (assume US +1)
        digits = re.sub(r'\\D', '', str(volunteer.phone))
        if len(digits) == 10:
            to_number = f"+1{digits}"
        elif len(digits) == 11 and digits.startswith('1'):
            to_number = f"+{digits}"
        else:
            # Skip invalid phone numbers
            fail_count += 1
            continue
            
        data_dict = {
            'To': to_number,
            'From': from_number,
            'Body': message_body
        }
        data = urllib.parse.urlencode(data_dict).encode()
        
        req = urllib.request.Request(url, data=data, method='POST')
        req.add_header("Authorization", f"Basic {auth_header}")
        
        try:
            with urllib.request.urlopen(req, timeout=10) as response:
                if response.status in [200, 201]:
                    success_count += 1
                else:
                    fail_count += 1
        except Exception as e:
            logger.error(f"Error sending SMS to volunteer {volunteer.phone}: {e}")
            fail_count += 1
            
    messages.success(request, f"Bulk SMS process completed: {success_count} successful, {fail_count} failed/skipped.")
    return redirect('volunteer_list')

@role_required(['admin', 'campaign_manager', 'campaign_staff', 'system_admin', 'campaign_admin'], permission='core.view_voter')
def door_visits(request):
    """
    Manage door knocking visits. Groups unvisited targeted voters by household.
    """
    selected_tenant_id = request.session.get("tenant_id")
    if not selected_tenant_id:
        messages.warning(request, "Please select a campaign first.")
        return redirect("index")

    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    
    city_filter = request.GET.get("city", "").strip()
    # Filters from GET parameters
    district_filter = request.GET.get('district', '').strip()
    neighborhood_filter = request.GET.get('neighborhood', '').strip()
    address_filter = request.GET.get('address', '').strip()

    # Initial queryset: unvisited targeted voters for this tenant
    voters = Voter.objects.filter(tenant=tenant, door_visit=False, is_targeted=True)

    if city_filter:
        voters = voters.filter(city__icontains=city_filter)
    # Apply filters if provided
    if district_filter:
        voters = voters.filter(district=district_filter)
    if neighborhood_filter:
        voters = voters.filter(neighborhood__icontains=neighborhood_filter)
    if address_filter:
        voters = voters.filter(Q(address__icontains=address_filter) | Q(address_street__icontains=address_filter))

    # Grouping by household (unique address)
    households_dict = {}
    for voter in voters:
        # Key for grouping is the unique address components
        key = (voter.address_street, voter.city, voter.state, voter.zip_code)
        if key not in households_dict:
            # Parse street name and number for sorting
            street_number = ""
            street_name = voter.address_street
            match = re.match(r'^(\d+)\s+(.*)$', voter.address_street)
            if match:
                street_number = match.group(1)
                street_name = match.group(2)
            
            try:
                street_number_sort = int(street_number)
            except ValueError:
                street_number_sort = 0

            households_dict[key] = {
                'address_street': voter.address_street,
                'city': voter.city,
                'state': voter.state,
                'zip_code': voter.zip_code,
                'neighborhood': voter.neighborhood,
                'district': voter.district,
                'latitude': float(voter.latitude) if voter.latitude else None,
                'longitude': float(voter.longitude) if voter.longitude else None,
                'street_name_sort': street_name.lower(),
                'street_number_sort': street_number_sort,
                'target_voters': [],
                'voters_json': []
            }
        households_dict[key]['target_voters'].append(voter)
        households_dict[key]['voters_json'].append({'id': voter.id, 'name': f"{voter.first_name} {voter.last_name}"})

    households_list = list(households_dict.values())
    for h in households_list:
        h['voters_json_str'] = json.dumps(h['voters_json'])

    households_list.sort(key=lambda x: (
        (x['neighborhood'] or '').lower(),
        x['street_name_sort'],
        x['street_number_sort']
    ))

    # Prepare data for Google Map (all filtered households with coordinates)
    map_data = [
        {
            'lat': h['latitude'],
            'lng': h['longitude'],
            'address_street': h['address_street'], 'city': h['city'], 'state': h['state'], 'zip_code': h['zip_code'], 'address': f"{h['address_street']}, {h['city']}, {h['state']}",
            'voters': ", ".join([f"{v.first_name} {v.last_name}" for v in h['target_voters']])
        }
        for h in households_list if h['latitude'] and h['longitude']
    ]

    paginator = Paginator(households_list, 50)
    page_number = request.GET.get('page')
    households_page = paginator.get_page(page_number)

    context = {
        'selected_tenant': tenant,
        'households': households_page,
        'district_filter': district_filter,
        'neighborhood_filter': neighborhood_filter,
        'address_filter': address_filter,
        "city_filter": city_filter,
        'map_data_json': json.dumps(map_data),
        'GOOGLE_MAPS_API_KEY': getattr(settings, 'GOOGLE_MAPS_API_KEY', ''),
        'visit_form': DoorVisitLogForm(),
    }
    return render(request, 'core/door_visits.html', context)

@role_required(['admin', 'campaign_manager', 'campaign_staff', 'system_admin', 'campaign_admin'], permission='core.view_voter')
def log_door_visit(request):
    """
    Mark all targeted voters at a specific address as visited, update their flags,
    and create interaction records.
    Can also render a standalone page for logging a visit.
    """
    selected_tenant_id = request.session.get("tenant_id")
    if not selected_tenant_id:
        return redirect("index")
    
    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    campaign_settings, _ = CampaignSettings.objects.get_or_create(tenant=tenant)
    
    # Capture query string for redirecting back with filters
    next_qs = request.POST.get("next_query_string", request.GET.get("next_query_string", ""))
    source = request.POST.get("source", request.GET.get("source", ""))
    
    redirect_url = reverse("door_visits")
    
    # Build redirect URL
    redirect_params = []
    if next_qs:
        redirect_params.append(next_qs)
    if source == "map":
        redirect_params.append("open_map=1")
    
    if redirect_params:
        redirect_url += "?" + "&".join(redirect_params)
    
    # Get address components from POST or GET
    address_street = request.POST.get("address_street", request.GET.get("address_street"))
    city = request.POST.get("city", request.GET.get("city"))
    state = request.POST.get("state", request.GET.get("state"))
    zip_code = request.POST.get("zip_code", request.GET.get("zip_code"))

    if not address_street:
        messages.warning(request, "No address provided.")
        return redirect(redirect_url)

    # Find targeted voters at this exact address
    voters = Voter.objects.filter(
        tenant=tenant,
        address_street=address_street,
        city=city,
        state=state,
        zip_code=zip_code,
        is_targeted=True
    )

    if not voters.exists() and request.method == "POST":
        messages.warning(request, f"No targeted voters found at {address_street}.")
        return redirect(redirect_url)
    
    voter_choices = [(v.id, f"{v.first_name} {v.last_name}") for v in voters]
    
    # Get the volunteer linked to the current user
    volunteer = Volunteer.objects.filter(user=request.user, tenant=tenant).first()
    
    if request.method == "POST":
        form = DoorVisitLogForm(request.POST, voter_choices=voter_choices)
        if form.is_valid():
            outcome = form.cleaned_data["outcome"]
            notes = form.cleaned_data["notes"]
            wants_yard_sign = form.cleaned_data["wants_yard_sign"]
            candidate_support = form.cleaned_data["candidate_support"]
            follow_up = form.cleaned_data["follow_up"]
            follow_up_voter_id = form.cleaned_data.get("follow_up_voter")
            call_notes = form.cleaned_data["call_notes"]
            
            # Determine date/time in campaign timezone
            campaign_tz_name = campaign_settings.timezone or "America/Chicago"
            try:
                tz = zoneinfo.ZoneInfo(campaign_tz_name)
            except:
                tz = zoneinfo.ZoneInfo("America/Chicago")
            
            interaction_date = timezone.now().astimezone(tz)
            
            # Get or create InteractionType
            interaction_type, _ = InteractionType.objects.get_or_create(tenant=tenant, name="Door Visit")
            
            # Get default caller for follow-ups
            default_caller = None
            if follow_up:
                default_caller = Volunteer.objects.filter(tenant=tenant, is_default_caller=True).first()

            for voter in voters:
                # 1) Update voter flags
                voter.door_visit = True
                
                # 2) If "Wants a Yard Sign" checkbox is selected
                if wants_yard_sign:
                    voter.yard_sign = "wants"
                
                # 3) Update support status if Supporting or Not Supporting
                if candidate_support in ["supporting", "not_supporting"]:
                    voter.candidate_support = candidate_support
                
                voter.save()
                
                # 4) Create interaction
                Interaction.objects.create(
                    voter=voter,
                    volunteer=volunteer,
                    type=interaction_type,
                    date=interaction_date,
                    description=outcome,
                    notes=notes
                )

                # 5) Create ScheduledCall if follow_up is checked and this is the selected voter
                if follow_up and follow_up_voter_id and str(voter.id) == follow_up_voter_id:
                    ScheduledCall.objects.create(
                        tenant=tenant,
                        voter=voter,
                        volunteer=default_caller,
                        comments=call_notes,
                        status="pending"
                    )
            
            if follow_up:
                messages.success(request, f"Door visit logged and follow-up call scheduled for {address_street}.")
            else:
                messages.success(request, f"Door visit logged for {address_street}.")
            return redirect(redirect_url)
        else:
            messages.error(request, "There was an error in the visit log form.")
            return redirect(redirect_url)
    else:
        # GET request: render standalone page
        form = DoorVisitLogForm(voter_choices=voter_choices)
        context = {
            'selected_tenant': tenant,
            'visit_form': form,
            'address_street': address_street,
            'city': city,
            'state': state,
            'zip_code': zip_code,
            'voters': voters,
            'next_query_string': next_qs,
            'source': source,
            'redirect_url': redirect_url,
        }
        return render(request, 'core/log_door_visit.html', context)

def door_visit_history(request):
    """
    Shows a distinct list of Door visit interactions for addresses.
    """
    selected_tenant_id = request.session.get("tenant_id")
    if not selected_tenant_id:
        messages.warning(request, "Please select a campaign first.")
        return redirect("index")
    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    
    # Date filter
    start_date = request.GET.get("start_date")
    end_date = request.GET.get("end_date")
    
    # Get all "Door Visit" interactions for this tenant
    interactions = Interaction.objects.filter(
        voter__tenant=tenant,
        type__name="Door Visit"
    ).select_related("voter", "volunteer")

    if start_date or end_date:
        try:
            if start_date:
                d = parse_date(start_date)
                if d:
                    start_dt = timezone.make_aware(datetime.combine(d, time.min))
                    interactions = interactions.filter(date__gte=start_dt)
            if end_date:
                d = parse_date(end_date)
                if d:
                    # Use lt with next day to capture everything on the end_date
                    end_dt = timezone.make_aware(datetime.combine(d + timedelta(days=1), time.min))
                    interactions = interactions.filter(date__lt=end_dt)
        except Exception as e:
            logger.error(f"Error filtering door visit history by date: {e}")

    # Summary of counts per volunteer
    # Grouping by household (unique address)
    visited_households = {}
    volunteer_counts = {}

    for interaction in interactions.order_by("-date"):
        v = interaction.voter
        addr = v.address.strip() if v.address else f"{v.address_street}, {v.city}, {v.state} {v.zip_code}".strip(", ")
        if not addr:
            continue
            
        key = addr.lower()
        
        if key not in visited_households:
            # Calculate volunteer summary - only once per household
            v_obj = interaction.volunteer
            v_name = f"{v_obj.first_name} {v_obj.last_name}".strip() or v_obj.email if v_obj else "N/A"
            volunteer_counts[v_name] = volunteer_counts.get(v_name, 0) + 1

            # Parse street name and number for sorting
            street_number = ""
            street_name = v.address_street or ""
            match = re.match(r'^(\d+)\s+(.*)$', street_name)
            if match:
                street_number = match.group(1)
                street_name = match.group(2)
            
            try:
                street_number_sort = int(street_number)
            except ValueError:
                street_number_sort = 0

            visited_households[key] = {
                'address_display': addr,
                'address_street': v.address_street,
                'city': v.city,
                'state': v.state,
                'zip_code': v.zip_code,
                'neighborhood': v.neighborhood,
                'district': v.district,
                'latitude': float(v.latitude) if v.latitude else None,
                'longitude': float(v.longitude) if v.longitude else None,
                'street_name_sort': street_name.lower(),
                'street_number_sort': street_number_sort,
                'last_visit_date': interaction.date,
                'target_voters': [],
                'voters_json': []
            }
        
        visited_households[key]["voters_json"].append({'id': v.id, 'name': f"{v.first_name} {v.last_name}"})
        visited_households[key]['target_voters'].append(v)

    # Sort volunteer counts by total (descending)
    sorted_volunteer_counts = sorted(volunteer_counts.items(), key=lambda x: x[1], reverse=True)

    history_list = list(visited_households.values())
    history_list.sort(key=lambda x: x["last_visit_date"], reverse=True)

    paginator = Paginator(history_list, 50)
    page_number = request.GET.get("page")
    history_page = paginator.get_page(page_number)

    context = {
        "selected_tenant": tenant,
        "history": history_page,
        "start_date": start_date, "end_date": end_date,
        "volunteer_counts": sorted_volunteer_counts,
    }
    return render(request, "core/door_visit_history.html", context)

@role_required(['admin', 'campaign_manager', 'campaign_staff', 'system_admin', 'campaign_admin'], permission='core.add_scheduledcall')
def schedule_call(request, voter_id):
    selected_tenant_id = request.session.get("tenant_id")
    if not selected_tenant_id:
        messages.warning(request, "Please select a campaign first.")
        return redirect("index")

    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    voter = get_object_or_404(Voter, id=voter_id, tenant=tenant)
    
    if request.method == 'POST':
        form = ScheduledCallForm(request.POST, tenant=tenant)
        if form.is_valid():
            call = form.save(commit=False)
            call.tenant = tenant
            call.voter = voter
            call.save()
            messages.success(request, f"Call for {voter} added to queue.")
        else:
            messages.error(request, "Error scheduling call.")
            
    referer = request.META.get('HTTP_REFERER')
    if referer:
        return redirect(referer)
    return redirect('voter_detail', voter_id=voter.id)

@role_required(['admin', 'campaign_manager', 'campaign_staff', 'system_admin', 'campaign_admin'], permission='core.add_scheduledcall')
def bulk_schedule_calls(request):
    if request.method != 'POST':
        return redirect('voter_advanced_search')
        
    selected_tenant_id = request.session.get("tenant_id")
    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    
    voter_ids = request.POST.getlist('selected_voters')
    volunteer_id = request.POST.get('volunteer')
    comments = request.POST.get('comments', '')
    
    volunteer = None
    if volunteer_id:
        volunteer = get_object_or_404(Volunteer, id=volunteer_id, tenant=tenant)
    else:
        # Fallback to default caller if not specified in POST but available
        volunteer = Volunteer.objects.filter(tenant=tenant, is_default_caller=True).first()
        
    voters = Voter.objects.filter(tenant=tenant, id__in=voter_ids)
    
    count = 0
    for voter in voters:
        ScheduledCall.objects.create(
            tenant=tenant,
            voter=voter,
            volunteer=volunteer,
            comments=comments
        )
        count += 1
        
    messages.success(request, f"{count} calls added to queue.")
    return redirect(request.META.get('HTTP_REFERER', 'voter_advanced_search'))

@role_required(['admin', 'campaign_manager', 'campaign_staff', 'system_admin', 'campaign_admin'], permission='core.view_scheduledcall')
def call_queue(request):
    selected_tenant_id = request.session.get("tenant_id")
    if not selected_tenant_id:
        messages.warning(request, "Please select a campaign first.")
        return redirect("index")

    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    calls = ScheduledCall.objects.filter(tenant=tenant, status='pending').order_by('created_at')
    
    paginator = Paginator(calls, 50)
    page_number = request.GET.get('page')
    calls_page = paginator.get_page(page_number)

    context = {
        'selected_tenant': tenant,
        'calls': calls_page,
    }
    return render(request, 'core/call_queue.html', context)

@role_required(['admin', 'campaign_manager', 'campaign_staff', 'system_admin', 'campaign_admin'], permission='core.change_scheduledcall')
def complete_call(request, call_id):
    selected_tenant_id = request.session.get("tenant_id")
    if not selected_tenant_id:
        messages.warning(request, "Please select a campaign first.")
        return redirect("index")

    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    call = get_object_or_404(ScheduledCall, id=call_id, tenant=tenant)

    if request.method == 'POST':
        # Get notes from post data taken during the call
        call_notes = request.POST.get('call_notes', '')

        # Create interaction for the completed call
        interaction_type, _ = InteractionType.objects.get_or_create(tenant=tenant, name="Phone Call")

        # Determine date/time in campaign timezone
        campaign_settings, _ = CampaignSettings.objects.get_or_create(tenant=tenant)
        campaign_tz_name = campaign_settings.timezone or 'America/Chicago'
        try:
            tz = zoneinfo.ZoneInfo(campaign_tz_name)
        except:
            tz = zoneinfo.ZoneInfo('America/Chicago')

        interaction_date = timezone.now().astimezone(tz);

        Interaction.objects.create(
            voter=call.voter,
            volunteer=call.volunteer,
            type=interaction_type,
            date=interaction_date,
            description="Called Voter",
            notes=call_notes
        )

        call.status = 'completed';
        call.save()
        messages.success(request, f"Call for {call.voter} marked as completed and interaction logged.")

    return redirect('call_queue')

@role_required(['admin', 'campaign_manager', 'campaign_staff', 'system_admin', 'campaign_admin'], permission='core.delete_scheduledcall')
def delete_call(request, call_id):
    selected_tenant_id = request.session.get("tenant_id")
    if not selected_tenant_id:
        messages.warning(request, "Please select a campaign first.")
        return redirect("index")

    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    call = get_object_or_404(ScheduledCall, id=call_id, tenant=tenant)
    
    if request.method == 'POST':
        call.delete()
        messages.success(request, "Call removed from queue.")
        
    return redirect('call_queue')

@login_required
def profile(request):
    try:
        volunteer = request.user.volunteer_profile
    except:
        volunteer = None

    if request.method == 'POST':
        u_form = UserUpdateForm(request.POST, instance=request.user)


@role_required(['admin', 'campaign_manager', 'campaign_staff', 'system_admin', 'campaign_admin'], permission='core.view_voter')
def neighborhood_counts(request):
    """
    Shows household counts by neighborhood after applying filters from door visits.
    """
    selected_tenant_id = request.session.get("tenant_id")
    if not selected_tenant_id:
        messages.warning(request, "Please select a campaign first.")
        return redirect("index")

    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    
    city_filter = request.GET.get("city", "").strip()
    district_filter = request.GET.get('district', '').strip()
    neighborhood_filter = request.GET.get('neighborhood', '').strip()
    address_filter = request.GET.get('address', '').strip()

    voters = Voter.objects.filter(tenant=tenant, door_visit=False, is_targeted=True)

    if city_filter:
        voters = voters.filter(city__icontains=city_filter)
    if district_filter:
        voters = voters.filter(district=district_filter)
    if neighborhood_filter:
        voters = voters.filter(neighborhood__icontains=neighborhood_filter)
    if address_filter:
        voters = voters.filter(Q(address__icontains=address_filter) | Q(address_street__icontains=address_filter))

    household_qs = voters.values('neighborhood', 'address_street', 'city', 'state', 'zip_code').distinct()
    
    neighborhood_counts_dict = {}
    for h in household_qs:
        nb = h['neighborhood']
        neighborhood_counts_dict[nb] = neighborhood_counts_dict.get(nb, 0) + 1
            
    neighborhood_list = [
        {'neighborhood': nb, 'display_name': nb or "Unknown", 'count': count}
        for nb, count in neighborhood_counts_dict.items()
    ]
    
    neighborhood_list.sort(key=lambda x: x['count'], reverse=True)

    context = {
        'selected_tenant': tenant,
        'neighborhoods': neighborhood_list,
        'city_filter': city_filter,
        'district_filter': district_filter,
        'neighborhood_filter': neighborhood_filter,
        'address_filter': address_filter,
    }
    return render(request, 'core/neighborhood_counts.html', context)

@role_required(['admin', 'campaign_manager', 'campaign_staff', 'system_admin', 'campaign_admin'], permission='core.view_voter')
def yard_sign_voters(request):
    """
    Manage yard sign requests. Groups voters who want a yard sign by household.
    """
    selected_tenant_id = request.session.get("tenant_id")
    if not selected_tenant_id:
        messages.warning(request, "Please select a campaign first.")
        return redirect("index")

    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    
    city_filter = request.GET.get("city", "").strip()
    district_filter = request.GET.get('district', '').strip()
    neighborhood_filter = request.GET.get('neighborhood', '').strip()
    address_filter = request.GET.get('address', '').strip()

    # Initial queryset: voters who want a yard sign for this tenant
    voters = Voter.objects.filter(tenant=tenant, yard_sign='wants')

    if city_filter:
        voters = voters.filter(city__icontains=city_filter)
    if district_filter:
        voters = voters.filter(district=district_filter)
    if neighborhood_filter:
        voters = voters.filter(neighborhood__icontains=neighborhood_filter)
    if address_filter:
        voters = voters.filter(Q(address__icontains=address_filter) | Q(address_street__icontains=address_filter))

    # Grouping by household (unique address)
    households_dict = {}
    for voter in voters:
        key = (voter.address_street, voter.city, voter.state, voter.zip_code)
        if key not in households_dict:
            street_number = ""
            street_name = voter.address_street or ""
            match = re.match(r'^(\d+)\s+(.*)$', voter.address_street)
            if match:
                street_number = match.group(1)
                street_name = match.group(2)
            
            try:
                street_number_sort = int(street_number)
            except ValueError:
                street_number_sort = 0

            households_dict[key] = {
                'address_street': voter.address_street,
                'city': voter.city,
                'state': voter.state,
                'zip_code': voter.zip_code,
                'neighborhood': voter.neighborhood,
                'district': voter.district,
                'latitude': float(voter.latitude) if voter.latitude else None,
                'longitude': float(voter.longitude) if voter.longitude else None,
                'street_name_sort': street_name.lower(),
                'street_number_sort': street_number_sort,
                'voters_who_want_sign': [],
            }
        households_dict[key]['voters_who_want_sign'].append(voter)

    households_list = list(households_dict.values())
    households_list.sort(key=lambda x: (
        (x['neighborhood'] or '').lower(),
        x['street_name_sort'],
        x['street_number_sort']
    ))

    # Prepare data for Google Map
    map_data = [
        {
            'lat': h['latitude'],
            'lng': h['longitude'],
            'address': f"{h['address_street']}, {h['city']}, {h['state']}",
            'voters': ", ".join([f"{v.first_name} {v.last_name}" for v in h['voters_who_want_sign']])
        }
        for h in households_list if h['latitude'] and h['longitude']
    ]

    paginator = Paginator(households_list, 50)
    page_number = request.GET.get('page')
    households_page = paginator.get_page(page_number)

    context = {
        'selected_tenant': tenant,
        'households': households_page,
        'district_filter': district_filter,
        'neighborhood_filter': neighborhood_filter,
        'address_filter': address_filter,
        "city_filter": city_filter,
        'map_data_json': json.dumps(map_data),
        'GOOGLE_MAPS_API_KEY': getattr(settings, 'GOOGLE_MAPS_API_KEY', ''),
    }
    return render(request, 'core/yard_sign_voters.html', context)


from django.core.mail import get_connection, EmailMessage

def get_tenant_email_connection(campaign_settings):
    if not campaign_settings.smtp_host:
        return None
    return get_connection(
        host=campaign_settings.smtp_host,
        port=campaign_settings.smtp_port,
        username=campaign_settings.smtp_username,
        password=campaign_settings.smtp_password,
        use_tls=campaign_settings.smtp_use_tls,
        use_ssl=campaign_settings.smtp_use_ssl,
    )

@role_required(['admin', 'campaign_manager', 'campaign_staff', 'system_admin', 'campaign_admin'])
def volunteer_bulk_send_email(request):
    selected_tenant_id = request.session.get("tenant_id")
    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    campaign_settings = CampaignSettings.objects.get(tenant=tenant)

    if request.method == 'POST':
        volunteer_ids = request.POST.getlist('selected_volunteers')
        subject = request.POST.get('subject')
        body = request.POST.get('body')

        volunteers = Volunteer.objects.filter(id__in=volunteer_ids, tenant=tenant).exclude(email='')
        if not volunteers.exists():
            messages.warning(request, "No volunteers with email addresses selected.")
            return redirect('volunteer_list')

        connection = get_tenant_email_connection(campaign_settings)
        if not connection:
            messages.error(request, "SMTP settings are not configured. Please check Campaign Settings.")
            return redirect('volunteer_list')

        from_email = campaign_settings.email_from_address or settings.DEFAULT_FROM_EMAIL
        
        sent_count = 0
        error_count = 0
        
        for volunteer in volunteers:
            try:
                email = EmailMessage(
                    subject,
                    body,
                    from_email,
                    [volunteer.email],
                    connection=connection,
                )
                email.send()
                sent_count += 1
            except Exception as e:
                logger.error(f"Error sending bulk email to volunteer {volunteer.email}: {e}")
                error_count += 1

        if sent_count > 0:
            messages.success(request, f"Successfully sent {sent_count} emails.")
        if error_count > 0:
            messages.error(request, f"Failed to send {error_count} emails.")
            
    return redirect('volunteer_list')

@role_required(['admin', 'campaign_manager', 'campaign_staff', 'system_admin', 'campaign_admin'])
def voter_bulk_send_email(request):
    selected_tenant_id = request.session.get("tenant_id")
    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    campaign_settings = CampaignSettings.objects.get(tenant=tenant)

    if request.method == 'POST':
        voter_ids = request.POST.getlist('selected_voters')
        subject = request.POST.get('subject')
        body = request.POST.get('body')

        voters = Voter.objects.filter(id__in=voter_ids, tenant=tenant).exclude(email='')
        if not voters.exists():
            messages.warning(request, "No voters with email addresses selected.")
            return redirect('voter_advanced_search')

        connection = get_tenant_email_connection(campaign_settings)
        if not connection:
            messages.error(request, "SMTP settings are not configured. Please check Campaign Settings.")
            return redirect('voter_advanced_search')

        from_email = campaign_settings.email_from_address or settings.DEFAULT_FROM_EMAIL
        email_type, _ = InteractionType.objects.get_or_create(tenant=tenant, name='Email')
        
        sent_count = 0
        error_count = 0
        
        for voter in voters:
            try:
                email = EmailMessage(
                    subject,
                    body,
                    from_email,
                    [voter.email],
                    connection=connection,
                )
                email.send()
                sent_count += 1
                
                # Log interaction
                Interaction.objects.create(
                    voter=voter,
                    type=email_type,
                    date=timezone.now(),
                    description=subject,
                    notes=body
                )
            except Exception as e:
                logger.error(f"Error sending bulk email to voter {voter.email}: {e}")
                error_count += 1

        if sent_count > 0:
            messages.success(request, f"Successfully sent {sent_count} emails and logged interactions.")
        if error_count > 0:
            messages.error(request, f"Failed to send {error_count} emails.")
            
    return redirect('voter_advanced_search')
