def door_visits(request):
    """
    Manage door knocking visits. Groups unvisited targeted voters by household.
    Optimized to handle large datasets more efficiently.
    """
    selected_tenant_id = request.session.get("tenant_id")
    if not selected_tenant_id:
        messages.warning(request, "Please select a campaign first.")
        return redirect("index")

    tenant = get_object_or_404(Tenant, id=selected_tenant_id)
    
    city_filter = request.GET.get("city", "").strip()
    district_filter = request.GET.get('district', '').strip()
    neighborhood_filter = request.GET.get('neighborhood', '').strip()
    address_filter = request.GET.get('address', '').strip()

    voters = Voter.objects.filter(tenant=tenant, is_inactive=False, door_visit=False, target_door_visit=True)

    if city_filter:
        voters = voters.filter(city__icontains=city_filter)
    if district_filter:
        voters = voters.filter(district=district_filter)
    if neighborhood_filter:
        voters = voters.filter(neighborhood__icontains=neighborhood_filter)
    if address_filter:
        voters = voters.filter(Q(address__icontains=address_filter) | Q(address_street__icontains=address_filter))

    # Optimization: Use iterator and lighter data structure
    # We only fetch needed fields. Compound index helps here.
    voters_iterator = voters.values(
        'id', 'first_name', 'last_name', 'address_street', 'city', 'state', 
        'zip_code', 'neighborhood', 'district', 'latitude', 'longitude', 'phone'
    ).iterator(chunk_size=2000)

    households_dict = {}
    for v in voters_iterator:
        street = (v['address_street'] or "").strip()
        city = (v['city'] or "").strip()
        state = (v['state'] or "").strip()
        zip_code = (v['zip_code'] or "").strip()
        
        key = (street.lower(), city.lower(), state.lower(), zip_code.lower())
        
        if key not in households_dict:
            street_number = ""
            street_name = street
            match_street = re.match(r'^(\d+)\s+(.*)$', street)
            if match_street:
                street_number = match_street.group(1)
                street_name = match_street.group(2)
            
            try:
                street_number_sort = int(street_number)
            except (ValueError, TypeError):
                street_number_sort = 0

            households_dict[key] = {
                'address_street': street,
                'city': city,
                'state': state,
                'zip_code': zip_code,
                'neighborhood': (v['neighborhood'] or "").strip(),
                'district': (v['district'] or "").strip(),
                'latitude': float(v['latitude']) if v['latitude'] else None,
                'longitude': float(v['longitude']) if v['longitude'] else None,
                'street_name_sort': street_name.lower(),
                'street_number_sort': street_number_sort,
                'target_voters': [],
                'voters_json': []
            }
        else:
            if not households_dict[key]['neighborhood'] and v['neighborhood']:
                households_dict[key]['neighborhood'] = v['neighborhood'].strip()
            if not households_dict[key]['district'] and v['district']:
                households_dict[key]['district'] = v['district'].strip()
                
        households_dict[key]['target_voters'].append(v)
        phone_display = format_phone_number(v['phone']) if v['phone'] else "<No Phone>"
        households_dict[key]['voters_json'].append({'id': v['id'], 'name': f"{v['first_name']} {v['last_name']} - {phone_display}" })

    households_list = list(households_dict.values())
    del households_dict # Free memory
    
    households_list.sort(key=lambda x: (
        not bool(x['neighborhood']), 
        (x['neighborhood'] or '').lower(),
        x['street_name_sort'],
        x['street_number_sort']
    ))

    # Optimization: Map data should only be for visible results or limited
    # We still keep the limit of 3000 for map
    map_data = []
    # Always try to show up to 3000 markers instead of showing 0 if count > 3000
    for h in households_list[:3000]:
        if h['latitude'] and h['longitude']:
            map_data.append({
                'lat': h['latitude'],
                'lng': h['longitude'],
                'address_street': h['address_street'],
                'city': h['city'],
                'state': h['state'],
                'zip_code': h['zip_code'],
                'address': f"{h['address_street']}, {h['city']}",
                'voters': ", ".join([f"{v['first_name']} {v['last_name']}" for v in h['target_voters']])
            })

    for h in households_list:
        h['voters_json_str'] = json.dumps(h['voters_json'])

    paginator = Paginator(households_list, 50)
    page_number = request.GET.get('page')
    households_page = paginator.get_page(page_number)

    context = {
        'selected_tenant': tenant,
        'households': households_page,
        'district_filter': district_filter,
        'neighborhood_filter': neighborhood_filter,
        'address_filter': address_filter,
        'city_filter': city_filter,
        'map_data_json': json.dumps(map_data),
        'map_limit_reached': len(households_list) > 3000,
        'GOOGLE_MAPS_API_KEY': getattr(settings, 'GOOGLE_MAPS_API_KEY', ''),
    }
    return render(request, 'core/door_visits.html', context)