diff --git a/core/__pycache__/admin.cpython-311.pyc b/core/__pycache__/admin.cpython-311.pyc index ac91214..b188522 100644 Binary files a/core/__pycache__/admin.cpython-311.pyc and b/core/__pycache__/admin.cpython-311.pyc differ diff --git a/core/__pycache__/forms.cpython-311.pyc b/core/__pycache__/forms.cpython-311.pyc index 92d31a3..422a457 100644 Binary files a/core/__pycache__/forms.cpython-311.pyc and b/core/__pycache__/forms.cpython-311.pyc differ diff --git a/core/__pycache__/models.cpython-311.pyc b/core/__pycache__/models.cpython-311.pyc index 7fdfe9b..89c49f2 100644 Binary files a/core/__pycache__/models.cpython-311.pyc and b/core/__pycache__/models.cpython-311.pyc differ diff --git a/core/__pycache__/views.cpython-311.pyc b/core/__pycache__/views.cpython-311.pyc index d9e576b..ac9719a 100644 Binary files a/core/__pycache__/views.cpython-311.pyc and b/core/__pycache__/views.cpython-311.pyc differ diff --git a/core/admin.py b/core/admin.py index 54572ac..4cf45bb 100644 --- a/core/admin.py +++ b/core/admin.py @@ -1,3 +1,6 @@ +from decimal import Decimal +from datetime import datetime, date +from django.db import transaction from django.http import HttpResponse from django.utils.safestring import mark_safe import csv @@ -10,6 +13,7 @@ from django.urls import path, reverse from django.shortcuts import render, redirect from django.template.response import TemplateResponse from .models import ( + format_phone_number, Tenant, TenantUserRole, InteractionType, DonationMethod, ElectionType, EventType, Voter, VotingRecord, Event, EventParticipation, Donation, Interaction, VoterLikelihood, CampaignSettings, Interest, Volunteer, VolunteerEvent, ParticipationStatus @@ -227,51 +231,58 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin): def import_voters(self, request): if request.method == "POST": if "_preview" in request.POST: - file_path = request.POST.get('file_path') - tenant_id = request.POST.get('tenant') + file_path = request.POST.get("file_path") + tenant_id = request.POST.get("tenant") tenant = Tenant.objects.get(id=tenant_id) mapping = {} for field_name, _ in VOTER_MAPPABLE_FIELDS: - mapping[field_name] = request.POST.get(f'map_{field_name}') + mapping[field_name] = request.POST.get(f"map_{field_name}") try: - with open(file_path, 'r', encoding='UTF-8') as f: + with open(file_path, "r", encoding="UTF-8") as f: + # Optimization: Fast count and partial preview + total_count = sum(1 for line in f) - 1 + f.seek(0) reader = csv.DictReader(f) - total_count = 0 - create_count = 0 - update_count = 0 - preview_data = [] - for row in reader: - total_count += 1 - voter_id = row.get(mapping.get('voter_id')) - exists = Voter.objects.filter(tenant=tenant, voter_id=voter_id).exists() - if exists: - update_count += 1 - action = 'update' + preview_rows = [] + voter_ids_for_preview = [] + for i, row in enumerate(reader): + if i < 10: + preview_rows.append(row) + v_id = row.get(mapping.get("voter_id")) + if v_id: + voter_ids_for_preview.append(v_id) else: - create_count += 1 - action = 'create' - - if len(preview_data) < 10: - preview_data.append({ - 'action': action, - 'identifier': voter_id, - 'details': f"{row.get(mapping.get('first_name', '')) or ''} {row.get(mapping.get('last_name', '')) or ''}".strip() - }) + break + + existing_preview_ids = set(Voter.objects.filter(tenant=tenant, voter_id__in=voter_ids_for_preview).values_list("voter_id", flat=True)) + + preview_data = [] + for row in preview_rows: + v_id = row.get(mapping.get("voter_id")) + action = "update" if v_id in existing_preview_ids else "create" + preview_data.append({ + "action": action, + "identifier": v_id, + "details": f"{row.get(mapping.get('first_name', '')) or ''} {row.get(mapping.get('last_name', '')) or ''}".strip() + }) + update_count = "N/A" + create_count = "N/A" + context = self.admin_site.each_context(request) context.update({ - 'title': "Import Preview", - 'total_count': total_count, - 'create_count': create_count, - 'update_count': update_count, - 'preview_data': preview_data, - 'mapping': mapping, - 'file_path': file_path, - 'tenant_id': tenant_id, - 'action_url': request.path, - 'opts': self.model._meta, + "title": "Import Preview", + "total_count": total_count, + "create_count": create_count, + "update_count": update_count, + "preview_data": preview_data, + "mapping": mapping, + "file_path": file_path, + "tenant_id": tenant_id, + "action_url": request.path, + "opts": self.model._meta, }) return render(request, "admin/import_preview.html", context) except Exception as e: @@ -279,133 +290,191 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin): return redirect("..") elif "_import" in request.POST: - file_path = request.POST.get('file_path') - tenant_id = request.POST.get('tenant') + file_path = request.POST.get("file_path") + tenant_id = request.POST.get("tenant") tenant = Tenant.objects.get(id=tenant_id) - mapping = {} - for field_name, _ in VOTER_MAPPABLE_FIELDS: - mapping[field_name] = request.POST.get(f'map_{field_name}') + mapping = {k: request.POST.get(f"map_{k}") for k, _ in VOTER_MAPPABLE_FIELDS if request.POST.get(f"map_{k}")} try: - with open(file_path, 'r', encoding='UTF-8') as f: - reader = csv.DictReader(f) - count = 0 - errors = 0 - failed_rows = [] - for row in reader: - try: - voter_data = {} - voter_id = '' - for field_name, csv_col in mapping.items(): - if csv_col: - val = row.get(csv_col) - if val is not None and str(val).strip() != '': - if field_name == 'voter_id': - voter_id = val - continue - - if field_name == 'is_targeted': - val = str(val).lower() in ['true', '1', 'yes'] - voter_data[field_name] = val - - if 'candidate_support' in voter_data: - if voter_data['candidate_support'] not in dict(Voter.SUPPORT_CHOICES): - voter_data['candidate_support'] = 'unknown' - if 'yard_sign' in voter_data: - if voter_data['yard_sign'] not in dict(Voter.YARD_SIGN_CHOICES): - voter_data['yard_sign'] = 'none' - if 'window_sticker' in voter_data: - if voter_data['window_sticker'] not in dict(Voter.WINDOW_STICKER_CHOICES): - voter_data['window_sticker'] = 'none' - if 'phone_type' in voter_data: - pt_val = str(voter_data['phone_type']).lower() - pt_choices = dict(Voter.PHONE_TYPE_CHOICES) - if pt_val not in pt_choices: - # Try to match by display name - found = False - for k, v in pt_choices.items(): - if v.lower() == pt_val: - voter_data['phone_type'] = k - found = True - break - if not found: - voter_data['phone_type'] = 'cell' - else: - voter_data['phone_type'] = pt_val - - voter, created = Voter.objects.get_or_create( - tenant=tenant, - voter_id=voter_id, - ) - for key, value in voter_data.items(): - setattr(voter, key, value) - - # Flag that coordinates were provided in the import to avoid geocoding - if "latitude" in voter_data and "longitude" in voter_data: - voter._coords_provided_in_import = True - - voter.save() - count += 1 - except Exception as e: - logger.error(f"Error importing: {e}") - row["Import Error"] = str(e) - failed_rows.append(row) - errors += 1 + count = 0 + errors = 0 + failed_rows = [] + batch_size = 500 # Optimized batch size + # Pre-calculate choice dicts and sets + support_choices = dict(Voter.SUPPORT_CHOICES) + yard_sign_choices = dict(Voter.YARD_SIGN_CHOICES) + window_sticker_choices = dict(Voter.WINDOW_STICKER_CHOICES) + phone_type_choices = dict(Voter.PHONE_TYPE_CHOICES) + phone_type_reverse = {v.lower(): k for k, v in phone_type_choices.items()} + + # Fields to fetch for change detection + valid_fields = {f.name for f in Voter._meta.get_fields()} + mapped_fields = {f for f in mapping.keys() if f in valid_fields} + fetch_fields = list(mapped_fields | {"voter_id", "address", "phone", "longitude"}) + update_fields = list(mapped_fields | {"address", "phone"}) + if "voter_id" in update_fields: update_fields.remove("voter_id") + + def chunk_reader(reader, size): + chunk = [] + for row in reader: + chunk.append(row) + if len(chunk) == size: + yield chunk + chunk = [] + if chunk: + yield chunk + + with open(file_path, "r", encoding="UTF-8") as f: + reader = csv.DictReader(f) + v_id_col = mapping.get("voter_id") + if not v_id_col: + raise ValueError("Voter ID mapping is missing") + + for chunk_index, chunk in enumerate(chunk_reader(reader, batch_size)): + with transaction.atomic(): + voter_ids = [row.get(v_id_col) for row in chunk if row.get(v_id_col)] + existing_voters = {v.voter_id: v for v in Voter.objects.filter(tenant=tenant, voter_id__in=voter_ids).only(*fetch_fields)} + + to_create = [] + to_update = [] + processed_in_batch = set() + + for row in chunk: + try: + voter_id = row.get(v_id_col) + if not voter_id or voter_id in processed_in_batch: + continue + processed_in_batch.add(voter_id) + + voter = existing_voters.get(voter_id) + created = False + if not voter: + voter = Voter(tenant=tenant, voter_id=voter_id) + created = True + + changed = created + + for field_name, csv_col in mapping.items(): + if field_name == "voter_id": continue + val = row.get(csv_col) + if val is None or str(val).strip() == "": continue + + # Type-specific conversions + if field_name == "is_targeted": + val = str(val).lower() in ["true", "1", "yes"] + elif field_name in ["birthdate", "registration_date"]: + try: + if isinstance(val, str): + val = datetime.strptime(val.strip(), "%Y-%m-%d").date() + except: + pass + elif field_name == "candidate_support": + if val not in support_choices: val = "unknown" + elif field_name == "yard_sign": + if val not in yard_sign_choices: val = "none" + elif field_name == "window_sticker": + if val not in window_sticker_choices: val = "none" + elif field_name == "phone_type": + val_lower = str(val).lower() + if val_lower in phone_type_choices: + val = val_lower + elif val_lower in phone_type_reverse: + val = phone_type_reverse[val_lower] + else: + val = "cell" + + if getattr(voter, field_name) != val: + setattr(voter, field_name, val) + changed = True + + # Special fields + old_phone = voter.phone + voter.phone = format_phone_number(voter.phone) + if voter.phone != old_phone: + changed = True + + if voter.longitude: + try: + new_lon = Decimal(str(voter.longitude)[:12]) + if voter.longitude != new_lon: + voter.longitude = new_lon + changed = True + except: + pass + + old_address = voter.address + parts = [voter.address_street, voter.city, voter.state, voter.zip_code] + voter.address = ", ".join([p for p in parts if p]) + if voter.address != old_address: + changed = True + + if not changed: + continue + + if created: + to_create.append(voter) + else: + to_update.append(voter) + + count += 1 + except Exception as e: + logger.error(f"Error importing row: {e}") + row["Import Error"] = str(e) + failed_rows.append(row) + errors += 1 + + if to_create: + Voter.objects.bulk_create(to_create) + if to_update: + Voter.objects.bulk_update(to_update, update_fields, batch_size=250) + + logger.info(f"Voter import progress: Processed batch {chunk_index + 1}. Total successes: {count}") + if os.path.exists(file_path): os.remove(file_path) self.message_user(request, f"Successfully imported {count} voters.") request.session[f"{self.model._meta.model_name}_import_errors"] = failed_rows request.session.modified = True - logger.info(f"Stored {len(failed_rows)} failed rows in session for {self.model._meta.model_name}") if errors > 0: error_url = reverse("admin:voter-download-errors") self.message_user(request, mark_safe(f"Failed to import {errors} rows. Download failed records"), level=messages.WARNING) return redirect("..") except Exception as e: + logger.exception("Voter import failed") self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) return redirect("..") else: form = VoterImportForm(request.POST, request.FILES) if form.is_valid(): - csv_file = request.FILES['file'] - tenant = form.cleaned_data['tenant'] + csv_file = request.FILES["file"] + tenant = form.cleaned_data["tenant"] - if not csv_file.name.endswith('.csv'): + if not csv_file.name.endswith(".csv"): self.message_user(request, "Please upload a CSV file.", level=messages.ERROR) return redirect("..") - with tempfile.NamedTemporaryFile(delete=False, suffix='.csv') as tmp: + with tempfile.NamedTemporaryFile(delete=False, suffix=".csv") as tmp: for chunk in csv_file.chunks(): tmp.write(chunk) file_path = tmp.name - with open(file_path, 'r', encoding='UTF-8') as f: + with open(file_path, "r", encoding="UTF-8") as f: reader = csv.reader(f) headers = next(reader) context = self.admin_site.each_context(request) context.update({ - 'title': "Map Voter Fields", - 'headers': headers, - 'model_fields': VOTER_MAPPABLE_FIELDS, - 'tenant_id': tenant.id, - 'file_path': file_path, - 'action_url': request.path, - 'opts': self.model._meta, + "title": "Map Voter Fields", + "headers": headers, + "model_fields": VOTER_MAPPABLE_FIELDS, + "tenant_id": tenant.id, + "file_path": file_path, + "action_url": request.path, + "opts": self.model._meta, }) return render(request, "admin/import_mapping.html", context) - else: - form = VoterImportForm() - - context = self.admin_site.each_context(request) - context['form'] = form - context['title'] = "Import Voters" - context['opts'] = self.model._meta - return render(request, "admin/import_csv.html", context) - -@admin.register(Event) class EventAdmin(BaseImportAdminMixin, admin.ModelAdmin): list_display = ('id', 'name', 'event_type', 'date', 'start_time', 'end_time', 'tenant') list_filter = ('tenant', 'date', 'event_type') diff --git a/core/admin.py.tmp b/core/admin.py.tmp new file mode 100644 index 0000000..1dd03f8 --- /dev/null +++ b/core/admin.py.tmp @@ -0,0 +1,22 @@ +from django.http import HttpResponse +from django.utils.safestring import mark_safe +import csv +import io +import logging +import tempfile +import os +from decimal import Decimal +from django.contrib import admin, messages +from django.urls import path, reverse +from django.shortcuts import render, redirect +from django.template.response import TemplateResponse +from .models import ( + Tenant, TenantUserRole, InteractionType, DonationMethod, ElectionType, EventType, Voter, + VotingRecord, Event, EventParticipation, Donation, Interaction, VoterLikelihood, CampaignSettings, + Interest, Volunteer, VolunteerEvent, ParticipationStatus, format_phone_number +) +from .forms import ( + VoterImportForm, EventImportForm, EventParticipationImportForm, + DonationImportForm, InteractionImportForm, VoterLikelihoodImportForm, + VolunteerImportForm +) diff --git a/core/forms.py b/core/forms.py index a281af4..95ff70e 100644 --- a/core/forms.py +++ b/core/forms.py @@ -8,13 +8,14 @@ class VoterForm(forms.ModelForm): 'first_name', 'last_name', 'nickname', 'birthdate', 'address_street', 'city', 'state', 'prior_state', 'zip_code', 'county', 'latitude', 'longitude', 'phone', 'phone_type', 'email', 'voter_id', 'district', 'precinct', - 'registration_date', 'is_targeted', 'candidate_support', 'yard_sign', 'window_sticker' + 'registration_date', 'is_targeted', 'candidate_support', 'yard_sign', 'window_sticker', 'notes' ] widgets = { 'birthdate': forms.DateInput(attrs={'type': 'date'}), 'registration_date': forms.DateInput(attrs={'type': 'date'}), 'latitude': forms.TextInput(attrs={'class': 'form-control bg-light'}), 'longitude': forms.TextInput(attrs={'class': 'form-control bg-light'}), + 'notes': forms.Textarea(attrs={'rows': 3}), } def __init__(self, *args, **kwargs): diff --git a/core/migrations/0022_voter_notes.py b/core/migrations/0022_voter_notes.py new file mode 100644 index 0000000..3f3f725 --- /dev/null +++ b/core/migrations/0022_voter_notes.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.7 on 2026-01-26 17:51 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0021_voter_phone_type'), + ] + + operations = [ + migrations.AddField( + model_name='voter', + name='notes', + field=models.TextField(blank=True), + ), + ] diff --git a/core/migrations/0023_alter_voter_address_street_alter_voter_birthdate_and_more.py b/core/migrations/0023_alter_voter_address_street_alter_voter_birthdate_and_more.py new file mode 100644 index 0000000..161345e --- /dev/null +++ b/core/migrations/0023_alter_voter_address_street_alter_voter_birthdate_and_more.py @@ -0,0 +1,83 @@ +# Generated by Django 5.2.7 on 2026-01-28 04:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0022_voter_notes'), + ] + + operations = [ + migrations.AlterField( + model_name='voter', + name='address_street', + field=models.CharField(blank=True, db_index=True, max_length=255), + ), + migrations.AlterField( + model_name='voter', + name='birthdate', + field=models.DateField(blank=True, db_index=True, null=True), + ), + migrations.AlterField( + model_name='voter', + name='candidate_support', + field=models.CharField(choices=[('unknown', 'Unknown'), ('supporting', 'Supporting'), ('not_supporting', 'Not Supporting')], db_index=True, default='unknown', max_length=20), + ), + migrations.AlterField( + model_name='voter', + name='city', + field=models.CharField(blank=True, db_index=True, max_length=100), + ), + migrations.AlterField( + model_name='voter', + name='district', + field=models.CharField(blank=True, db_index=True, max_length=100), + ), + migrations.AlterField( + model_name='voter', + name='first_name', + field=models.CharField(db_index=True, max_length=100), + ), + migrations.AlterField( + model_name='voter', + name='is_targeted', + field=models.BooleanField(db_index=True, default=False), + ), + migrations.AlterField( + model_name='voter', + name='last_name', + field=models.CharField(db_index=True, max_length=100), + ), + migrations.AlterField( + model_name='voter', + name='precinct', + field=models.CharField(blank=True, db_index=True, max_length=100), + ), + migrations.AlterField( + model_name='voter', + name='state', + field=models.CharField(blank=True, db_index=True, max_length=2), + ), + migrations.AlterField( + model_name='voter', + name='voter_id', + field=models.CharField(blank=True, db_index=True, max_length=50), + ), + migrations.AlterField( + model_name='voter', + name='window_sticker', + field=models.CharField(choices=[('none', 'None'), ('wants', 'Wants Sticker'), ('has', 'Has Sticker')], db_index=True, default='none', max_length=20, verbose_name='Window Sticker Status'), + ), + migrations.AlterField( + model_name='voter', + name='yard_sign', + field=models.CharField(choices=[('none', 'None'), ('wants', 'Wants a yard sign'), ('has', 'Has a yard sign')], db_index=True, default='none', max_length=20), + ), + migrations.AlterField( + model_name='voter', + name='zip_code', + field=models.CharField(blank=True, db_index=True, max_length=20), + ), + ] diff --git a/core/migrations/__pycache__/0022_voter_notes.cpython-311.pyc b/core/migrations/__pycache__/0022_voter_notes.cpython-311.pyc new file mode 100644 index 0000000..90c4524 Binary files /dev/null and b/core/migrations/__pycache__/0022_voter_notes.cpython-311.pyc differ diff --git a/core/migrations/__pycache__/0023_alter_voter_address_street_alter_voter_birthdate_and_more.cpython-311.pyc b/core/migrations/__pycache__/0023_alter_voter_address_street_alter_voter_birthdate_and_more.cpython-311.pyc new file mode 100644 index 0000000..f2c7943 Binary files /dev/null and b/core/migrations/__pycache__/0023_alter_voter_address_street_alter_voter_birthdate_and_more.cpython-311.pyc differ diff --git a/core/models.py b/core/models.py index e847349..e88c2d4 100644 --- a/core/models.py +++ b/core/models.py @@ -133,30 +133,31 @@ class Voter(models.Model): ] tenant = models.ForeignKey(Tenant, on_delete=models.CASCADE, related_name='voters') - voter_id = models.CharField(max_length=50, blank=True) - first_name = models.CharField(max_length=100) - last_name = models.CharField(max_length=100) + voter_id = models.CharField(max_length=50, blank=True, db_index=True) + first_name = models.CharField(max_length=100, db_index=True) + last_name = models.CharField(max_length=100, db_index=True) nickname = models.CharField(max_length=100, blank=True) - birthdate = models.DateField(null=True, blank=True) + birthdate = models.DateField(null=True, blank=True, db_index=True) address = models.TextField(blank=True) - address_street = models.CharField(max_length=255, blank=True) - city = models.CharField(max_length=100, blank=True) - state = models.CharField(max_length=2, blank=True) + address_street = models.CharField(max_length=255, blank=True, db_index=True) + city = models.CharField(max_length=100, blank=True, db_index=True) + state = models.CharField(max_length=2, blank=True, db_index=True) prior_state = models.CharField(max_length=2, blank=True) - zip_code = models.CharField(max_length=20, blank=True) + zip_code = models.CharField(max_length=20, blank=True, db_index=True) county = models.CharField(max_length=100, blank=True) latitude = models.DecimalField(max_digits=12, decimal_places=9, null=True, blank=True) longitude = models.DecimalField(max_digits=12, decimal_places=9, null=True, blank=True) phone = models.CharField(max_length=20, blank=True) phone_type = models.CharField(max_length=10, choices=PHONE_TYPE_CHOICES, default='cell') email = models.EmailField(blank=True) - district = models.CharField(max_length=100, blank=True) - precinct = models.CharField(max_length=100, blank=True) + district = models.CharField(max_length=100, blank=True, db_index=True) + precinct = models.CharField(max_length=100, blank=True, db_index=True) registration_date = models.DateField(null=True, blank=True) - is_targeted = models.BooleanField(default=False) - candidate_support = models.CharField(max_length=20, choices=SUPPORT_CHOICES, default='unknown') - yard_sign = models.CharField(max_length=20, choices=YARD_SIGN_CHOICES, default='none') - window_sticker = models.CharField(max_length=20, choices=WINDOW_STICKER_CHOICES, default='none', verbose_name='Window Sticker Status') + is_targeted = models.BooleanField(default=False, db_index=True) + candidate_support = models.CharField(max_length=20, choices=SUPPORT_CHOICES, default='unknown', db_index=True) + yard_sign = models.CharField(max_length=20, choices=YARD_SIGN_CHOICES, default='none', db_index=True) + window_sticker = models.CharField(max_length=20, choices=WINDOW_STICKER_CHOICES, default='none', verbose_name='Window Sticker Status', db_index=True) + notes = models.TextField(blank=True) created_at = models.DateTimeField(auto_now_add=True) diff --git a/core/templates/core/voter_advanced_search.html b/core/templates/core/voter_advanced_search.html index 731ed40..4bf18c5 100644 --- a/core/templates/core/voter_advanced_search.html +++ b/core/templates/core/voter_advanced_search.html @@ -78,23 +78,25 @@ {% csrf_token %} {% for key, value in request.GET.items %} - {% if key != 'csrfmiddlewaretoken' %} + {% if key != 'csrfmiddlewaretoken' and key != 'page' %} {% endif %} {% endfor %}
{{ voter.notes|default:"No notes available." }}
+