from decimal import Decimal from datetime import datetime, date from django.db import transaction from django.http import HttpResponse from django.utils.safestring import mark_safe import csv import io import logging import tempfile import os from django.contrib import admin, messages from django.urls import path, reverse from django.shortcuts import render, redirect from django.template.response import TemplateResponse from .models import ( format_phone_number, Tenant, TenantUserRole, InteractionType, DonationMethod, ElectionType, EventType, Voter, VotingRecord, Event, EventParticipation, Donation, Interaction, VoterLikelihood, CampaignSettings, Interest, Volunteer, VolunteerEvent, ParticipationStatus, VolunteerRole ) from .forms import ( VoterImportForm, EventImportForm, EventParticipationImportForm, DonationImportForm, InteractionImportForm, VoterLikelihoodImportForm, VolunteerImportForm, VotingRecordImportForm ) logger = logging.getLogger(__name__) VOTER_MAPPABLE_FIELDS = [ ('voter_id', 'Voter ID'), ('first_name', 'First Name'), ('last_name', 'Last Name'), ('nickname', 'Nickname'), ('birthdate', 'Birthdate'), ('address_street', 'Street Address'), ('city', 'City'), ('state', 'State'), ('prior_state', 'Prior State'), ('zip_code', 'Zip Code'), ('county', 'County'), ('neighborhood', 'Neighborhood'), ('phone', 'Phone'), ('notes', 'Notes'), ('phone_type', 'Phone Type'), ('email', 'Email'), ('district', 'District'), ('precinct', 'Precinct'), ('registration_date', 'Registration Date'), ('is_targeted', 'Is Targeted'), ('candidate_support', 'Candidate Support'), ('yard_sign', 'Yard Sign'), ('window_sticker', 'Window Sticker'), ('latitude', 'Latitude'), ('longitude', 'Longitude'), ('secondary_phone', 'Secondary Phone'), ('secondary_phone_type', 'Secondary Phone Type'), ] EVENT_MAPPABLE_FIELDS = [ ('name', 'Name'), ('date', 'Date'), ('start_time', 'Start Time'), ('end_time', 'End Time'), ('event_type', 'Event Type (Name)'), ('description', 'Description'), ('location_name', 'Location Name'), ('address', 'Address'), ('city', 'City'), ('state', 'State'), ('zip_code', 'Zip Code'), ('latitude', 'Latitude'), ('longitude', 'Longitude'), ] EVENT_PARTICIPATION_MAPPABLE_FIELDS = [ ('voter_id', 'Voter ID'), ('event_name', 'Event Name'), ('participation_status', 'Participation Status'), ] DONATION_MAPPABLE_FIELDS = [ ('voter_id', 'Voter ID'), ('date', 'Date'), ('amount', 'Amount'), ('method', 'Donation Method (Name)'), ] INTERACTION_MAPPABLE_FIELDS = [ ('voter_id', 'Voter ID'), ('volunteer_email', 'Volunteer Email'), ('date', 'Date'), ('type', 'Interaction Type (Name)'), ('description', 'Description'), ('notes', 'Notes'), ] VOLUNTEER_MAPPABLE_FIELDS = [ ('first_name', 'First Name'), ('last_name', 'Last Name'), ('email', 'Email'), ('phone', 'Phone'), ('notes', 'Notes'), ] VOTER_LIKELIHOOD_MAPPABLE_FIELDS = [ ('voter_id', 'Voter ID'), ('election_type', 'Election Type (Name)'), ('likelihood', 'Likelihood'), ] VOTING_RECORD_MAPPABLE_FIELDS = [ ('voter_id', 'Voter ID'), ('election_date', 'Election Date'), ('election_description', 'Election Description'), ('primary_party', 'Primary Party'), ] class BaseImportAdminMixin: def download_errors(self, request): logger.info(f"download_errors called for {self.model._meta.model_name}") session_key = f"{self.model._meta.model_name}_import_errors" failed_rows = request.session.get(session_key, []) if not failed_rows: self.message_user(request, "No error log found in session.", level=messages.WARNING) return redirect("..") response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = f"attachment; filename={self.model._meta.model_name}_import_errors.csv" if failed_rows: all_keys = set() for r in failed_rows: all_keys.update(r.keys()) writer = csv.DictWriter(response, fieldnames=sorted(list(all_keys))) writer.writeheader() writer.writerows(failed_rows) return response def chunk_reader(self, reader, size): chunk = [] for row in reader: chunk.append(row) if len(chunk) == size: yield chunk chunk = [] if chunk: yield chunk class TenantUserRoleInline(admin.TabularInline): model = TenantUserRole extra = 1 class CampaignSettingsInline(admin.StackedInline): model = CampaignSettings can_delete = False @admin.register(Tenant) class TenantAdmin(admin.ModelAdmin): list_display = ('name', 'created_at') search_fields = ('name',) inlines = [TenantUserRoleInline, CampaignSettingsInline] @admin.register(TenantUserRole) class TenantUserRoleAdmin(admin.ModelAdmin): list_display = ('user', 'tenant', 'role') list_filter = ('tenant', 'role') search_fields = ('user__username', 'tenant__name') @admin.register(InteractionType) class InteractionTypeAdmin(admin.ModelAdmin): list_display = ('name', 'tenant', 'is_active') list_filter = ('tenant', 'is_active') search_fields = ('name',) @admin.register(DonationMethod) class DonationMethodAdmin(admin.ModelAdmin): list_display = ('name', 'tenant', 'is_active') list_filter = ('tenant', 'is_active') search_fields = ('name',) @admin.register(VolunteerRole) class VolunteerRoleAdmin(admin.ModelAdmin): list_display = ("name", "tenant", "is_active") list_filter = ("tenant", "is_active") search_fields = ("name",) @admin.register(ElectionType) class ElectionTypeAdmin(admin.ModelAdmin): list_display = ('name', 'tenant', 'is_active') list_filter = ('tenant', 'is_active') search_fields = ('name',) @admin.register(EventType) class EventTypeAdmin(admin.ModelAdmin): list_display = ('name', 'tenant', 'is_active', 'default_volunteer_role') list_filter = ('tenant', 'is_active') search_fields = ('name',) filter_horizontal = ('available_roles',) @admin.register(ParticipationStatus) class ParticipationStatusAdmin(admin.ModelAdmin): list_display = ('name', 'tenant', 'is_active') list_filter = ('tenant', 'is_active') search_fields = ('name',) change_list_template = 'admin/participationstatus_change_list.html' def changelist_view(self, request, extra_context=None): extra_context = extra_context or {} from core.models import Tenant extra_context['tenants'] = Tenant.objects.all() return super().changelist_view(request, extra_context=extra_context) @admin.register(Interest) class InterestAdmin(admin.ModelAdmin): list_display = ('name', 'tenant') list_filter = ('tenant',) fields = ('tenant', 'name') search_fields = ('name',) class VotingRecordInline(admin.TabularInline): model = VotingRecord extra = 1 class DonationInline(admin.TabularInline): model = Donation extra = 1 class InteractionInline(admin.TabularInline): model = Interaction extra = 1 autocomplete_fields = ['voter', 'type', 'volunteer'] class VoterLikelihoodInline(admin.TabularInline): model = VoterLikelihood extra = 1 class VolunteerEventInline(admin.TabularInline): model = VolunteerEvent extra = 1 @admin.register(Voter) class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin): list_display = ('first_name', 'last_name', 'nickname', 'voter_id', 'tenant', 'district', 'candidate_support', 'is_targeted', 'city', 'state', 'prior_state') list_filter = ('tenant', 'candidate_support', 'is_targeted', 'phone_type', 'yard_sign', 'district', 'city', 'state', 'prior_state') search_fields = ('first_name', 'last_name', 'nickname', 'voter_id', 'address', 'city', 'state', 'prior_state', 'zip_code', 'county') inlines = [VotingRecordInline, DonationInline, InteractionInline, VoterLikelihoodInline] readonly_fields = ('address',) change_list_template = "admin/voter_change_list.html" def changelist_view(self, request, extra_context=None): extra_context = extra_context or {} from core.models import Tenant extra_context["tenants"] = Tenant.objects.all() return super().changelist_view(request, extra_context=extra_context) def get_urls(self): urls = super().get_urls() my_urls = [ path('download-errors/', self.admin_site.admin_view(self.download_errors), name='voter-download-errors'), path('import-voters/', self.admin_site.admin_view(self.import_voters), name='import-voters'), ] return my_urls + urls def import_voters(self, request): if request.method == "POST": if "_preview" in request.POST: file_path = request.POST.get("file_path") tenant_id = request.POST.get("tenant") tenant = Tenant.objects.get(id=tenant_id) mapping = {} for field_name, _ in VOTER_MAPPABLE_FIELDS: mapping[field_name] = request.POST.get(f"map_{field_name}") try: with open(file_path, "r", encoding="utf-8-sig") as f: # Optimization: Skip full count for very large files in preview if needed, # but here we'll keep it for accuracy unless it's a known bottleneck. # For now, let's just do a fast line count. total_count = sum(1 for line in f) - 1 f.seek(0) reader = csv.DictReader(f) preview_rows = [] voter_ids_for_preview = [] for i, row in enumerate(reader): if i < 10: preview_rows.append(row) v_id = row.get(mapping.get("voter_id")) if v_id: voter_ids_for_preview.append(v_id) else: break existing_preview_ids = set(Voter.objects.filter(tenant=tenant, voter_id__in=voter_ids_for_preview).values_list("voter_id", flat=True)) preview_data = [] for row in preview_rows: v_id = row.get(mapping.get("voter_id")) action = "update" if v_id in existing_preview_ids else "create" preview_data.append({ "action": action, "identifier": v_id, "details": f"{row.get(mapping.get('first_name', '')) or ''} {row.get(mapping.get('last_name', '')) or ''}".strip() }) context = self.admin_site.each_context(request) context.update({ "title": "Import Preview", "total_count": total_count, "create_count": "N/A", "update_count": "N/A", "preview_data": preview_data, "mapping": mapping, "file_path": file_path, "tenant_id": tenant_id, "action_url": request.path, "opts": self.model._meta, }) return render(request, "admin/import_preview.html", context) except Exception as e: self.message_user(request, f"Error processing preview: {e}", level=messages.ERROR) return redirect("..") elif "_import" in request.POST: file_path = request.POST.get("file_path") tenant_id = request.POST.get("tenant") tenant = Tenant.objects.get(id=tenant_id) mapping = {k: request.POST.get(f"map_{k}") for k, _ in VOTER_MAPPABLE_FIELDS if request.POST.get(f"map_{k}")} try: count = 0 created_count = 0 updated_count = 0 skipped_no_change = 0 skipped_no_id = 0 errors = 0 failed_rows = [] batch_size = 2000 # Increased batch size # Pre-calculate choices and reverse mappings support_choices = dict(Voter.SUPPORT_CHOICES) support_reverse = {v.lower(): k for k, v in support_choices.items()} yard_sign_choices = dict(Voter.YARD_SIGN_CHOICES) yard_sign_reverse = {v.lower(): k for k, v in yard_sign_choices.items()} window_sticker_choices = dict(Voter.WINDOW_STICKER_CHOICES) window_sticker_reverse = {v.lower(): k for k, v in window_sticker_choices.items()} phone_type_choices = dict(Voter.PHONE_TYPE_CHOICES) phone_type_reverse = {v.lower(): k for k, v in phone_type_choices.items()} # Identify what type of data is being imported to skip unnecessary logic mapped_fields = set(mapping.keys()) is_address_related = any(f in mapped_fields for f in ["address_street", "city", "state", "zip_code"]) is_phone_related = any(f in mapped_fields for f in ["phone", "secondary_phone", "phone_type", "secondary_phone_type"]) is_coords_related = any(f in mapped_fields for f in ["latitude", "longitude"]) with open(file_path, "r", encoding="utf-8-sig") as f: # Optimization: Use csv.reader instead of DictReader for performance raw_reader = csv.reader(f) headers = next(raw_reader) header_to_idx = {h: i for i, h in enumerate(headers)} v_id_col_name = mapping.get("voter_id") if not v_id_col_name or v_id_col_name not in header_to_idx: raise ValueError(f"Voter ID mapping '{v_id_col_name}' is missing or invalid") v_id_idx = header_to_idx[v_id_col_name] # Map internal field names to CSV column indices mapping_indices = {k: header_to_idx[v] for k, v in mapping.items() if v in header_to_idx} # Optimization: Only fetch needed fields fields_to_fetch = {"id", "voter_id"} | mapped_fields if is_address_related: fields_to_fetch.add("address") print(f"DEBUG: Starting optimized voter import. Tenant: {tenant.name}. Fields: {mapped_fields}") total_processed = 0 # Use chunk_reader with the raw_reader for chunk in self.chunk_reader(raw_reader, batch_size): with transaction.atomic(): voter_ids = [] chunk_data = [] for row in chunk: if len(row) <= v_id_idx: continue v_id = row[v_id_idx].strip() if v_id: voter_ids.append(v_id) chunk_data.append((v_id, row)) else: skipped_no_id += 1 # Fetch existing voters in one query existing_voters = { v.voter_id: v for v in Voter.objects.filter(tenant=tenant, voter_id__in=voter_ids) .only(*fields_to_fetch) } to_create = [] to_update = [] batch_updated_fields = set() processed_in_batch = set() for voter_id, row in chunk_data: total_processed += 1 try: if voter_id in processed_in_batch: continue processed_in_batch.add(voter_id) voter = existing_voters.get(voter_id) created = False if not voter: voter = Voter(tenant=tenant, voter_id=voter_id) created = True changed = created record_updated_fields = set() # Process mapped fields for field_name, idx in mapping_indices.items(): if field_name == "voter_id": continue if idx >= len(row): continue val = row[idx].strip() if val == "" and not created: continue # Skip empty updates for existing records unless specifically desired? # Type conversion and normalization if field_name == "is_targeted": val = val.lower() in ["true", "1", "yes"] elif field_name in ["birthdate", "registration_date"]: parsed_date = None for fmt in ["%Y-%m-%d", "%m/%d/%Y", "%d/%m/%Y", "%Y/%m/%d"]: try: parsed_date = datetime.strptime(val, fmt).date() break except: continue if parsed_date: val = parsed_date else: continue elif field_name == "candidate_support": val_lower = val.lower() if val_lower in support_choices: val = val_lower elif val_lower in support_reverse: val = support_reverse[val_lower] else: val = "unknown" elif field_name == "yard_sign": val_lower = val.lower() if val_lower in yard_sign_choices: val = val_lower elif val_lower in yard_sign_reverse: val = yard_sign_reverse[val_lower] else: val = "none" elif field_name == "window_sticker": val_lower = val.lower() if val_lower in window_sticker_choices: val = val_lower elif val_lower in window_sticker_reverse: val = window_sticker_reverse[val_lower] else: val = "none" elif field_name in ["phone_type", "secondary_phone_type"]: val_lower = val.lower() if val_lower in phone_type_choices: val = val_lower elif val_lower in phone_type_reverse: val = phone_type_reverse[val_lower] else: val = "cell" if getattr(voter, field_name) != val: setattr(voter, field_name, val) changed = True record_updated_fields.add(field_name) # Optimization: Only perform transformations if related fields are mapped if is_phone_related or created: old_p = voter.phone voter.phone = format_phone_number(voter.phone) if voter.phone != old_p: changed = True record_updated_fields.add("phone") old_sp = voter.secondary_phone voter.secondary_phone = format_phone_number(voter.secondary_phone) if voter.secondary_phone != old_sp: changed = True record_updated_fields.add("secondary_phone") if (is_coords_related or created) and voter.longitude: try: new_lon = Decimal(str(voter.longitude)[:12]) if voter.longitude != new_lon: voter.longitude = new_lon changed = True record_updated_fields.add("longitude") except: pass if is_address_related or created: old_addr = voter.address parts = [voter.address_street, voter.city, voter.state, voter.zip_code] voter.address = ", ".join([p for p in parts if p]) if voter.address != old_addr: changed = True record_updated_fields.add("address") if not changed: skipped_no_change += 1 continue if created: to_create.append(voter) created_count += 1 else: to_update.append(voter) batch_updated_fields.update(record_updated_fields) updated_count += 1 count += 1 except Exception as e: errors += 1 if len(failed_rows) < 1000: row_dict = dict(zip(headers, row)) row_dict["Import Error"] = str(e) failed_rows.append(row_dict) if to_create: Voter.objects.bulk_create(to_create, batch_size=batch_size) if to_update: Voter.objects.bulk_update(to_update, list(batch_updated_fields), batch_size=batch_size) print(f"DEBUG: Voter import progress: {total_processed} processed. {count} created/updated. Errors: {errors}") if os.path.exists(file_path): os.remove(file_path) self.message_user(request, f"Import complete: {count} voters created/updated. ({created_count} new, {updated_count} updated, {skipped_no_change} skipped with no changes, {skipped_no_id} skipped missing ID, {errors} errors)") request.session[f"{self.model._meta.model_name}_import_errors"] = failed_rows request.session.modified = True if errors > 0: error_url = reverse("admin:voter-download-errors") self.message_user(request, mark_safe(f"Failed to import {errors} rows. Download failed records"), level=messages.WARNING) return redirect("..") except Exception as e: print(f"DEBUG: Voter import failed: {e}") self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) return redirect("..") else: form = VoterImportForm(request.POST, request.FILES) if form.is_valid(): csv_file = request.FILES["file"] tenant = form.cleaned_data["tenant"] if not csv_file.name.endswith(".csv"): self.message_user(request, "Please upload a CSV file.", level=messages.ERROR) return redirect("..") with tempfile.NamedTemporaryFile(delete=False, suffix=".csv") as tmp: for chunk in csv_file.chunks(): tmp.write(chunk) file_path = tmp.name with open(file_path, "r", encoding="utf-8-sig") as f: reader = csv.reader(f) headers = next(reader) context = self.admin_site.each_context(request) context.update({ "title": "Map Voter Fields", "headers": headers, "model_fields": VOTER_MAPPABLE_FIELDS, "tenant_id": tenant.id, "file_path": file_path, "action_url": request.path, "opts": self.model._meta, }) return render(request, "admin/import_mapping.html", context) else: form = VoterImportForm() context = self.admin_site.each_context(request) context["form"] = form context["title"] = "Import Voters" context["opts"] = self.model._meta return render(request, "admin/import_csv.html", context) @admin.register(Event) class EventAdmin(BaseImportAdminMixin, admin.ModelAdmin): list_display = ('id', 'name', 'event_type', 'date', 'location_name', 'city', 'state', 'tenant') list_filter = ('tenant', 'date', 'event_type', 'city', 'state') search_fields = ('name', 'description', 'location_name', 'address', 'city', 'state', 'zip_code') change_list_template = "admin/event_change_list.html" def changelist_view(self, request, extra_context=None): extra_context = extra_context or {} from core.models import Tenant extra_context["tenants"] = Tenant.objects.all() return super().changelist_view(request, extra_context=extra_context) def get_urls(self): urls = super().get_urls() my_urls = [ path('download-errors/', self.admin_site.admin_view(self.download_errors), name='event-download-errors'), path('import-events/', self.admin_site.admin_view(self.import_events), name='import-events'), ] return my_urls + urls def import_events(self, request): if request.method == "POST": if "_preview" in request.POST: file_path = request.POST.get('file_path') tenant_id = request.POST.get('tenant') tenant = Tenant.objects.get(id=tenant_id) mapping = {} for field_name, _ in EVENT_MAPPABLE_FIELDS: mapping[field_name] = request.POST.get(f'map_{field_name}') try: with open(file_path, 'r', encoding='UTF-8') as f: reader = csv.DictReader(f) total_count = 0 create_count = 0 update_count = 0 preview_data = [] for row in reader: total_count += 1 date = row.get(mapping.get('date')) event_type_name = row.get(mapping.get('event_type')) event_name = row.get(mapping.get('name')) exists = False if date and event_type_name: q = Event.objects.filter(tenant=tenant, date=date, event_type__name=event_type_name) if event_name: q = q.filter(name=event_name) exists = q.exists() if exists: update_count += 1 action = 'update' else: create_count += 1 action = 'create' if len(preview_data) < 10: preview_data.append({ 'action': action, 'identifier': f"{event_name or 'No Name'} ({date} - {event_type_name})", 'details': f"{row.get(mapping.get('city', '')) or ''}, {row.get(mapping.get('state', '')) or ''}" }) context = self.admin_site.each_context(request) context.update({ 'title': "Import Preview", 'total_count': total_count, 'create_count': create_count, 'update_count': update_count, 'preview_data': preview_data, 'mapping': mapping, 'file_path': file_path, 'tenant_id': tenant_id, 'action_url': request.path, 'opts': self.model._meta, }) return render(request, "admin/import_preview.html", context) except Exception as e: self.message_user(request, f"Error processing preview: {e}", level=messages.ERROR) return redirect("..") elif "_import" in request.POST: file_path = request.POST.get('file_path') tenant_id = request.POST.get('tenant') tenant = Tenant.objects.get(id=tenant_id) mapping = {} for field_name, _ in EVENT_MAPPABLE_FIELDS: mapping[field_name] = request.POST.get(f'map_{field_name}') try: with open(file_path, 'r', encoding='UTF-8') as f: reader = csv.DictReader(f) count = 0 errors = 0 failed_rows = [] for row in reader: try: date = row.get(mapping.get('date')) if mapping.get('date') else None event_type_name = row.get(mapping.get('event_type')) if mapping.get('event_type') else None description = row.get(mapping.get('description')) if mapping.get('description') else None location_name = row.get(mapping.get('location_name')) if mapping.get('location_name') else None name = row.get(mapping.get('name')) if mapping.get('name') else None start_time = row.get(mapping.get('start_time')) if mapping.get('start_time') else None end_time = row.get(mapping.get('end_time')) if mapping.get('end_time') else None address = row.get(mapping.get('address')) if mapping.get('address') else None city = row.get(mapping.get('city')) if mapping.get('city') else None state = row.get(mapping.get('state')) if mapping.get('state') else None zip_code = row.get(mapping.get('zip_code')) if mapping.get('zip_code') else None latitude = row.get(mapping.get('latitude')) if mapping.get('latitude') else None longitude = row.get(mapping.get('longitude')) if mapping.get('longitude') else None if not date or not event_type_name: row["Import Error"] = "Missing date or event type" failed_rows.append(row) errors += 1 continue event_type, _ = EventType.objects.get_or_create( tenant=tenant, name=event_type_name ) defaults = {} if description and description.strip(): defaults['description'] = description if location_name and location_name.strip(): defaults['location_name'] = location_name if name and name.strip(): defaults['name'] = name if start_time and start_time.strip(): defaults['start_time'] = start_time if end_time and end_time.strip(): defaults['end_time'] = end_time if address and address.strip(): defaults['address'] = address if city and city.strip(): defaults['city'] = city if state and state.strip(): defaults['state'] = state if zip_code and zip_code.strip(): defaults['zip_code'] = zip_code if latitude and latitude.strip(): defaults['latitude'] = latitude if longitude and longitude.strip(): defaults['longitude'] = longitude defaults['date'] = date defaults['event_type'] = event_type Event.objects.update_or_create( tenant=tenant, name=name or '', defaults=defaults ) count += 1 except Exception as e: logger.error(f"Error importing: {e}") row["Import Error"] = str(e) failed_rows.append(row) errors += 1 if os.path.exists(file_path): os.remove(file_path) self.message_user(request, f"Successfully imported {count} events.") # Optimization: Limit error log size in session to avoid overflow request.session[f"{self.model._meta.model_name}_import_errors"] = failed_rows[:1000] request.session.modified = True logger.info(f"Stored {len(failed_rows)} failed rows in session for {self.model._meta.model_name}") if errors > 0: error_url = reverse("admin:event-download-errors") self.message_user(request, mark_safe(f"Failed to import {errors} rows. Download failed records"), level=messages.WARNING) return redirect("..") except Exception as e: self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) return redirect("..") else: form = EventImportForm(request.POST, request.FILES) if form.is_valid(): csv_file = request.FILES['file'] tenant = form.cleaned_data['tenant'] if not csv_file.name.endswith('.csv'): self.message_user(request, "Please upload a CSV file.", level=messages.ERROR) return redirect("..") with tempfile.NamedTemporaryFile(delete=False, suffix='.csv') as tmp: for chunk in csv_file.chunks(): tmp.write(chunk) file_path = tmp.name with open(file_path, 'r', encoding='UTF-8') as f: reader = csv.reader(f) headers = next(reader) context = self.admin_site.each_context(request) context.update({ 'title': "Map Event Fields", 'headers': headers, 'model_fields': EVENT_MAPPABLE_FIELDS, 'tenant_id': tenant.id, 'file_path': file_path, 'action_url': request.path, 'opts': self.model._meta, }) return render(request, "admin/import_mapping.html", context) else: form = EventImportForm() context = self.admin_site.each_context(request) context['form'] = form context['title'] = "Import Events" context['opts'] = self.model._meta return render(request, "admin/import_csv.html", context) @admin.register(Volunteer) class VolunteerAdmin(BaseImportAdminMixin, admin.ModelAdmin): list_display = ('first_name', 'last_name', 'email', 'phone', 'tenant', 'user') ordering = ("last_name", "first_name") list_filter = ('tenant',) fields = ('tenant', 'user', 'first_name', 'last_name', 'email', 'phone', 'notes', 'interests') search_fields = ('first_name', 'last_name', 'email', 'phone') inlines = [VolunteerEventInline, InteractionInline] filter_horizontal = ('interests',) change_list_template = "admin/volunteer_change_list.html" def changelist_view(self, request, extra_context=None): extra_context = extra_context or {} from core.models import Tenant extra_context["tenants"] = Tenant.objects.all() return super().changelist_view(request, extra_context=extra_context) def get_urls(self): urls = super().get_urls() my_urls = [ path('download-errors/', self.admin_site.admin_view(self.download_errors), name='volunteer-download-errors'), path('import-volunteers/', self.admin_site.admin_view(self.import_volunteers), name='import-volunteers'), ] return my_urls + urls def import_volunteers(self, request): if request.method == "POST": if "_preview" in request.POST: file_path = request.POST.get('file_path') tenant_id = request.POST.get('tenant') tenant = Tenant.objects.get(id=tenant_id) mapping = {} for field_name, _ in VOLUNTEER_MAPPABLE_FIELDS: mapping[field_name] = request.POST.get(f'map_{field_name}') try: with open(file_path, 'r', encoding='UTF-8') as f: reader = csv.DictReader(f) total_count = 0 create_count = 0 update_count = 0 preview_data = [] for row in reader: total_count += 1 email = row.get(mapping.get('email')) exists = Volunteer.objects.filter(tenant=tenant, email=email).exists() if exists: update_count += 1 action = 'update' else: create_count += 1 action = 'create' if len(preview_data) < 10: preview_data.append({ 'action': action, 'identifier': email, 'details': f"{row.get(mapping.get('first_name', '')) or ''} {row.get(mapping.get('last_name', '')) or ''}".strip() }) context = self.admin_site.each_context(request) context.update({ 'title': "Import Preview", 'total_count': total_count, 'create_count': create_count, 'update_count': update_count, 'preview_data': preview_data, 'mapping': mapping, 'file_path': file_path, 'tenant_id': tenant_id, 'action_url': request.path, 'opts': self.model._meta, }) return render(request, "admin/import_preview.html", context) except Exception as e: self.message_user(request, f"Error processing preview: {e}", level=messages.ERROR) return redirect("..") elif "_import" in request.POST: file_path = request.POST.get('file_path') tenant_id = request.POST.get('tenant') tenant = Tenant.objects.get(id=tenant_id) mapping = {} for field_name, _ in VOLUNTEER_MAPPABLE_FIELDS: mapping[field_name] = request.POST.get(f'map_{field_name}') try: with open(file_path, 'r', encoding='UTF-8') as f: reader = csv.DictReader(f) count = 0 errors = 0 failed_rows = [] for row in reader: try: email = row.get(mapping.get('email')) if not email: row["Import Error"] = "Missing email" failed_rows.append(row) errors += 1 continue volunteer_data = {} for field_name, csv_col in mapping.items(): if csv_col: val = row.get(csv_col) if val is not None and str(val).strip() != '': if field_name == 'email': continue volunteer_data[field_name] = val Volunteer.objects.update_or_create( tenant=tenant, email=email, defaults=volunteer_data ) count += 1 except Exception as e: logger.error(f"Error importing volunteer: {e}") row["Import Error"] = str(e) failed_rows.append(row) errors += 1 if os.path.exists(file_path): os.remove(file_path) self.message_user(request, f"Successfully imported {count} volunteers.") # Optimization: Limit error log size in session to avoid overflow request.session[f"{self.model._meta.model_name}_import_errors"] = failed_rows[:1000] request.session.modified = True if errors > 0: error_url = reverse("admin:volunteer-download-errors") self.message_user(request, mark_safe(f"Failed to import {errors} rows. Download failed records"), level=messages.WARNING) return redirect("..") except Exception as e: self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) return redirect("..") else: form = VolunteerImportForm(request.POST, request.FILES) if form.is_valid(): csv_file = request.FILES['file'] tenant = form.cleaned_data['tenant'] if not csv_file.name.endswith('.csv'): self.message_user(request, "Please upload a CSV file.", level=messages.ERROR) return redirect("..") with tempfile.NamedTemporaryFile(delete=False, suffix='.csv') as tmp: for chunk in csv_file.chunks(): tmp.write(chunk) file_path = tmp.name with open(file_path, 'r', encoding='UTF-8') as f: reader = csv.reader(f) headers = next(reader) context = self.admin_site.each_context(request) context.update({ 'title': "Map Volunteer Fields", 'headers': headers, 'model_fields': VOLUNTEER_MAPPABLE_FIELDS, 'tenant_id': tenant.id, 'file_path': file_path, 'action_url': request.path, 'opts': self.model._meta, }) return render(request, "admin/import_mapping.html", context) else: form = VolunteerImportForm() context = self.admin_site.each_context(request) context['form'] = form context['title'] = "Import Volunteers" context['opts'] = self.model._meta return render(request, "admin/import_csv.html", context) @admin.register(VolunteerEvent) class VolunteerEventAdmin(admin.ModelAdmin): list_display = ('volunteer', 'event', 'role_type') list_filter = ('event__tenant', 'event', 'role_type') autocomplete_fields = ["volunteer", "event"] @admin.register(EventParticipation) class EventParticipationAdmin(BaseImportAdminMixin, admin.ModelAdmin): list_display = ('voter', 'event', 'participation_status') list_filter = ('event__tenant', 'event', 'participation_status') autocomplete_fields = ["voter", "event"] change_list_template = "admin/eventparticipation_change_list.html" def get_urls(self): urls = super().get_urls() my_urls = [ path('download-errors/', self.admin_site.admin_view(self.download_errors), name='eventparticipation-download-errors'), path('import-event-participations/', self.admin_site.admin_view(self.import_event_participations), name='import-event-participations'), ] return my_urls + urls def import_event_participations(self, request): if request.method == "POST": if "_preview" in request.POST: file_path = request.POST.get('file_path') tenant_id = request.POST.get('tenant') tenant = Tenant.objects.get(id=tenant_id) mapping = {} for field_name, _ in EVENT_PARTICIPATION_MAPPABLE_FIELDS: mapping[field_name] = request.POST.get(f'map_{field_name}') try: with open(file_path, 'r', encoding='UTF-8') as f: reader = csv.DictReader(f) total_count = 0 create_count = 0 update_count = 0 preview_data = [] for row in reader: total_count += 1 voter_id = row.get(mapping.get('voter_id')) event_name = row.get(mapping.get('event_name')) exists = False if voter_id: try: voter = Voter.objects.get(tenant=tenant, voter_id=voter_id) if event_name: exists = EventParticipation.objects.filter(voter=voter, event__name=event_name).exists() except Voter.DoesNotExist: pass if exists: update_count += 1 action = 'update' else: create_count += 1 action = 'create' if len(preview_data) < 10: preview_data.append({ 'action': action, 'identifier': f"Voter: {voter_id}", 'details': f"Participation: {row.get(mapping.get('participation_status', '')) or ''}" }) context = self.admin_site.each_context(request) context.update({ 'title': "Import Preview", 'total_count': total_count, 'create_count': create_count, 'update_count': update_count, 'preview_data': preview_data, 'mapping': mapping, 'file_path': file_path, 'tenant_id': tenant_id, 'action_url': request.path, 'opts': self.model._meta, }) return render(request, "admin/import_preview.html", context) except Exception as e: self.message_user(request, f"Error processing preview: {e}", level=messages.ERROR) return redirect("..") elif "_import" in request.POST: file_path = request.POST.get('file_path') tenant_id = request.POST.get('tenant') tenant = Tenant.objects.get(id=tenant_id) mapping = {} for field_name, _ in EVENT_PARTICIPATION_MAPPABLE_FIELDS: mapping[field_name] = request.POST.get(f'map_{field_name}') try: with open(file_path, 'r', encoding='UTF-8') as f: reader = csv.DictReader(f) count = 0 errors = 0 failed_rows = [] for row in reader: try: voter_id = row.get(mapping.get('voter_id')) if mapping.get('voter_id') else None participation_status_val = row.get(mapping.get('participation_status')) if mapping.get('participation_status') else None if not voter_id: row["Import Error"] = "Missing voter ID" failed_rows.append(row) errors += 1 continue try: voter = Voter.objects.get(tenant=tenant, voter_id=voter_id) except Voter.DoesNotExist: error_msg = f"Voter with ID {voter_id} not found" logger.error(error_msg) row["Import Error"] = error_msg failed_rows.append(row) errors += 1 continue event = None event_name = row.get(mapping.get('event_name')) if mapping.get('event_name') else None if event_name: try: event = Event.objects.get(tenant=tenant, name=event_name) except Event.DoesNotExist: pass if not event: error_msg = "Event not found (check Event Name)" logger.error(error_msg) row["Import Error"] = error_msg failed_rows.append(row) errors += 1 continue defaults = {} if participation_status_val and participation_status_val.strip(): status_obj, _ = ParticipationStatus.objects.get_or_create(tenant=tenant, name=participation_status_val.strip()) defaults['participation_status'] = status_obj else: # Default to 'Invited' if not specified status_obj, _ = ParticipationStatus.objects.get_or_create(tenant=tenant, name='Invited') defaults['participation_status'] = status_obj EventParticipation.objects.update_or_create( event=event, voter=voter, defaults=defaults ) count += 1 except Exception as e: logger.error(f"Error importing: {e}") row["Import Error"] = str(e) failed_rows.append(row) errors += 1 if os.path.exists(file_path): os.remove(file_path) self.message_user(request, f"Successfully imported {count} participations.") # Optimization: Limit error log size in session to avoid overflow request.session[f"{self.model._meta.model_name}_import_errors"] = failed_rows[:1000] request.session.modified = True logger.info(f"Stored {len(failed_rows)} failed rows in session for {self.model._meta.model_name}") if errors > 0: error_url = reverse("admin:eventparticipation-download-errors") self.message_user(request, mark_safe(f"Failed to import {errors} rows. Download failed records"), level=messages.WARNING) return redirect("..") except Exception as e: self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) return redirect("..") else: form = EventParticipationImportForm(request.POST, request.FILES) if form.is_valid(): csv_file = request.FILES['file'] tenant = form.cleaned_data['tenant'] if not csv_file.name.endswith('.csv'): self.message_user(request, "Please upload a CSV file.", level=messages.ERROR) return redirect("..") with tempfile.NamedTemporaryFile(delete=False, suffix='.csv') as tmp: for chunk in csv_file.chunks(): tmp.write(chunk) file_path = tmp.name with open(file_path, 'r', encoding='UTF-8') as f: reader = csv.reader(f) headers = next(reader) context = self.admin_site.each_context(request) context.update({ 'title': "Map Participation Fields", 'headers': headers, 'model_fields': EVENT_PARTICIPATION_MAPPABLE_FIELDS, 'tenant_id': tenant.id, 'file_path': file_path, 'action_url': request.path, 'opts': self.model._meta, }) return render(request, "admin/import_mapping.html", context) else: form = EventParticipationImportForm() context = self.admin_site.each_context(request) context['form'] = form context['title'] = "Import Participations" context['opts'] = self.model._meta return render(request, "admin/import_csv.html", context) @admin.register(Donation) class DonationAdmin(BaseImportAdminMixin, admin.ModelAdmin): list_display = ('id', 'voter', 'date', 'amount', 'method') list_filter = ('voter__tenant', 'date', 'method') search_fields = ('voter__first_name', 'voter__last_name', 'voter__voter_id') autocomplete_fields = ["voter"] change_list_template = "admin/donation_change_list.html" def get_urls(self): urls = super().get_urls() my_urls = [ path('download-errors/', self.admin_site.admin_view(self.download_errors), name='donation-download-errors'), path('import-donations/', self.admin_site.admin_view(self.import_donations), name='import-donations'), ] return my_urls + urls def import_donations(self, request): if request.method == "POST": if "_preview" in request.POST: file_path = request.POST.get('file_path') tenant_id = request.POST.get('tenant') tenant = Tenant.objects.get(id=tenant_id) mapping = {} for field_name, _ in DONATION_MAPPABLE_FIELDS: mapping[field_name] = request.POST.get(f'map_{field_name}') try: with open(file_path, 'r', encoding='UTF-8') as f: reader = csv.DictReader(f) total_count = 0 create_count = 0 update_count = 0 preview_data = [] for row in reader: total_count += 1 voter_id = row.get(mapping.get('voter_id')) date = row.get(mapping.get('date')) amount = row.get(mapping.get('amount')) exists = False if voter_id and date and amount: exists = Donation.objects.filter(voter__tenant=tenant, voter__voter_id=voter_id, date=date, amount=amount).exists() if exists: update_count += 1 action = 'update' else: create_count += 1 action = 'create' if len(preview_data) < 10: preview_data.append({ 'action': action, 'identifier': f"Voter: {voter_id}", 'details': f"Date: {date}, Amount: {amount}" }) context = self.admin_site.each_context(request) context.update({ 'title': "Import Preview", 'total_count': total_count, 'create_count': create_count, 'update_count': update_count, 'preview_data': preview_data, 'mapping': mapping, 'file_path': file_path, 'tenant_id': tenant_id, 'action_url': request.path, 'opts': self.model._meta, }) return render(request, "admin/import_preview.html", context) except Exception as e: self.message_user(request, f"Error processing preview: {e}", level=messages.ERROR) return redirect("..") elif "_import" in request.POST: file_path = request.POST.get('file_path') tenant_id = request.POST.get('tenant') tenant = Tenant.objects.get(id=tenant_id) mapping = {} for field_name, _ in DONATION_MAPPABLE_FIELDS: mapping[field_name] = request.POST.get(f'map_{field_name}') try: with open(file_path, 'r', encoding='UTF-8') as f: reader = csv.DictReader(f) count = 0 errors = 0 failed_rows = [] for row in reader: try: voter_id = row.get(mapping.get('voter_id')) if mapping.get('voter_id') else None if not voter_id: row["Import Error"] = "Missing voter ID" failed_rows.append(row) errors += 1 continue try: voter = Voter.objects.get(tenant=tenant, voter_id=voter_id) except Voter.DoesNotExist: row["Import Error"] = f"Voter {voter_id} not found" failed_rows.append(row) errors += 1 continue date = row.get(mapping.get('date')) amount = row.get(mapping.get('amount')) method_name = row.get(mapping.get('method')) if not date or not amount: row["Import Error"] = "Missing date or amount" failed_rows.append(row) errors += 1 continue method = None if method_name and method_name.strip(): method, _ = DonationMethod.objects.get_or_create( tenant=tenant, name=method_name ) defaults = {} if method: defaults['method'] = method Donation.objects.update_or_create( voter=voter, date=date, amount=amount, defaults=defaults ) count += 1 except Exception as e: logger.error(f"Error importing: {e}") row["Import Error"] = str(e) failed_rows.append(row) errors += 1 if os.path.exists(file_path): os.remove(file_path) self.message_user(request, f"Successfully imported {count} donations.") # Optimization: Limit error log size in session to avoid overflow request.session[f"{self.model._meta.model_name}_import_errors"] = failed_rows[:1000] request.session.modified = True logger.info(f"Stored {len(failed_rows)} failed rows in session for {self.model._meta.model_name}") if errors > 0: error_url = reverse("admin:donation-download-errors") self.message_user(request, mark_safe(f"Failed to import {errors} rows. Download failed records"), level=messages.WARNING) return redirect("..") except Exception as e: self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) return redirect("..") else: form = DonationImportForm(request.POST, request.FILES) if form.is_valid(): csv_file = request.FILES['file'] tenant = form.cleaned_data['tenant'] if not csv_file.name.endswith('.csv'): self.message_user(request, "Please upload a CSV file.", level=messages.ERROR) return redirect("..") with tempfile.NamedTemporaryFile(delete=False, suffix='.csv') as tmp: for chunk in csv_file.chunks(): tmp.write(chunk) file_path = tmp.name with open(file_path, 'r', encoding='UTF-8') as f: reader = csv.reader(f) headers = next(reader) context = self.admin_site.each_context(request) context.update({ 'title': "Map Donation Fields", 'headers': headers, 'model_fields': DONATION_MAPPABLE_FIELDS, 'tenant_id': tenant.id, 'file_path': file_path, 'action_url': request.path, 'opts': self.model._meta, }) return render(request, "admin/import_mapping.html", context) else: form = DonationImportForm() context = self.admin_site.each_context(request) context['form'] = form context['title'] = "Import Donations" context['opts'] = self.model._meta return render(request, "admin/import_csv.html", context) @admin.register(Interaction) class InteractionAdmin(BaseImportAdminMixin, admin.ModelAdmin): list_display = ('id', 'voter', 'volunteer', 'type', 'date', 'description') list_filter = ('voter__tenant', 'type', 'date', 'volunteer') search_fields = ('voter__first_name', 'voter__last_name', 'voter__voter_id', 'description', 'volunteer__first_name', 'volunteer__last_name') autocomplete_fields = ["voter", "volunteer"] change_list_template = "admin/interaction_change_list.html" def get_urls(self): urls = super().get_urls() my_urls = [ path('download-errors/', self.admin_site.admin_view(self.download_errors), name='interaction-download-errors'), path('import-interactions/', self.admin_site.admin_view(self.import_interactions), name='import-interactions'), ] return my_urls + urls def import_interactions(self, request): if request.method == "POST": if "_preview" in request.POST: file_path = request.POST.get('file_path') tenant_id = request.POST.get('tenant') tenant = Tenant.objects.get(id=tenant_id) mapping = {} for field_name, _ in INTERACTION_MAPPABLE_FIELDS: mapping[field_name] = request.POST.get(f'map_{field_name}') try: with open(file_path, 'r', encoding='UTF-8') as f: reader = csv.DictReader(f) total_count = 0 create_count = 0 update_count = 0 preview_data = [] for row in reader: total_count += 1 voter_id = row.get(mapping.get('voter_id')) date = row.get(mapping.get('date')) exists = False if voter_id and date: exists = Interaction.objects.filter(voter__tenant=tenant, voter__voter_id=voter_id, date=date).exists() if exists: update_count += 1 action = 'update' else: create_count += 1 action = 'create' if len(preview_data) < 10: preview_data.append({ 'action': action, 'identifier': f"Voter: {voter_id}", 'details': f"Date: {date}, Desc: {row.get(mapping.get('description', '')) or ''}" }) context = self.admin_site.each_context(request) context.update({ 'title': "Import Preview", 'total_count': total_count, 'create_count': create_count, 'update_count': update_count, 'preview_data': preview_data, 'mapping': mapping, 'file_path': file_path, 'tenant_id': tenant_id, 'action_url': request.path, 'opts': self.model._meta, }) return render(request, "admin/import_preview.html", context) except Exception as e: self.message_user(request, f"Error processing preview: {e}", level=messages.ERROR) return redirect("..") elif "_import" in request.POST: file_path = request.POST.get('file_path') tenant_id = request.POST.get('tenant') tenant = Tenant.objects.get(id=tenant_id) mapping = {} for field_name, _ in INTERACTION_MAPPABLE_FIELDS: mapping[field_name] = request.POST.get(f'map_{field_name}') try: with open(file_path, 'r', encoding='UTF-8') as f: reader = csv.DictReader(f) count = 0 errors = 0 failed_rows = [] for row in reader: try: voter_id = row.get(mapping.get('voter_id')) if mapping.get('voter_id') else None if not voter_id: row["Import Error"] = "Missing voter ID" failed_rows.append(row) errors += 1 continue try: voter = Voter.objects.get(tenant=tenant, voter_id=voter_id) except Voter.DoesNotExist: row["Import Error"] = f"Voter {voter_id} not found" failed_rows.append(row) errors += 1 continue date = row.get(mapping.get('date')) type_name = row.get(mapping.get('type')) volunteer_email = row.get(mapping.get('volunteer_email')) description = row.get(mapping.get('description')) notes = row.get(mapping.get('notes')) if not date or not description: row["Import Error"] = "Missing date or description" failed_rows.append(row) errors += 1 continue volunteer = None if volunteer_email and volunteer_email.strip(): try: volunteer = Volunteer.objects.get(tenant=tenant, email=volunteer_email.strip()) except Volunteer.DoesNotExist: pass interaction_type = None if type_name and type_name.strip(): interaction_type, _ = InteractionType.objects.get_or_create( tenant=tenant, name=type_name ) defaults = {} if volunteer: defaults['volunteer'] = volunteer if interaction_type: defaults['type'] = interaction_type if description and description.strip(): defaults['description'] = description if notes and notes.strip(): defaults['notes'] = notes Interaction.objects.update_or_create( voter=voter, date=date, defaults=defaults ) count += 1 except Exception as e: logger.error(f"Error importing: {e}") row["Import Error"] = str(e) failed_rows.append(row) errors += 1 if os.path.exists(file_path): os.remove(file_path) self.message_user(request, f"Successfully imported {count} interactions.") # Optimization: Limit error log size in session to avoid overflow request.session[f"{self.model._meta.model_name}_import_errors"] = failed_rows[:1000] request.session.modified = True logger.info(f"Stored {len(failed_rows)} failed rows in session for {self.model._meta.model_name}") if errors > 0: error_url = reverse("admin:interaction-download-errors") self.message_user(request, mark_safe(f"Failed to import {errors} rows. Download failed records"), level=messages.WARNING) return redirect("..") except Exception as e: self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) return redirect("..") else: form = InteractionImportForm(request.POST, request.FILES) if form.is_valid(): csv_file = request.FILES['file'] tenant = form.cleaned_data['tenant'] if not csv_file.name.endswith('.csv'): self.message_user(request, "Please upload a CSV file.", level=messages.ERROR) return redirect("..") with tempfile.NamedTemporaryFile(delete=False, suffix='.csv') as tmp: for chunk in csv_file.chunks(): tmp.write(chunk) file_path = tmp.name with open(file_path, 'r', encoding='UTF-8') as f: reader = csv.reader(f) headers = next(reader) context = self.admin_site.each_context(request) context.update({ 'title': "Map Interaction Fields", 'headers': headers, 'model_fields': INTERACTION_MAPPABLE_FIELDS, 'tenant_id': tenant.id, 'file_path': file_path, 'action_url': request.path, 'opts': self.model._meta, }) return render(request, "admin/import_mapping.html", context) else: form = InteractionImportForm() context = self.admin_site.each_context(request) context['form'] = form context['title'] = "Import Interactions" context['opts'] = self.model._meta return render(request, "admin/import_csv.html", context) @admin.register(VoterLikelihood) class VoterLikelihoodAdmin(BaseImportAdminMixin, admin.ModelAdmin): list_display = ('id', 'voter', 'election_type', 'likelihood') list_filter = ('voter__tenant', 'election_type', 'likelihood') search_fields = ('voter__first_name', 'voter__last_name', 'voter__voter_id') autocomplete_fields = ["voter"] change_list_template = "admin/voterlikelihood_change_list.html" def get_urls(self): urls = super().get_urls() my_urls = [ path('download-errors/', self.admin_site.admin_view(self.download_errors), name='voterlikelihood-download-errors'), path('import-likelihoods/', self.admin_site.admin_view(self.import_likelihoods), name='import-likelihoods'), ] return my_urls + urls def import_likelihoods(self, request): if request.method == "POST": if "_preview" in request.POST: file_path = request.POST.get('file_path') tenant_id = request.POST.get('tenant') tenant = Tenant.objects.get(id=tenant_id) mapping = {k: request.POST.get(f"map_{k}") for k, _ in VOTER_LIKELIHOOD_MAPPABLE_FIELDS if request.POST.get(f"map_{k}")} try: with open(file_path, 'r', encoding='utf-8-sig') as f: total_count = sum(1 for line in f) - 1 f.seek(0) reader = csv.DictReader(f) preview_rows = [] voter_ids_for_preview = set() election_types_for_preview = set() v_id_col = mapping.get('voter_id') et_col = mapping.get('election_type') if not v_id_col or not et_col: raise ValueError("Missing mapping for Voter ID or Election Type") for i, row in enumerate(reader): if i < 10: preview_rows.append(row) v_id = row.get(v_id_col) et_name = row.get(et_col) if v_id: voter_ids_for_preview.add(str(v_id).strip()) if et_name: election_types_for_preview.add(str(et_name).strip()) else: break existing_likelihoods = set(VoterLikelihood.objects.filter( voter__tenant=tenant, voter__voter_id__in=voter_ids_for_preview, election_type__name__in=election_types_for_preview ).values_list("voter__voter_id", "election_type__name")) preview_data = [] for row in preview_rows: v_id = str(row.get(v_id_col, '')).strip() et_name = str(row.get(et_col, '')).strip() action = "update" if (v_id, et_name) in existing_likelihoods else "create" preview_data.append({ "action": action, "identifier": f"Voter: {v_id}, Election: {et_name}", "details": f"Likelihood: {row.get(mapping.get('likelihood', '')) or ''}" }) context = self.admin_site.each_context(request) context.update({ "title": "Import Preview", "total_count": total_count, "create_count": "N/A", "update_count": "N/A", "preview_data": preview_data, "mapping": mapping, "file_path": file_path, "tenant_id": tenant_id, "action_url": request.path, "opts": self.model._meta, }) return render(request, "admin/import_preview.html", context) except Exception as e: self.message_user(request, f"Error processing preview: {e}", level=messages.ERROR) return redirect("..") elif "_import" in request.POST: file_path = request.POST.get('file_path') tenant_id = request.POST.get('tenant') tenant = Tenant.objects.get(id=tenant_id) mapping = {k: request.POST.get(f"map_{k}") for k, _ in VOTER_LIKELIHOOD_MAPPABLE_FIELDS if request.POST.get(f"map_{k}")} try: count = 0 created_count = 0 updated_count = 0 skipped_no_change = 0 skipped_no_id = 0 errors = 0 failed_rows = [] batch_size = 2000 likelihood_choices = dict(VoterLikelihood.LIKELIHOOD_CHOICES) likelihood_reverse = {v.lower(): k for k, v in likelihood_choices.items()} election_types = {et.name: et for et in ElectionType.objects.filter(tenant=tenant)} with open(file_path, "r", encoding="utf-8-sig") as f: raw_reader = csv.reader(f) headers = next(raw_reader) h_idx = {h: i for i, h in enumerate(headers)} v_id_col = mapping.get("voter_id") et_col = mapping.get("election_type") l_col = mapping.get("likelihood") if not v_id_col or not et_col or not l_col: raise ValueError("Missing mapping for Voter ID, Election Type, or Likelihood") v_idx = h_idx[v_id_col] e_idx = h_idx[et_col] l_idx = h_idx[l_col] total_processed = 0 for chunk in self.chunk_reader(raw_reader, batch_size): with transaction.atomic(): voter_ids = [] chunk_data = [] for row in chunk: if len(row) <= max(v_idx, e_idx, l_idx): continue v_id = row[v_idx].strip() et_name = row[e_idx].strip() l_val = row[l_idx].strip() if v_id and et_name and l_val: voter_ids.append(v_id) chunk_data.append((v_id, et_name, l_val, row)) else: skipped_no_id += 1 voters = {v.voter_id: v for v in Voter.objects.filter(tenant=tenant, voter_id__in=voter_ids).only("id", "voter_id")} et_names = [d[1] for d in chunk_data] existing_likelihoods = { (vl.voter.voter_id, vl.election_type.name): vl for vl in VoterLikelihood.objects.filter( voter__tenant=tenant, voter__voter_id__in=voter_ids, election_type__name__in=et_names ).only("id", "likelihood", "voter__voter_id", "election_type__name").select_related("voter", "election_type") } to_create = [] to_update = [] processed_in_batch = set() for v_id, et_name, l_val, row in chunk_data: total_processed += 1 try: if (v_id, et_name) in processed_in_batch: continue processed_in_batch.add((v_id, et_name)) voter = voters.get(v_id) if not voter: errors += 1 continue if et_name not in election_types: election_type, _ = ElectionType.objects.get_or_create(tenant=tenant, name=et_name) election_types[et_name] = election_type election_type = election_types[et_name] normalized_l = None l_val_lower = l_val.lower().replace(' ', '_') if l_val_lower in likelihood_choices: normalized_l = l_val_lower elif l_val_lower in likelihood_reverse: normalized_l = likelihood_reverse[l_val_lower] else: for k, v in likelihood_choices.items(): if v.lower() == l_val.lower(): normalized_l = k break if not normalized_l: errors += 1 continue vl = existing_likelihoods.get((v_id, et_name)) if not vl: to_create.append(VoterLikelihood(voter=voter, election_type=election_type, likelihood=normalized_l)) created_count += 1 elif vl.likelihood != normalized_l: vl.likelihood = normalized_l to_update.append(vl) updated_count += 1 else: skipped_no_change += 1 count += 1 except Exception as e: errors += 1 if to_create: VoterLikelihood.objects.bulk_create(to_create, batch_size=batch_size) if to_update: VoterLikelihood.objects.bulk_update(to_update, ["likelihood"], batch_size=batch_size) print(f"DEBUG: Likelihood import progress: {total_processed} processed. {count} created/updated.") if os.path.exists(file_path): os.remove(file_path) self.message_user(request, f"Import complete: {count} likelihoods created/updated. ({created_count} new, {updated_count} updated, {skipped_no_change} skipped, {errors} errors)") return redirect("..") except Exception as e: self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) return redirect("..") else: form = VoterLikelihoodImportForm(request.POST, request.FILES) if form.is_valid(): csv_file = request.FILES['file'] tenant = form.cleaned_data['tenant'] if not csv_file.name.endswith('.csv'): self.message_user(request, "Please upload a CSV file.", level=messages.ERROR) return redirect("..") with tempfile.NamedTemporaryFile(delete=False, suffix='.csv') as tmp: for chunk in csv_file.chunks(): tmp.write(chunk) file_path = tmp.name with open(file_path, 'r', encoding='utf-8-sig') as f: reader = csv.reader(f) headers = next(reader) context = self.admin_site.each_context(request) context.update({ 'title': "Map Likelihood Fields", 'headers': headers, 'model_fields': VOTER_LIKELIHOOD_MAPPABLE_FIELDS, 'tenant_id': tenant.id, 'file_path': file_path, 'action_url': request.path, 'opts': self.model._meta, }) return render(request, "admin/import_mapping.html", context) else: form = VoterLikelihoodImportForm() context = self.admin_site.each_context(request) context['form'] = form context['title'] = "Import Likelihoods" context['opts'] = self.model._meta return render(request, "admin/import_csv.html", context) @admin.register(CampaignSettings) class CampaignSettingsAdmin(admin.ModelAdmin): list_display = ('tenant', 'donation_goal', 'twilio_from_number', 'timezone') list_filter = ('tenant',) fields = ('tenant', 'donation_goal', 'twilio_account_sid', 'twilio_auth_token', 'twilio_from_number', 'timezone') @admin.register(VotingRecord) class VotingRecordAdmin(BaseImportAdminMixin, admin.ModelAdmin): list_display = ('voter', 'election_date', 'election_description', 'primary_party') list_filter = ('voter__tenant', 'election_date', 'primary_party') search_fields = ('voter__first_name', 'voter__last_name', 'voter__voter_id', 'election_description') autocomplete_fields = ["voter"] change_list_template = "admin/votingrecord_change_list.html" def changelist_view(self, request, extra_context=None): extra_context = extra_context or {} from core.models import Tenant extra_context["tenants"] = Tenant.objects.all() return super().changelist_view(request, extra_context=extra_context) def get_urls(self): urls = super().get_urls() my_urls = [ path('download-errors/', self.admin_site.admin_view(self.download_errors), name='votingrecord-download-errors'), path('import-voting-records/', self.admin_site.admin_view(self.import_voting_records), name='import-voting-records'), ] return my_urls + urls def import_voting_records(self, request): if request.method == "POST": if "_preview" in request.POST: file_path = request.POST.get('file_path') tenant_id = request.POST.get('tenant') tenant = Tenant.objects.get(id=tenant_id) mapping = {k: request.POST.get(f"map_{k}") for k, _ in VOTING_RECORD_MAPPABLE_FIELDS if request.POST.get(f"map_{k}")} try: with open(file_path, 'r', encoding='utf-8-sig') as f: total_count = sum(1 for line in f) - 1 f.seek(0) reader = csv.DictReader(f) preview_rows = [] voter_ids_for_preview = set() v_id_col = mapping.get('voter_id') ed_col = mapping.get('election_date') desc_col = mapping.get('election_description') if not v_id_col or not ed_col or not desc_col: raise ValueError("Missing mapping for Voter ID, Election Date, or Description") for i, row in enumerate(reader): if i < 10: preview_rows.append(row) v_id = row.get(v_id_col) if v_id: voter_ids_for_preview.add(str(v_id).strip()) else: break existing_records = set(VotingRecord.objects.filter( voter__tenant=tenant, voter__voter_id__in=voter_ids_for_preview ).values_list("voter__voter_id", "election_date", "election_description")) preview_data = [] for row in preview_rows: v_id = str(row.get(v_id_col, '')).strip() e_date_raw = row.get(ed_col) e_desc = str(row.get(desc_col, '')).strip() e_date = None if e_date_raw: for fmt in ["%Y-%m-%d", "%m/%d/%Y", "%d/%m/%Y", "%Y/%m/%d"]: try: e_date = datetime.strptime(str(e_date_raw).strip(), fmt).date() break except: continue action = "update" if (v_id, e_date, e_desc) in existing_records else "create" preview_data.append({ "action": action, "identifier": f"Voter: {v_id}, Election: {e_desc}", "details": f"Date: {e_date or e_date_raw}" }) context = self.admin_site.each_context(request) context.update({ "title": "Import Preview", "total_count": total_count, "create_count": "N/A", "update_count": "N/A", "preview_data": preview_data, "mapping": mapping, "file_path": file_path, "tenant_id": tenant_id, "action_url": request.path, "opts": self.model._meta, }) return render(request, "admin/import_preview.html", context) except Exception as e: self.message_user(request, f"Error processing preview: {e}", level=messages.ERROR) return redirect("..") elif "_import" in request.POST: file_path = request.POST.get('file_path') tenant_id = request.POST.get('tenant') tenant = Tenant.objects.get(id=tenant_id) mapping = {k: request.POST.get(f"map_{k}") for k, _ in VOTING_RECORD_MAPPABLE_FIELDS if request.POST.get(f"map_{k}")} try: count = 0 created_count = 0 updated_count = 0 skipped_no_change = 0 errors = 0 batch_size = 2000 with open(file_path, "r", encoding="utf-8-sig") as f: raw_reader = csv.reader(f) headers = next(raw_reader) h_idx = {h: i for i, h in enumerate(headers)} v_id_col = mapping.get("voter_id") ed_col = mapping.get("election_date") desc_col = mapping.get("election_description") party_col = mapping.get("primary_party") if not v_id_col or not ed_col or not desc_col: raise ValueError("Missing mapping for Voter ID, Election Date, or Description") v_idx = h_idx[v_id_col] ed_idx = h_idx[ed_col] desc_idx = h_idx[desc_col] p_idx = h_idx.get(party_col) total_processed = 0 for chunk in self.chunk_reader(raw_reader, batch_size): with transaction.atomic(): voter_ids = [row[v_idx].strip() for row in chunk if len(row) > v_idx and row[v_idx].strip()] voters = {v.voter_id: v for v in Voter.objects.filter(tenant=tenant, voter_id__in=voter_ids).only("id", "voter_id")} existing_records = { (vr.voter.voter_id, vr.election_date, vr.election_description): vr for vr in VotingRecord.objects.filter( voter__tenant=tenant, voter__voter_id__in=voter_ids ).only("id", "election_date", "election_description", "voter__voter_id").select_related("voter") } to_create = [] to_update = [] processed_in_batch = set() for row in chunk: total_processed += 1 try: if len(row) <= max(v_idx, ed_idx, desc_idx): continue v_id = row[v_idx].strip() raw_ed = row[ed_idx].strip() desc = row[desc_idx].strip() party = row[p_idx].strip() if p_idx is not None and len(row) > p_idx else "" if not v_id or not raw_ed or not desc: continue if (v_id, raw_ed, desc) in processed_in_batch: continue processed_in_batch.add((v_id, raw_ed, desc)) voter = voters.get(v_id) if not voter: errors += 1 continue e_date = None for fmt in ["%Y-%m-%d", "%m/%d/%Y", "%d/%m/%Y", "%Y/%m/%d"]: try: e_date = datetime.strptime(raw_ed, fmt).date() break except: continue if not e_date: errors += 1 continue vr = existing_records.get((v_id, e_date, desc)) if not vr: to_create.append(VotingRecord(voter=voter, election_date=e_date, election_description=desc, primary_party=party)) created_count += 1 elif vr.primary_party != party: vr.primary_party = party to_update.append(vr) updated_count += 1 else: skipped_no_change += 1 count += 1 except Exception as e: errors += 1 if to_create: VotingRecord.objects.bulk_create(to_create, batch_size=batch_size) if to_update: VotingRecord.objects.bulk_update(to_update, ["primary_party"], batch_size=batch_size) print(f"DEBUG: Voting record import progress: {total_processed} processed. {count} created/updated.") if os.path.exists(file_path): os.remove(file_path) self.message_user(request, f"Import complete: {count} voting records created/updated. ({created_count} new, {updated_count} updated, {skipped_no_change} skipped, {errors} errors)") return redirect("..") except Exception as e: self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) return redirect("..") else: form = VotingRecordImportForm(request.POST, request.FILES) if form.is_valid(): csv_file = request.FILES['file'] tenant = form.cleaned_data['tenant'] if not csv_file.name.endswith('.csv'): self.message_user(request, "Please upload a CSV file.", level=messages.ERROR) return redirect("..") with tempfile.NamedTemporaryFile(delete=False, suffix='.csv') as tmp: for chunk in csv_file.chunks(): tmp.write(chunk) file_path = tmp.name with open(file_path, 'r', encoding='utf-8-sig') as f: reader = csv.reader(f) headers = next(reader) context = self.admin_site.each_context(request) context.update({ 'title': "Map Voting Record Fields", 'headers': headers, 'model_fields': VOTING_RECORD_MAPPABLE_FIELDS, 'tenant_id': tenant.id, 'file_path': file_path, 'action_url': request.path, 'opts': self.model._meta, }) return render(request, "admin/import_mapping.html", context) else: form = VotingRecordImportForm() context = self.admin_site.each_context(request) context['form'] = form context['title'] = "Import Voting Records" context['opts'] = self.model._meta return render(request, "admin/import_csv.html", context)