621 lines
32 KiB
Python
621 lines
32 KiB
Python
from django import forms
|
|
from decimal import Decimal, InvalidOperation
|
|
from datetime import datetime, date
|
|
import csv
|
|
import io
|
|
import logging
|
|
import tempfile
|
|
import os
|
|
import zoneinfo
|
|
from django.db import transaction
|
|
from django.http import HttpResponse
|
|
from django.utils.safestring import mark_safe
|
|
from django.utils.dateparse import parse_date, parse_datetime
|
|
from django.utils import timezone as django_timezone
|
|
from django.contrib import admin, messages
|
|
from django.urls import path, reverse
|
|
from django.shortcuts import render, redirect
|
|
from django.template.response import TemplateResponse
|
|
|
|
from .models import (
|
|
format_phone_number,
|
|
Tenant, TenantUserRole, InteractionType, DonationMethod, ElectionType, EventType, Voter,
|
|
VotingRecord, Event, EventParticipation, Donation, Interaction, VoterLikelihood, CampaignSettings,
|
|
Interest, Volunteer, VolunteerEvent, ParticipationStatus, VolunteerRole, ScheduledCall
|
|
)
|
|
from .forms import (
|
|
VoterImportForm, EventImportForm, EventParticipationImportForm,
|
|
DonationImportForm, InteractionImportForm, VoterLikelihoodImportForm,
|
|
VolunteerImportForm, VotingRecordImportForm
|
|
)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
def parse_any_date(date_str, tz_name=None):
|
|
if not date_str or not isinstance(date_str, str): return None
|
|
date_str = date_str.strip()
|
|
if not date_str: return None
|
|
dt = parse_datetime(date_str)
|
|
if dt:
|
|
if django_timezone.is_naive(dt) and tz_name:
|
|
try: dt = django_timezone.make_aware(dt, zoneinfo.ZoneInfo(tz_name))
|
|
except: pass
|
|
return dt
|
|
d = parse_date(date_str)
|
|
if d: return d
|
|
formats = ["%m/%d/%Y", "%m/%d/%y", "%d/%m/%Y", "%d/%m/%y", "%Y-%m-%d", "%m-%d-%Y", "%d-%m-%Y", "%Y/%m/%d", "%m/%d/%Y %H:%M:%S", "%Y-%m-%d %H:%M:%S", "%m/%d/%Y %I:%M %p", "%m/%d/%Y %H:%M", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%dT%H:%M:%SZ"]
|
|
for fmt in formats:
|
|
try:
|
|
dt = datetime.strptime(date_str, fmt)
|
|
if any(x in fmt for x in ["%H", "%I", "T"]):
|
|
if django_timezone.is_naive(dt) and tz_name:
|
|
try: dt = django_timezone.make_aware(dt, zoneinfo.ZoneInfo(tz_name))
|
|
except: pass
|
|
return dt
|
|
return dt.date()
|
|
except ValueError: continue
|
|
return None
|
|
|
|
def _robust_decode(content):
|
|
if not content: return ""
|
|
for enc in ["utf-8-sig", "utf-8", "iso-8859-1", "windows-1252"]:
|
|
try: return content.decode(enc)
|
|
except UnicodeDecodeError: continue
|
|
return content.decode("utf-8", errors="replace")
|
|
|
|
def _read_csv_robust(file_path):
|
|
"""
|
|
Optimized version: Read and decode the file into memory once,
|
|
but return a StringIO for stream-like processing.
|
|
"""
|
|
with open(file_path, "rb") as f:
|
|
content = _robust_decode(f.read())
|
|
return io.StringIO(content)
|
|
|
|
class BaseImportAdminMixin:
|
|
actions = ["export_as_csv"]
|
|
|
|
def export_as_csv(self, request, queryset):
|
|
meta = self.model._meta
|
|
field_names = [field.name for field in meta.fields]
|
|
include_voter_id = "voter" in field_names and self.model != Voter
|
|
response = HttpResponse(content_type="text/csv")
|
|
response["Content-Disposition"] = f"attachment; filename={meta.model_name}_export.csv"
|
|
writer = csv.writer(response)
|
|
headers = []
|
|
for name in field_names:
|
|
headers.append(name)
|
|
if name == "voter" and include_voter_id: headers.append("voter_id")
|
|
writer.writerow(headers)
|
|
for obj in queryset:
|
|
row = []
|
|
for field in field_names:
|
|
value = getattr(obj, field)
|
|
if isinstance(value, (datetime, date)): value = value.strftime("%Y-%m-%d %H:%M:%S") if isinstance(value, datetime) else value.strftime("%Y-%m-%d")
|
|
elif hasattr(value, "id"): value = str(value)
|
|
row.append(value)
|
|
if field == "voter" and include_voter_id: row.append(obj.voter.voter_id if obj.voter else "")
|
|
writer.writerow(row)
|
|
return response
|
|
export_as_csv.short_description = "Export Selected as CSV"
|
|
|
|
def download_errors(self, request):
|
|
failed_rows = request.session.get(f"{self.model._meta.model_name}_import_errors", [])
|
|
if not failed_rows:
|
|
self.message_user(request, "No errors found.", level=messages.WARNING)
|
|
return redirect("../")
|
|
output = io.StringIO()
|
|
if failed_rows:
|
|
writer = csv.DictWriter(output, fieldnames=failed_rows[0].keys())
|
|
writer.writeheader()
|
|
writer.writerows(failed_rows)
|
|
response = HttpResponse(output.getvalue(), content_type='text/csv')
|
|
response['Content-Disposition'] = f'attachment; filename="{self.model._meta.model_name}_import_errors.csv"'
|
|
return response
|
|
|
|
VOTER_MAPPABLE_FIELDS = [
|
|
('voter_id', 'Voter ID'),
|
|
('first_name', 'First Name'),
|
|
('last_name', 'Last Name'),
|
|
('nickname', 'Nickname'),
|
|
('birthdate', 'Birthdate'),
|
|
('address_street', 'Street Address'),
|
|
('city', 'City'),
|
|
('state', 'State'),
|
|
('zip_code', 'Zip Code'),
|
|
('phone', 'Phone'),
|
|
('email', 'Email'),
|
|
('is_targeted', 'Is Targeted'),
|
|
('target_door_visit', 'Target Door Visit'),
|
|
('candidate_support', 'Candidate Support'),
|
|
('yard_sign', 'Yard Sign'),
|
|
('ever_had_yard_sign', 'Ever Had Yard Sign'),
|
|
('ever_had_large_sign', 'Ever Had Large Sign'),
|
|
('is_inactive', 'Is Inactive'),
|
|
('door_visit', 'Door Visit'),
|
|
('voted', 'Voted'),
|
|
('neighborhood', 'Neighborhood'),
|
|
('district', 'District'),
|
|
('precinct', 'Precinct'),
|
|
('registration_date', 'Registration Date'),
|
|
('call_queue_status', 'Call Queue Status'),
|
|
]
|
|
|
|
INTERACTION_MAPPABLE_FIELDS = [('voter_id', 'Voter ID'), ('volunteer_email', 'Volunteer Email'), ('date', 'Date'), ('type', 'Type'), ('description', 'Description'), ('notes', 'Notes')]
|
|
VOLUNTEER_MAPPABLE_FIELDS = [('first_name', 'First Name'), ('last_name', 'Last Name'), ('email', 'Email'), ('phone', 'Phone')]
|
|
VOTER_LIKELIHOOD_MAPPABLE_FIELDS = [('voter_id', 'Voter ID'), ('election_type', 'Election Type'), ('likelihood', 'Likelihood')]
|
|
VOTING_RECORD_MAPPABLE_FIELDS = [('voter_id', 'Voter ID'), ('election_date', 'Election Date'), ('election_description', 'Description'), ('primary_party', 'Primary Party')]
|
|
EVENT_MAPPABLE_FIELDS = [('name', 'Name'), ('date', 'Date'), ('event_type', 'Event Type'), ('location_name', 'Location'), ('address', 'Address'), ('city', 'City'), ('state', 'State'), ('zip_code', 'Zip Code'), ('start_time', 'Start Time'), ('end_time', 'End Time')]
|
|
|
|
@admin.register(Voter)
|
|
class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin):
|
|
list_display = ('voter_id', 'first_name', 'last_name', 'city', 'state', 'is_inactive', 'target_door_visit', 'ever_had_yard_sign', 'ever_had_large_sign', 'tenant')
|
|
list_filter = ('tenant', 'is_inactive', 'ever_had_yard_sign', 'ever_had_large_sign', 'target_door_visit', 'candidate_support', 'call_queue_status')
|
|
search_fields = ('voter_id', 'first_name', 'last_name', 'email', 'phone')
|
|
change_list_template = "admin/voter_change_list.html"
|
|
|
|
def get_urls(self):
|
|
return [
|
|
path('download-errors/', self.admin_site.admin_view(self.download_errors), name='voter-download-errors'),
|
|
path('import-voters/', self.admin_site.admin_view(self.import_voters), name='import-voters')
|
|
] + super().get_urls()
|
|
|
|
def import_voters(self, request):
|
|
if request.method == "POST":
|
|
if "_preview" in request.POST:
|
|
file_path, tenant_id = request.POST.get("file_path"), request.POST.get("tenant")
|
|
tenant, mapping = Tenant.objects.get(id=tenant_id), {fn: request.POST.get(f"map_{fn}") for fn, _ in VOTER_MAPPABLE_FIELDS}
|
|
try:
|
|
with _read_csv_robust(file_path) as f:
|
|
total_count = sum(1 for line in f) - 1
|
|
f.seek(0)
|
|
reader = csv.DictReader(f)
|
|
preview_rows, v_ids = [], []
|
|
for i, row in enumerate(reader):
|
|
if i < 10:
|
|
preview_rows.append(row)
|
|
vid = row.get(mapping.get("voter_id"))
|
|
if vid: v_ids.append(vid.strip())
|
|
else: break
|
|
existing = set(Voter.objects.filter(tenant=tenant, voter_id__in=v_ids).values_list("voter_id", flat=True))
|
|
preview_data = [{
|
|
"action": "update" if r.get(mapping.get("voter_id"), "").strip() in existing else "create",
|
|
"identifier": f"Voter ID: {r.get(mapping.get('voter_id'))}",
|
|
"details": f"Name: {r.get(mapping.get('first_name', ''))} {r.get(mapping.get('last_name', ''))}"
|
|
} for r in preview_rows]
|
|
context = self.admin_site.each_context(request)
|
|
context.update({
|
|
"title": "Import Preview",
|
|
"total_count": total_count,
|
|
"create_count": sum(1 for d in preview_data if d['action'] == 'create'),
|
|
"update_count": sum(1 for d in preview_data if d['action'] == 'update'),
|
|
"preview_data": preview_data,
|
|
"mapping": mapping,
|
|
"file_path": file_path,
|
|
"tenant_id": tenant_id,
|
|
"action_url": request.path,
|
|
"opts": self.model._meta
|
|
})
|
|
return render(request, "admin/import_preview.html", context)
|
|
except Exception as e:
|
|
self.message_user(request, f"Error: {e}", level=messages.ERROR)
|
|
return redirect("../")
|
|
|
|
elif "_import" in request.POST:
|
|
file_path, tenant_id = request.POST.get("file_path"), request.POST.get("tenant")
|
|
tenant, mapping = Tenant.objects.get(id=tenant_id), {fn: request.POST.get(f"map_{fn}") for fn, _ in VOTER_MAPPABLE_FIELDS}
|
|
try:
|
|
created, updated, errors, failed = 0, 0, 0, []
|
|
|
|
with _read_csv_robust(file_path) as f:
|
|
reader = csv.DictReader(f)
|
|
chunk_size = 500
|
|
chunk = []
|
|
for row in reader:
|
|
chunk.append(row)
|
|
if len(chunk) >= chunk_size:
|
|
c, u, e, f_rows = self._process_voter_chunk(tenant, mapping, chunk)
|
|
created += c; updated += u; errors += e; failed.extend(f_rows)
|
|
chunk = []
|
|
if chunk:
|
|
c, u, e, f_rows = self._process_voter_chunk(tenant, mapping, chunk)
|
|
created += c; updated += u; errors += e; failed.extend(f_rows)
|
|
|
|
# Efficient post-import cleanup for the entire tenant
|
|
self._run_voter_post_import_cleanup(tenant)
|
|
|
|
if os.path.exists(file_path): os.remove(file_path)
|
|
self.message_user(request, f"Import complete: {created} created, {updated} updated, {errors} errors")
|
|
request.session[f"{self.model._meta.model_name}_import_errors"] = failed[:1000]
|
|
request.session.modified = True
|
|
return redirect("../")
|
|
except Exception as e:
|
|
logger.error(f"Voter import failed: {e}", exc_info=True)
|
|
self.message_user(request, f"Error: {e}", level=messages.ERROR)
|
|
return redirect("../")
|
|
else:
|
|
form = VoterImportForm(request.POST, request.FILES)
|
|
if form.is_valid():
|
|
with tempfile.NamedTemporaryFile(delete=False, suffix='.csv') as tmp:
|
|
for chunk in request.FILES['file'].chunks(): tmp.write(chunk)
|
|
file_path = tmp.name
|
|
with _read_csv_robust(file_path) as f:
|
|
headers = next(csv.reader(f))
|
|
context = self.admin_site.each_context(request)
|
|
context.update({
|
|
"title": "Map Voter Fields",
|
|
"headers": headers,
|
|
"model_fields": VOTER_MAPPABLE_FIELDS,
|
|
"tenant_id": form.cleaned_data['tenant'].id,
|
|
"file_path": file_path,
|
|
"action_url": request.path,
|
|
"opts": self.model._meta
|
|
})
|
|
return render(request, "admin/import_mapping.html", context)
|
|
return render(request, "admin/import_csv.html", {'form': VoterImportForm(), 'title': "Import Voters", 'opts': self.model._meta, 'action_url': request.path})
|
|
|
|
def _process_voter_chunk(self, tenant, mapping, chunk):
|
|
created, updated, errors = 0, 0, 0
|
|
failed = []
|
|
voter_ids = [row.get(mapping.get("voter_id"), "").strip() for row in chunk if row.get(mapping.get("voter_id"))]
|
|
existing_voters = {v.voter_id: v for v in Voter.objects.filter(tenant=tenant, voter_id__in=voter_ids)}
|
|
|
|
to_create = []
|
|
to_update = []
|
|
|
|
# We'll use a transaction for each chunk to keep it atomic but not lock the whole table for long
|
|
with transaction.atomic():
|
|
for row in chunk:
|
|
try:
|
|
vid = row.get(mapping.get("voter_id"), "").strip()
|
|
if not vid:
|
|
row["Import Error"] = "Missing Voter ID"
|
|
failed.append(row); errors += 1; continue
|
|
|
|
defaults = {}
|
|
for fn, _ in VOTER_MAPPABLE_FIELDS:
|
|
if fn == "voter_id": continue
|
|
val = row.get(mapping.get(fn), "").strip()
|
|
if not val: continue
|
|
if fn in ["birthdate", "registration_date"]:
|
|
defaults[fn] = parse_any_date(val)
|
|
elif fn in ["is_targeted", "is_inactive", "target_door_visit", "door_visit", "voted"]:
|
|
defaults[fn] = val.lower() in ['true', '1', 'yes']
|
|
elif fn == "phone":
|
|
defaults[fn] = format_phone_number(val)
|
|
elif fn == "email":
|
|
defaults[fn] = val.lower()
|
|
elif fn == "call_queue_status":
|
|
# Try to match label if it's not a valid internal value
|
|
valid_keys = [c[0] for c in Voter.CALL_QUEUE_STATUS_CHOICES]
|
|
if val not in valid_keys:
|
|
label_map = {c[1].lower(): c[0] for c in Voter.CALL_QUEUE_STATUS_CHOICES}
|
|
if val.lower() in label_map:
|
|
defaults[fn] = label_map[val.lower()]
|
|
else:
|
|
defaults[fn] = val
|
|
else:
|
|
defaults[fn] = val
|
|
else:
|
|
defaults[fn] = val
|
|
|
|
if defaults.get("voted") is True:
|
|
defaults["target_door_visit"] = False
|
|
defaults["call_queue_status"] = "no_call_required"
|
|
|
|
voter = existing_voters.get(vid)
|
|
if voter:
|
|
for k, v in defaults.items(): setattr(voter, k, v)
|
|
voter._skip_geocode = True # Important for performance
|
|
to_update.append(voter)
|
|
updated += 1
|
|
else:
|
|
voter = Voter(tenant=tenant, voter_id=vid, **defaults)
|
|
voter._skip_geocode = True
|
|
to_create.append(voter)
|
|
created += 1
|
|
except Exception as e:
|
|
row["Import Error"] = str(e)
|
|
failed.append(row); errors += 1
|
|
|
|
if to_create:
|
|
Voter.objects.bulk_create(to_create)
|
|
if to_update:
|
|
# bulk_update requires specifying fields
|
|
fields = [fn for fn, _ in VOTER_MAPPABLE_FIELDS if fn != 'voter_id']
|
|
Voter.objects.bulk_update(to_update, fields)
|
|
|
|
return created, updated, errors, failed
|
|
|
|
def _run_voter_post_import_cleanup(self, tenant):
|
|
"""
|
|
Runs the logic that was previously in signals but optimized for bulk.
|
|
"""
|
|
from django.db.models import Exists, OuterRef
|
|
|
|
# 0. Ensure consistency for voters who voted
|
|
Voter.objects.filter(tenant=tenant, voted=True).update(
|
|
target_door_visit=False,
|
|
call_queue_status="no_call_required"
|
|
)
|
|
ScheduledCall.objects.filter(tenant=tenant, voter__voted=True, status="pending").update(status="cancelled")
|
|
|
|
# 1. Update target_door_visit logic (based on signal logic)
|
|
# Set target_door_visit = False if door_visit = False and someone in household is targeted or has support
|
|
# This is a bit complex to do in one query, but let's do the most important parts.
|
|
|
|
# Signal 1: Update target_door_visit = False if someone in household attended event or has support
|
|
subquery = Voter.objects.filter(
|
|
address_street=OuterRef('address_street'),
|
|
city=OuterRef('city'),
|
|
state=OuterRef('state'),
|
|
zip_code=OuterRef('zip_code'),
|
|
tenant=tenant,
|
|
is_targeted=True
|
|
)
|
|
|
|
# Set target_door_visit = False if NO ONE in household is targeted
|
|
Voter.objects.filter(
|
|
tenant=tenant,
|
|
door_visit=False,
|
|
target_door_visit=True
|
|
).annotate(has_targeted=Exists(subquery)).filter(has_targeted=False).update(target_door_visit=False)
|
|
|
|
# Signal 2: Update candidate_support to 'supporting' if someone in household has yard sign AND voter is > 30
|
|
from datetime import date
|
|
today = date.today()
|
|
thirty_years_ago = today.replace(year=today.year - 30) if today.month != 2 or today.day != 29 else today.replace(year=today.year - 30, day=28)
|
|
|
|
sign_subquery = Voter.objects.filter(
|
|
address_street=OuterRef('address_street'),
|
|
city=OuterRef('city'),
|
|
state=OuterRef('state'),
|
|
zip_code=OuterRef('zip_code'),
|
|
tenant=tenant,
|
|
yard_sign__in=['wants', 'has']
|
|
)
|
|
|
|
Voter.objects.filter(
|
|
tenant=tenant,
|
|
birthdate__lte=thirty_years_ago
|
|
).exclude(
|
|
candidate_support='supporting'
|
|
).annotate(household_has_sign=Exists(sign_subquery)).filter(household_has_sign=True).update(candidate_support='supporting')
|
|
|
|
class MassAssignVolunteerForm(forms.Form):
|
|
volunteer = forms.ModelChoiceField(queryset=Volunteer.objects.none(), required=True)
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
tenant_ids = kwargs.pop('tenant_ids', [])
|
|
super().__init__(*args, **kwargs)
|
|
if tenant_ids:
|
|
self.fields['volunteer'].queryset = Volunteer.objects.filter(tenant_id__in=tenant_ids).order_by('first_name', 'last_name')
|
|
else:
|
|
self.fields['volunteer'].queryset = Volunteer.objects.all().order_by('first_name', 'last_name')
|
|
|
|
|
|
@admin.register(Interaction)
|
|
class InteractionAdmin(BaseImportAdminMixin, admin.ModelAdmin):
|
|
list_display = ('voter', 'date', 'type', 'description', 'volunteer')
|
|
list_filter = ('voter__tenant', 'type', 'volunteer')
|
|
search_fields = ('voter__first_name', 'voter__last_name', 'voter__voter_id', 'description')
|
|
autocomplete_fields = ['voter', 'volunteer']
|
|
change_list_template = 'admin/interaction_change_list.html'
|
|
actions = ['mass_assign_volunteer']
|
|
|
|
@admin.action(description="Assign selected interactions to a volunteer")
|
|
def mass_assign_volunteer(self, request, queryset):
|
|
tenant_ids = list(queryset.values_list('voter__tenant_id', flat=True).distinct())
|
|
|
|
if 'apply' in request.POST:
|
|
form = MassAssignVolunteerForm(request.POST, tenant_ids=tenant_ids)
|
|
if form.is_valid():
|
|
volunteer = form.cleaned_data['volunteer']
|
|
updated = queryset.update(volunteer=volunteer)
|
|
self.message_user(request, f"Successfully assigned {updated} interactions to {volunteer}.", messages.SUCCESS)
|
|
return None
|
|
else:
|
|
form = MassAssignVolunteerForm(tenant_ids=tenant_ids)
|
|
|
|
return TemplateResponse(request, "admin/mass_assign_volunteer.html", {
|
|
'queryset': queryset,
|
|
'form': form,
|
|
'opts': self.model._meta,
|
|
'action_checkbox_name': admin.helpers.ACTION_CHECKBOX_NAME,
|
|
})
|
|
|
|
|
|
def get_urls(self):
|
|
return [
|
|
path('download-errors/', self.admin_site.admin_view(self.download_errors), name='interaction-download-errors'),
|
|
path('import-interactions/', self.admin_site.admin_view(self.import_interactions), name='import-interactions')
|
|
] + super().get_urls()
|
|
|
|
def import_interactions(self, request):
|
|
if request.method == "POST":
|
|
if "_preview" in request.POST:
|
|
file_path, tenant_id = request.POST.get('file_path'), request.POST.get('tenant')
|
|
tenant = Tenant.objects.get(id=tenant_id)
|
|
campaign_tz = getattr(tenant.settings, 'timezone', 'UTC')
|
|
mapping = {fn: request.POST.get(f'map_{fn}') for fn, _ in INTERACTION_MAPPABLE_FIELDS}
|
|
try:
|
|
with _read_csv_robust(file_path) as f:
|
|
reader = csv.DictReader(f)
|
|
total_count, create_count, update_count, preview_data = 0, 0, 0, []
|
|
for row in reader:
|
|
total_count += 1
|
|
vid, type_name, date_str = row.get(mapping.get('voter_id')), row.get(mapping.get('type')), row.get(mapping.get('date'))
|
|
parsed_date = parse_any_date(date_str, campaign_tz)
|
|
exists = False
|
|
if vid and type_name and parsed_date:
|
|
try:
|
|
voter = Voter.objects.get(tenant=tenant, voter_id=vid)
|
|
exists = Interaction.objects.filter(voter=voter, type__name=type_name, date=parsed_date).exists()
|
|
except: pass
|
|
if exists: update_count += 1
|
|
else: create_count += 1
|
|
if len(preview_data) < 10:
|
|
preview_data.append({'action': 'update' if exists else 'create', 'identifier': f"Voter ID: {vid}", 'details': f"Type: {type_name}, Date: {date_str}"})
|
|
context = self.admin_site.each_context(request)
|
|
context.update({'title': "Import Preview", 'total_count': total_count, 'create_count': create_count, 'update_count': update_count, 'preview_data': preview_data, 'mapping': mapping, 'file_path': file_path, 'tenant_id': tenant_id, 'action_url': request.path, 'opts': self.model._meta})
|
|
return render(request, "admin/import_preview.html", context)
|
|
except Exception as e:
|
|
self.message_user(request, f"Error: {e}", level=messages.ERROR)
|
|
return redirect("../")
|
|
|
|
elif "_import" in request.POST:
|
|
file_path, tenant_id = request.POST.get('file_path'), request.POST.get('tenant')
|
|
tenant = Tenant.objects.get(id=tenant_id)
|
|
campaign_tz = getattr(tenant.settings, 'timezone', 'UTC')
|
|
mapping = {fn: request.POST.get(f'map_{fn}') for fn, _ in INTERACTION_MAPPABLE_FIELDS}
|
|
try:
|
|
count, errors, failed = 0, 0, []
|
|
# Optimized to avoid loading ALL voters
|
|
with _read_csv_robust(file_path) as f:
|
|
reader = csv.DictReader(f)
|
|
chunk_size = 500
|
|
chunk = []
|
|
for row in reader:
|
|
chunk.append(row)
|
|
if len(chunk) >= chunk_size:
|
|
c, e, f_rows = self._process_interaction_chunk(tenant, mapping, chunk, campaign_tz)
|
|
count += c; errors += e; failed.extend(f_rows)
|
|
chunk = []
|
|
if chunk:
|
|
c, e, f_rows = self._process_interaction_chunk(tenant, mapping, chunk, campaign_tz)
|
|
count += c; errors += e; failed.extend(f_rows)
|
|
|
|
if os.path.exists(file_path): os.remove(file_path)
|
|
self.message_user(request, f"Imported {count} interactions, {errors} errors")
|
|
request.session[f"{self.model._meta.model_name}_import_errors"] = failed[:1000]
|
|
request.session.modified = True
|
|
return redirect("../")
|
|
except Exception as e:
|
|
self.message_user(request, f"Error: {e}", level=messages.ERROR)
|
|
return redirect("../")
|
|
else:
|
|
form = InteractionImportForm(request.POST, request.FILES)
|
|
if form.is_valid():
|
|
with tempfile.NamedTemporaryFile(delete=False, suffix='.csv') as tmp:
|
|
for chunk in request.FILES['file'].chunks(): tmp.write(chunk)
|
|
file_path = tmp.name
|
|
with _read_csv_robust(file_path) as f:
|
|
headers = next(csv.reader(f))
|
|
context = self.admin_site.each_context(request)
|
|
context.update({'title': "Map Interaction Fields", 'headers': headers, 'model_fields': INTERACTION_MAPPABLE_FIELDS, 'tenant_id': form.cleaned_data['tenant'].id, 'file_path': file_path, 'action_url': request.path, 'opts': self.model._meta})
|
|
return render(request, "admin/import_mapping.html", context)
|
|
return render(request, "admin/import_csv.html", {'form': InteractionImportForm(), 'title': "Import Interactions", 'opts': self.model._meta, 'action_url': request.path})
|
|
|
|
def _process_interaction_chunk(self, tenant, mapping, chunk, campaign_tz):
|
|
count, errors = 0, 0
|
|
failed = []
|
|
voter_ids = [row.get(mapping.get("voter_id"), "").strip() for row in chunk if row.get(mapping.get("voter_id"))]
|
|
voters = {v.voter_id: v for v in Voter.objects.filter(tenant=tenant, voter_id__in=voter_ids)}
|
|
|
|
# Pre-fetch interaction types
|
|
type_names = [row.get(mapping.get("type"), "").strip() for row in chunk if row.get(mapping.get("type"))]
|
|
types = {t.name: t for t in InteractionType.objects.filter(tenant=tenant, name__in=type_names)}
|
|
|
|
to_create = []
|
|
with transaction.atomic():
|
|
for row in chunk:
|
|
try:
|
|
vid, type_name, date_str = row.get(mapping.get('voter_id'), "").strip(), row.get(mapping.get('type'), "").strip(), row.get(mapping.get('date'), "").strip()
|
|
if not vid or not type_name or not date_str:
|
|
row["Import Error"] = "Missing fields"; failed.append(row); errors += 1; continue
|
|
|
|
voter = voters.get(vid)
|
|
if not voter:
|
|
row["Import Error"] = f"Voter {vid} not found"; failed.append(row); errors += 1; continue
|
|
|
|
it_type = types.get(type_name)
|
|
if not it_type:
|
|
it_type, created = InteractionType.objects.get_or_create(tenant=tenant, name=type_name)
|
|
types[type_name] = it_type
|
|
|
|
parsed_date = parse_any_date(date_str, campaign_tz)
|
|
if not parsed_date:
|
|
row["Import Error"] = f"Invalid date: {date_str}"; failed.append(row); errors += 1; continue
|
|
|
|
# Interaction model uses DateTimeField, so if we got a date, we should make it a datetime
|
|
if isinstance(parsed_date, date) and not isinstance(parsed_date, datetime):
|
|
parsed_date = datetime.combine(parsed_date, datetime.min.time())
|
|
if django_timezone.is_naive(parsed_date):
|
|
parsed_date = django_timezone.make_aware(parsed_date, zoneinfo.ZoneInfo(campaign_tz))
|
|
|
|
to_create.append(Interaction(
|
|
voter=voter,
|
|
type=it_type,
|
|
date=parsed_date,
|
|
description=row.get(mapping.get('description'), "")[:255],
|
|
notes=row.get(mapping.get('notes'), "")
|
|
))
|
|
count += 1
|
|
except Exception as e:
|
|
row["Import Error"] = str(e)
|
|
failed.append(row); errors += 1
|
|
|
|
if to_create:
|
|
Interaction.objects.bulk_create(to_create)
|
|
return count, errors, failed
|
|
|
|
@admin.register(DonationMethod)
|
|
class DonationMethodAdmin(admin.ModelAdmin):
|
|
list_display = ('name', 'tenant', 'is_active')
|
|
list_filter = ('tenant', 'is_active')
|
|
|
|
@admin.register(Donation)
|
|
class DonationAdmin(BaseImportAdminMixin, admin.ModelAdmin):
|
|
list_display = ('voter', 'amount', 'date', 'method', 'tenant_name')
|
|
list_filter = ('voter__tenant', 'method', 'date')
|
|
search_fields = ('voter__first_name', 'voter__last_name', 'voter__voter_id')
|
|
autocomplete_fields = ['voter']
|
|
|
|
def tenant_name(self, obj): return obj.voter.tenant.name
|
|
tenant_name.short_description = "Tenant"
|
|
|
|
@admin.register(InteractionType)
|
|
class InteractionTypeAdmin(admin.ModelAdmin):
|
|
list_display = ('name', 'tenant', 'is_active')
|
|
list_filter = ('tenant', 'is_active')
|
|
|
|
@admin.register(ElectionType)
|
|
class ElectionTypeAdmin(admin.ModelAdmin):
|
|
list_display = ('name', 'tenant', 'is_active')
|
|
list_filter = ('tenant', 'is_active')
|
|
|
|
@admin.register(VoterLikelihood)
|
|
class VoterLikelihoodAdmin(BaseImportAdminMixin, admin.ModelAdmin):
|
|
list_display = ('voter', 'election_type', 'likelihood')
|
|
list_filter = ('voter__tenant', 'election_type', 'likelihood')
|
|
autocomplete_fields = ['voter']
|
|
|
|
@admin.register(VotingRecord)
|
|
class VotingRecordAdmin(BaseImportAdminMixin, admin.ModelAdmin):
|
|
list_display = ('voter', 'election_date', 'election_description', 'primary_party')
|
|
list_filter = ('voter__tenant', 'election_date', 'primary_party')
|
|
autocomplete_fields = ['voter']
|
|
|
|
@admin.register(Event)
|
|
class EventAdmin(BaseImportAdminMixin, admin.ModelAdmin):
|
|
list_display = ('name', 'date', 'event_type', 'tenant')
|
|
list_filter = ('tenant', 'event_type', 'date')
|
|
search_fields = ('name', 'location_name')
|
|
|
|
@admin.register(Volunteer)
|
|
class VolunteerAdmin(BaseImportAdminMixin, admin.ModelAdmin):
|
|
list_display = ('first_name', 'last_name', 'email', 'phone', 'tenant')
|
|
list_filter = ('tenant',)
|
|
search_fields = ('first_name', 'last_name', 'email')
|
|
|
|
@admin.register(CampaignSettings)
|
|
class CampaignSettingsAdmin(admin.ModelAdmin):
|
|
list_display = ('tenant', 'timezone', 'donation_goal')
|
|
list_filter = ('tenant',)
|
|
|
|
@admin.register(ScheduledCall)
|
|
class ScheduledCallAdmin(admin.ModelAdmin):
|
|
list_display = ('voter', 'volunteer', 'status', 'created_at', 'tenant')
|
|
list_filter = ('tenant', 'status', 'volunteer')
|
|
autocomplete_fields = ['voter', 'volunteer']
|