diff --git a/core/__pycache__/admin.cpython-311.pyc b/core/__pycache__/admin.cpython-311.pyc index 257d103..eff3575 100644 Binary files a/core/__pycache__/admin.cpython-311.pyc and b/core/__pycache__/admin.cpython-311.pyc differ diff --git a/core/__pycache__/models.cpython-311.pyc b/core/__pycache__/models.cpython-311.pyc index e220ca6..249a4d1 100644 Binary files a/core/__pycache__/models.cpython-311.pyc and b/core/__pycache__/models.cpython-311.pyc differ diff --git a/core/admin.py b/core/admin.py index 46d66ff..a166bb4 100644 --- a/core/admin.py +++ b/core/admin.py @@ -1,10 +1,12 @@ +from django.http import HttpResponse +from django.utils.safestring import mark_safe import csv import io import logging import tempfile import os from django.contrib import admin, messages -from django.urls import path +from django.urls import path, reverse from django.shortcuts import render, redirect from django.template.response import TemplateResponse from .models import ( @@ -78,6 +80,29 @@ VOTER_LIKELIHOOD_MAPPABLE_FIELDS = [ ('likelihood', 'Likelihood'), ] +class BaseImportAdminMixin: + def download_errors(self, request): + logger.info(f"download_errors called for {self.model._meta.model_name}") + session_key = f"{self.model._meta.model_name}_import_errors" + failed_rows = request.session.get(session_key, []) + if not failed_rows: + self.message_user(request, "No error log found in session.", level=messages.WARNING) + return redirect("..") + + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = f"attachment; filename={self.model._meta.model_name}_import_errors.csv" + + if failed_rows: + all_keys = set() + for r in failed_rows: + all_keys.update(r.keys()) + + writer = csv.DictWriter(response, fieldnames=sorted(list(all_keys))) + writer.writeheader() + writer.writerows(failed_rows) + + return response + class TenantUserRoleInline(admin.TabularInline): model = TenantUserRole extra = 1 @@ -88,7 +113,7 @@ class CampaignSettingsInline(admin.StackedInline): @admin.register(Tenant) class TenantAdmin(admin.ModelAdmin): - list_display = ('name', 'slug', 'created_at') + list_display = ('name', 'created_at') search_fields = ('name',) inlines = [TenantUserRoleInline, CampaignSettingsInline] @@ -139,23 +164,77 @@ class VoterLikelihoodInline(admin.TabularInline): extra = 1 @admin.register(Voter) -class VoterAdmin(admin.ModelAdmin): +class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin): list_display = ('first_name', 'last_name', 'nickname', 'voter_id', 'tenant', 'district', 'candidate_support', 'is_targeted', 'city', 'state', 'prior_state') list_filter = ('tenant', 'candidate_support', 'is_targeted', 'yard_sign', 'district', 'city', 'state', 'prior_state') search_fields = ('first_name', 'last_name', 'nickname', 'voter_id', 'address', 'city', 'state', 'prior_state', 'zip_code', 'county') inlines = [VotingRecordInline, DonationInline, InteractionInline, VoterLikelihoodInline] + readonly_fields = ('address',) change_list_template = "admin/voter_change_list.html" def get_urls(self): urls = super().get_urls() my_urls = [ + path('download-errors/', self.admin_site.admin_view(self.download_errors), name='voter-download-errors'), path('import-voters/', self.admin_site.admin_view(self.import_voters), name='import-voters'), ] return my_urls + urls def import_voters(self, request): if request.method == "POST": - if "_import" in request.POST: + if "_preview" in request.POST: + file_path = request.POST.get('file_path') + tenant_id = request.POST.get('tenant') + tenant = Tenant.objects.get(id=tenant_id) + + mapping = {} + for field_name, _ in VOTER_MAPPABLE_FIELDS: + mapping[field_name] = request.POST.get(f'map_{field_name}') + + try: + with open(file_path, 'r', encoding='UTF-8') as f: + reader = csv.DictReader(f) + total_count = 0 + create_count = 0 + update_count = 0 + preview_data = [] + for row in reader: + total_count += 1 + voter_id = row.get(mapping.get('voter_id')) + exists = Voter.objects.filter(tenant=tenant, voter_id=voter_id).exists() + if exists: + update_count += 1 + action = 'update' + else: + create_count += 1 + action = 'create' + + if len(preview_data) < 10: + preview_data.append({ + 'action': action, + 'identifier': voter_id, + 'details': f"{row.get(mapping.get('first_name', '')) or ''} {row.get(mapping.get('last_name', '')) or ''}".strip() + }) + + context = self.admin_site.each_context(request) + context.update({ + 'title': "Import Preview", + 'total_count': total_count, + 'create_count': create_count, + 'update_count': update_count, + 'preview_data': preview_data, + 'mapping': mapping, + 'file_path': file_path, + 'tenant_id': tenant_id, + 'action_url': request.path, + 'opts': self.model._meta, + }) + return render(request, "admin/import_preview.html", context) + except Exception as e: + self.message_user(request, f"Error processing preview: {e}", level=messages.ERROR) + return redirect("..") + + elif "_import" in request.POST: file_path = request.POST.get('file_path') tenant_id = request.POST.get('tenant') tenant = Tenant.objects.get(id=tenant_id) @@ -169,19 +248,23 @@ class VoterAdmin(admin.ModelAdmin): reader = csv.DictReader(f) count = 0 errors = 0 + failed_rows = [] for row in reader: try: voter_data = {} + voter_id = '' for field_name, csv_col in mapping.items(): if csv_col: val = row.get(csv_col) - if val is not None: + if val is not None and str(val).strip() != '': + if field_name == 'voter_id': + voter_id = val + continue + if field_name == 'is_targeted': val = str(val).lower() in ['true', '1', 'yes'] voter_data[field_name] = val - voter_id = voter_data.pop('voter_id', '') - if 'candidate_support' in voter_data: if voter_data['candidate_support'] not in dict(Voter.SUPPORT_CHOICES): voter_data['candidate_support'] = 'unknown' @@ -191,10 +274,6 @@ class VoterAdmin(admin.ModelAdmin): if 'window_sticker' in voter_data: if voter_data['window_sticker'] not in dict(Voter.WINDOW_STICKER_CHOICES): voter_data['window_sticker'] = 'none' - - for d_field in ['registration_date', 'birthdate', 'latitude', 'longitude']: - if d_field in voter_data and not voter_data[d_field]: - del voter_data[d_field] Voter.objects.update_or_create( tenant=tenant, @@ -203,14 +282,20 @@ class VoterAdmin(admin.ModelAdmin): ) count += 1 except Exception as e: - logger.error(f"Error importing voter row: {e}") + logger.error(f"Error importing: {e}") + row["Import Error"] = str(e) + failed_rows.append(row) errors += 1 if os.path.exists(file_path): os.remove(file_path) self.message_user(request, f"Successfully imported {count} voters.") + request.session[f"{self.model._meta.model_name}_import_errors"] = failed_rows + request.session.modified = True + logger.info(f"Stored {len(failed_rows)} failed rows in session for {self.model._meta.model_name}") if errors > 0: - self.message_user(request, f"Failed to import {errors} rows.", level=messages.WARNING) + error_url = reverse("admin:voter-download-errors") + self.message_user(request, mark_safe(f"Failed to import {errors} rows. Download failed records"), level=messages.WARNING) return redirect("..") except Exception as e: self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) @@ -255,7 +340,7 @@ class VoterAdmin(admin.ModelAdmin): return render(request, "admin/import_csv.html", context) @admin.register(Event) -class EventAdmin(admin.ModelAdmin): +class EventAdmin(BaseImportAdminMixin, admin.ModelAdmin): list_display = ('id', 'event_type', 'date', 'tenant') list_filter = ('tenant', 'date', 'event_type') change_list_template = "admin/event_change_list.html" @@ -263,13 +348,67 @@ class EventAdmin(admin.ModelAdmin): def get_urls(self): urls = super().get_urls() my_urls = [ + path('download-errors/', self.admin_site.admin_view(self.download_errors), name='event-download-errors'), path('import-events/', self.admin_site.admin_view(self.import_events), name='import-events'), ] return my_urls + urls def import_events(self, request): if request.method == "POST": - if "_import" in request.POST: + if "_preview" in request.POST: + file_path = request.POST.get('file_path') + tenant_id = request.POST.get('tenant') + tenant = Tenant.objects.get(id=tenant_id) + mapping = {} + for field_name, _ in EVENT_MAPPABLE_FIELDS: + mapping[field_name] = request.POST.get(f'map_{field_name}') + try: + with open(file_path, 'r', encoding='UTF-8') as f: + reader = csv.DictReader(f) + total_count = 0 + create_count = 0 + update_count = 0 + preview_data = [] + for row in reader: + total_count += 1 + date = row.get(mapping.get('date')) + event_type_name = row.get(mapping.get('event_type')) + exists = False + if date and event_type_name: + exists = Event.objects.filter(tenant=tenant, date=date, event_type__name=event_type_name).exists() + + if exists: + update_count += 1 + action = 'update' + else: + create_count += 1 + action = 'create' + + if len(preview_data) < 10: + preview_data.append({ + 'action': action, + 'identifier': f"{date} - {event_type_name}", + 'details': row.get(mapping.get('description', '')) or '' + }) + context = self.admin_site.each_context(request) + context.update({ + 'title': "Import Preview", + 'total_count': total_count, + 'create_count': create_count, + 'update_count': update_count, + 'preview_data': preview_data, + 'mapping': mapping, + 'file_path': file_path, + 'tenant_id': tenant_id, + 'action_url': request.path, + 'opts': self.model._meta, + }) + return render(request, "admin/import_preview.html", context) + except Exception as e: + self.message_user(request, f"Error processing preview: {e}", level=messages.ERROR) + return redirect("..") + + elif "_import" in request.POST: file_path = request.POST.get('file_path') tenant_id = request.POST.get('tenant') tenant = Tenant.objects.get(id=tenant_id) @@ -283,13 +422,16 @@ class EventAdmin(admin.ModelAdmin): reader = csv.DictReader(f) count = 0 errors = 0 + failed_rows = [] for row in reader: try: date = row.get(mapping.get('date')) if mapping.get('date') else None event_type_name = row.get(mapping.get('event_type')) if mapping.get('event_type') else None - description = row.get(mapping.get('description')) if mapping.get('description') else '' + description = row.get(mapping.get('description')) if mapping.get('description') else None if not date or not event_type_name: + row["Import Error"] = "Missing date or event type" + failed_rows.append(row) errors += 1 continue @@ -298,22 +440,32 @@ class EventAdmin(admin.ModelAdmin): name=event_type_name ) - Event.objects.create( + defaults = {} + if description and description.strip(): + defaults['description'] = description + + Event.objects.update_or_create( tenant=tenant, date=date, event_type=event_type, - description=description + defaults=defaults ) count += 1 except Exception as e: - logger.error(f"Error importing event row: {e}") + logger.error(f"Error importing: {e}") + row["Import Error"] = str(e) + failed_rows.append(row) errors += 1 if os.path.exists(file_path): os.remove(file_path) self.message_user(request, f"Successfully imported {count} events.") + request.session[f"{self.model._meta.model_name}_import_errors"] = failed_rows + request.session.modified = True + logger.info(f"Stored {len(failed_rows)} failed rows in session for {self.model._meta.model_name}") if errors > 0: - self.message_user(request, f"Failed to import {errors} rows.", level=messages.WARNING) + error_url = reverse("admin:event-download-errors") + self.message_user(request, mark_safe(f"Failed to import {errors} rows. Download failed records"), level=messages.WARNING) return redirect("..") except Exception as e: self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) @@ -358,7 +510,7 @@ class EventAdmin(admin.ModelAdmin): return render(request, "admin/import_csv.html", context) @admin.register(EventParticipation) -class EventParticipationAdmin(admin.ModelAdmin): +class EventParticipationAdmin(BaseImportAdminMixin, admin.ModelAdmin): list_display = ('voter', 'event', 'participation_type') list_filter = ('event__tenant', 'event', 'participation_type') change_list_template = "admin/eventparticipation_change_list.html" @@ -366,13 +518,77 @@ class EventParticipationAdmin(admin.ModelAdmin): def get_urls(self): urls = super().get_urls() my_urls = [ + path('download-errors/', self.admin_site.admin_view(self.download_errors), name='eventparticipation-download-errors'), path('import-event-participations/', self.admin_site.admin_view(self.import_event_participations), name='import-event-participations'), ] return my_urls + urls def import_event_participations(self, request): if request.method == "POST": - if "_import" in request.POST: + if "_preview" in request.POST: + file_path = request.POST.get('file_path') + tenant_id = request.POST.get('tenant') + tenant = Tenant.objects.get(id=tenant_id) + mapping = {} + for field_name, _ in EVENT_PARTICIPATION_MAPPABLE_FIELDS: + mapping[field_name] = request.POST.get(f'map_{field_name}') + try: + with open(file_path, 'r', encoding='UTF-8') as f: + reader = csv.DictReader(f) + total_count = 0 + create_count = 0 + update_count = 0 + preview_data = [] + for row in reader: + total_count += 1 + voter_id = row.get(mapping.get('voter_id')) + event_id = row.get(mapping.get('event_id')) + event_date = row.get(mapping.get('event_date')) + event_type_name = row.get(mapping.get('event_type')) + + exists = False + if voter_id: + try: + voter = Voter.objects.get(tenant=tenant, voter_id=voter_id) + if event_id: + exists = EventParticipation.objects.filter(voter=voter, event_id=event_id).exists() + elif event_date and event_type_name: + exists = EventParticipation.objects.filter(voter=voter, event__date=event_date, event__event_type__name=event_type_name).exists() + except Voter.DoesNotExist: + pass + + if exists: + update_count += 1 + action = 'update' + else: + create_count += 1 + action = 'create' + + if len(preview_data) < 10: + preview_data.append({ + 'action': action, + 'identifier': f"Voter: {voter_id}", + 'details': f"Participation: {row.get(mapping.get('participation_type', '')) or ''}" + }) + context = self.admin_site.each_context(request) + context.update({ + 'title': "Import Preview", + 'total_count': total_count, + 'create_count': create_count, + 'update_count': update_count, + 'preview_data': preview_data, + 'mapping': mapping, + 'file_path': file_path, + 'tenant_id': tenant_id, + 'action_url': request.path, + 'opts': self.model._meta, + }) + return render(request, "admin/import_preview.html", context) + except Exception as e: + self.message_user(request, f"Error processing preview: {e}", level=messages.ERROR) + return redirect("..") + + elif "_import" in request.POST: file_path = request.POST.get('file_path') tenant_id = request.POST.get('tenant') tenant = Tenant.objects.get(id=tenant_id) @@ -386,19 +602,25 @@ class EventParticipationAdmin(admin.ModelAdmin): reader = csv.DictReader(f) count = 0 errors = 0 + failed_rows = [] for row in reader: try: voter_id = row.get(mapping.get('voter_id')) if mapping.get('voter_id') else None - participation_type = row.get(mapping.get('participation_type')) if mapping.get('participation_type') else 'invited' + participation_type_val = row.get(mapping.get('participation_type')) if mapping.get('participation_type') else None if not voter_id: + row["Import Error"] = "Missing voter ID" + failed_rows.append(row) errors += 1 continue try: voter = Voter.objects.get(tenant=tenant, voter_id=voter_id) except Voter.DoesNotExist: - logger.error(f"Voter not found: {voter_id} in tenant {tenant.name}") + error_msg = f"Voter with ID {voter_id} not found" + logger.error(error_msg) + row["Import Error"] = error_msg + failed_rows.append(row) errors += 1 continue @@ -421,28 +643,41 @@ class EventParticipationAdmin(admin.ModelAdmin): pass if not event: - logger.error(f"Event not found for row") + error_msg = "Event not found (check ID, date, or type)" + logger.error(error_msg) + row["Import Error"] = error_msg + failed_rows.append(row) errors += 1 continue - if participation_type not in dict(EventParticipation.PARTICIPATION_TYPE_CHOICES): - participation_type = 'invited' + defaults = {} + if participation_type_val and participation_type_val.strip(): + if participation_type_val in dict(EventParticipation.PARTICIPATION_TYPE_CHOICES): + defaults['participation_type'] = participation_type_val + else: + defaults['participation_type'] = 'invited' EventParticipation.objects.update_or_create( event=event, voter=voter, - defaults={'participation_type': participation_type} + defaults=defaults ) count += 1 except Exception as e: - logger.error(f"Error importing participation row: {e}") + logger.error(f"Error importing: {e}") + row["Import Error"] = str(e) + failed_rows.append(row) errors += 1 if os.path.exists(file_path): os.remove(file_path) self.message_user(request, f"Successfully imported {count} participations.") + request.session[f"{self.model._meta.model_name}_import_errors"] = failed_rows + request.session.modified = True + logger.info(f"Stored {len(failed_rows)} failed rows in session for {self.model._meta.model_name}") if errors > 0: - self.message_user(request, f"Failed to import {errors} rows.", level=messages.WARNING) + error_url = reverse("admin:eventparticipation-download-errors") + self.message_user(request, mark_safe(f"Failed to import {errors} rows. Download failed records"), level=messages.WARNING) return redirect("..") except Exception as e: self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) @@ -487,7 +722,7 @@ class EventParticipationAdmin(admin.ModelAdmin): return render(request, "admin/import_csv.html", context) @admin.register(Donation) -class DonationAdmin(admin.ModelAdmin): +class DonationAdmin(BaseImportAdminMixin, admin.ModelAdmin): list_display = ('id', 'voter', 'date', 'amount', 'method') list_filter = ('voter__tenant', 'date', 'method') search_fields = ('voter__first_name', 'voter__last_name', 'voter__voter_id') @@ -496,13 +731,68 @@ class DonationAdmin(admin.ModelAdmin): def get_urls(self): urls = super().get_urls() my_urls = [ + path('download-errors/', self.admin_site.admin_view(self.download_errors), name='donation-download-errors'), path('import-donations/', self.admin_site.admin_view(self.import_donations), name='import-donations'), ] return my_urls + urls def import_donations(self, request): if request.method == "POST": - if "_import" in request.POST: + if "_preview" in request.POST: + file_path = request.POST.get('file_path') + tenant_id = request.POST.get('tenant') + tenant = Tenant.objects.get(id=tenant_id) + mapping = {} + for field_name, _ in DONATION_MAPPABLE_FIELDS: + mapping[field_name] = request.POST.get(f'map_{field_name}') + try: + with open(file_path, 'r', encoding='UTF-8') as f: + reader = csv.DictReader(f) + total_count = 0 + create_count = 0 + update_count = 0 + preview_data = [] + for row in reader: + total_count += 1 + voter_id = row.get(mapping.get('voter_id')) + date = row.get(mapping.get('date')) + amount = row.get(mapping.get('amount')) + exists = False + if voter_id and date and amount: + exists = Donation.objects.filter(voter__tenant=tenant, voter__voter_id=voter_id, date=date, amount=amount).exists() + + if exists: + update_count += 1 + action = 'update' + else: + create_count += 1 + action = 'create' + + if len(preview_data) < 10: + preview_data.append({ + 'action': action, + 'identifier': f"Voter: {voter_id}", + 'details': f"Date: {date}, Amount: {amount}" + }) + context = self.admin_site.each_context(request) + context.update({ + 'title': "Import Preview", + 'total_count': total_count, + 'create_count': create_count, + 'update_count': update_count, + 'preview_data': preview_data, + 'mapping': mapping, + 'file_path': file_path, + 'tenant_id': tenant_id, + 'action_url': request.path, + 'opts': self.model._meta, + }) + return render(request, "admin/import_preview.html", context) + except Exception as e: + self.message_user(request, f"Error processing preview: {e}", level=messages.ERROR) + return redirect("..") + + elif "_import" in request.POST: file_path = request.POST.get('file_path') tenant_id = request.POST.get('tenant') tenant = Tenant.objects.get(id=tenant_id) @@ -516,16 +806,21 @@ class DonationAdmin(admin.ModelAdmin): reader = csv.DictReader(f) count = 0 errors = 0 + failed_rows = [] for row in reader: try: voter_id = row.get(mapping.get('voter_id')) if mapping.get('voter_id') else None if not voter_id: + row["Import Error"] = "Missing voter ID" + failed_rows.append(row) errors += 1 continue try: voter = Voter.objects.get(tenant=tenant, voter_id=voter_id) except Voter.DoesNotExist: + row["Import Error"] = f"Voter {voter_id} not found" + failed_rows.append(row) errors += 1 continue @@ -534,32 +829,44 @@ class DonationAdmin(admin.ModelAdmin): method_name = row.get(mapping.get('method')) if not date or not amount: + row["Import Error"] = "Missing date or amount" + failed_rows.append(row) errors += 1 continue method = None - if method_name: + if method_name and method_name.strip(): method, _ = DonationMethod.objects.get_or_create( tenant=tenant, name=method_name ) - Donation.objects.create( + defaults = {} + if method: + defaults['method'] = method + + Donation.objects.update_or_create( voter=voter, date=date, amount=amount, - method=method + defaults=defaults ) count += 1 except Exception as e: - logger.error(f"Error importing donation row: {e}") + logger.error(f"Error importing: {e}") + row["Import Error"] = str(e) + failed_rows.append(row) errors += 1 if os.path.exists(file_path): os.remove(file_path) self.message_user(request, f"Successfully imported {count} donations.") + request.session[f"{self.model._meta.model_name}_import_errors"] = failed_rows + request.session.modified = True + logger.info(f"Stored {len(failed_rows)} failed rows in session for {self.model._meta.model_name}") if errors > 0: - self.message_user(request, f"Failed to import {errors} rows.", level=messages.WARNING) + error_url = reverse("admin:donation-download-errors") + self.message_user(request, mark_safe(f"Failed to import {errors} rows. Download failed records"), level=messages.WARNING) return redirect("..") except Exception as e: self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) @@ -604,7 +911,7 @@ class DonationAdmin(admin.ModelAdmin): return render(request, "admin/import_csv.html", context) @admin.register(Interaction) -class InteractionAdmin(admin.ModelAdmin): +class InteractionAdmin(BaseImportAdminMixin, admin.ModelAdmin): list_display = ('id', 'voter', 'type', 'date', 'description') list_filter = ('voter__tenant', 'type', 'date') search_fields = ('voter__first_name', 'voter__last_name', 'voter__voter_id', 'description') @@ -613,13 +920,67 @@ class InteractionAdmin(admin.ModelAdmin): def get_urls(self): urls = super().get_urls() my_urls = [ + path('download-errors/', self.admin_site.admin_view(self.download_errors), name='interaction-download-errors'), path('import-interactions/', self.admin_site.admin_view(self.import_interactions), name='import-interactions'), ] return my_urls + urls def import_interactions(self, request): if request.method == "POST": - if "_import" in request.POST: + if "_preview" in request.POST: + file_path = request.POST.get('file_path') + tenant_id = request.POST.get('tenant') + tenant = Tenant.objects.get(id=tenant_id) + mapping = {} + for field_name, _ in INTERACTION_MAPPABLE_FIELDS: + mapping[field_name] = request.POST.get(f'map_{field_name}') + try: + with open(file_path, 'r', encoding='UTF-8') as f: + reader = csv.DictReader(f) + total_count = 0 + create_count = 0 + update_count = 0 + preview_data = [] + for row in reader: + total_count += 1 + voter_id = row.get(mapping.get('voter_id')) + date = row.get(mapping.get('date')) + exists = False + if voter_id and date: + exists = Interaction.objects.filter(voter__tenant=tenant, voter__voter_id=voter_id, date=date).exists() + + if exists: + update_count += 1 + action = 'update' + else: + create_count += 1 + action = 'create' + + if len(preview_data) < 10: + preview_data.append({ + 'action': action, + 'identifier': f"Voter: {voter_id}", + 'details': f"Date: {date}, Desc: {row.get(mapping.get('description', '')) or ''}" + }) + context = self.admin_site.each_context(request) + context.update({ + 'title': "Import Preview", + 'total_count': total_count, + 'create_count': create_count, + 'update_count': update_count, + 'preview_data': preview_data, + 'mapping': mapping, + 'file_path': file_path, + 'tenant_id': tenant_id, + 'action_url': request.path, + 'opts': self.model._meta, + }) + return render(request, "admin/import_preview.html", context) + except Exception as e: + self.message_user(request, f"Error processing preview: {e}", level=messages.ERROR) + return redirect("..") + + elif "_import" in request.POST: file_path = request.POST.get('file_path') tenant_id = request.POST.get('tenant') tenant = Tenant.objects.get(id=tenant_id) @@ -633,52 +994,71 @@ class InteractionAdmin(admin.ModelAdmin): reader = csv.DictReader(f) count = 0 errors = 0 + failed_rows = [] for row in reader: try: voter_id = row.get(mapping.get('voter_id')) if mapping.get('voter_id') else None if not voter_id: + row["Import Error"] = "Missing voter ID" + failed_rows.append(row) errors += 1 continue try: voter = Voter.objects.get(tenant=tenant, voter_id=voter_id) except Voter.DoesNotExist: + row["Import Error"] = f"Voter {voter_id} not found" + failed_rows.append(row) errors += 1 continue date = row.get(mapping.get('date')) type_name = row.get(mapping.get('type')) description = row.get(mapping.get('description')) - notes = row.get(mapping.get('notes')) if mapping.get('notes') else '' + notes = row.get(mapping.get('notes')) if not date or not description: + row["Import Error"] = "Missing date or description" + failed_rows.append(row) errors += 1 continue interaction_type = None - if type_name: + if type_name and type_name.strip(): interaction_type, _ = InteractionType.objects.get_or_create( tenant=tenant, name=type_name ) - Interaction.objects.create( + defaults = {} + if interaction_type: + defaults['type'] = interaction_type + if description and description.strip(): + defaults['description'] = description + if notes and notes.strip(): + defaults['notes'] = notes + + Interaction.objects.update_or_create( voter=voter, date=date, - type=interaction_type, - description=description, - notes=notes + defaults=defaults ) count += 1 except Exception as e: - logger.error(f"Error importing interaction row: {e}") + logger.error(f"Error importing: {e}") + row["Import Error"] = str(e) + failed_rows.append(row) errors += 1 if os.path.exists(file_path): os.remove(file_path) self.message_user(request, f"Successfully imported {count} interactions.") + request.session[f"{self.model._meta.model_name}_import_errors"] = failed_rows + request.session.modified = True + logger.info(f"Stored {len(failed_rows)} failed rows in session for {self.model._meta.model_name}") if errors > 0: - self.message_user(request, f"Failed to import {errors} rows.", level=messages.WARNING) + error_url = reverse("admin:interaction-download-errors") + self.message_user(request, mark_safe(f"Failed to import {errors} rows. Download failed records"), level=messages.WARNING) return redirect("..") except Exception as e: self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) @@ -723,7 +1103,7 @@ class InteractionAdmin(admin.ModelAdmin): return render(request, "admin/import_csv.html", context) @admin.register(VoterLikelihood) -class VoterLikelihoodAdmin(admin.ModelAdmin): +class VoterLikelihoodAdmin(BaseImportAdminMixin, admin.ModelAdmin): list_display = ('id', 'voter', 'election_type', 'likelihood') list_filter = ('voter__tenant', 'election_type', 'likelihood') search_fields = ('voter__first_name', 'voter__last_name', 'voter__voter_id') @@ -732,13 +1112,67 @@ class VoterLikelihoodAdmin(admin.ModelAdmin): def get_urls(self): urls = super().get_urls() my_urls = [ + path('download-errors/', self.admin_site.admin_view(self.download_errors), name='voterlikelihood-download-errors'), path('import-likelihoods/', self.admin_site.admin_view(self.import_likelihoods), name='import-likelihoods'), ] return my_urls + urls def import_likelihoods(self, request): if request.method == "POST": - if "_import" in request.POST: + if "_preview" in request.POST: + file_path = request.POST.get('file_path') + tenant_id = request.POST.get('tenant') + tenant = Tenant.objects.get(id=tenant_id) + mapping = {} + for field_name, _ in VOTER_LIKELIHOOD_MAPPABLE_FIELDS: + mapping[field_name] = request.POST.get(f'map_{field_name}') + try: + with open(file_path, 'r', encoding='UTF-8') as f: + reader = csv.DictReader(f) + total_count = 0 + create_count = 0 + update_count = 0 + preview_data = [] + for row in reader: + total_count += 1 + voter_id = row.get(mapping.get('voter_id')) + election_type_name = row.get(mapping.get('election_type')) + exists = False + if voter_id and election_type_name: + exists = VoterLikelihood.objects.filter(voter__tenant=tenant, voter__voter_id=voter_id, election_type__name=election_type_name).exists() + + if exists: + update_count += 1 + action = 'update' + else: + create_count += 1 + action = 'create' + + if len(preview_data) < 10: + preview_data.append({ + 'action': action, + 'identifier': f"Voter: {voter_id}", + 'details': f"Election: {election_type_name}, Likelihood: {row.get(mapping.get('likelihood', '')) or ''}" + }) + context = self.admin_site.each_context(request) + context.update({ + 'title': "Import Preview", + 'total_count': total_count, + 'create_count': create_count, + 'update_count': update_count, + 'preview_data': preview_data, + 'mapping': mapping, + 'file_path': file_path, + 'tenant_id': tenant_id, + 'action_url': request.path, + 'opts': self.model._meta, + }) + return render(request, "admin/import_preview.html", context) + except Exception as e: + self.message_user(request, f"Error processing preview: {e}", level=messages.ERROR) + return redirect("..") + + elif "_import" in request.POST: file_path = request.POST.get('file_path') tenant_id = request.POST.get('tenant') tenant = Tenant.objects.get(id=tenant_id) @@ -752,16 +1186,21 @@ class VoterLikelihoodAdmin(admin.ModelAdmin): reader = csv.DictReader(f) count = 0 errors = 0 + failed_rows = [] for row in reader: try: voter_id = row.get(mapping.get('voter_id')) if mapping.get('voter_id') else None if not voter_id: + row["Import Error"] = "Missing voter ID" + failed_rows.append(row) errors += 1 continue try: voter = Voter.objects.get(tenant=tenant, voter_id=voter_id) except Voter.DoesNotExist: + row["Import Error"] = f"Voter {voter_id} not found" + failed_rows.append(row) errors += 1 continue @@ -769,6 +1208,8 @@ class VoterLikelihoodAdmin(admin.ModelAdmin): likelihood_val = row.get(mapping.get('likelihood')) if not election_type_name or not likelihood_val: + row["Import Error"] = "Missing election type or likelihood value" + failed_rows.append(row) errors += 1 continue @@ -791,24 +1232,36 @@ class VoterLikelihoodAdmin(admin.ModelAdmin): break if not normalized_likelihood: + row["Import Error"] = f"Invalid likelihood value: {likelihood_val}" + failed_rows.append(row) errors += 1 continue + defaults = {} + if normalized_likelihood and normalized_likelihood.strip(): + defaults['likelihood'] = normalized_likelihood + VoterLikelihood.objects.update_or_create( voter=voter, election_type=election_type, - defaults={'likelihood': normalized_likelihood} + defaults=defaults ) count += 1 except Exception as e: - logger.error(f"Error importing likelihood row: {e}") + logger.error(f"Error importing: {e}") + row["Import Error"] = str(e) + failed_rows.append(row) errors += 1 if os.path.exists(file_path): os.remove(file_path) self.message_user(request, f"Successfully imported {count} likelihoods.") + request.session[f"{self.model._meta.model_name}_import_errors"] = failed_rows + request.session.modified = True + logger.info(f"Stored {len(failed_rows)} failed rows in session for {self.model._meta.model_name}") if errors > 0: - self.message_user(request, f"Failed to import {errors} rows.", level=messages.WARNING) + error_url = reverse("admin:voterlikelihood-download-errors") + self.message_user(request, mark_safe(f"Failed to import {errors} rows. Download failed records"), level=messages.WARNING) return redirect("..") except Exception as e: self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) diff --git a/core/models.py b/core/models.py index d77bba5..97216d5 100644 --- a/core/models.py +++ b/core/models.py @@ -1,33 +1,25 @@ -from decimal import Decimal from django.db import models -from django.utils.text import slugify from django.contrib.auth.models import User -from django.conf import settings -import urllib.request -import urllib.parse import json +import urllib.parse +import urllib.request import logging +from decimal import Decimal +from django.conf import settings logger = logging.getLogger(__name__) class Tenant(models.Model): - name = models.CharField(max_length=255) - slug = models.SlugField(unique=True, blank=True) - description = models.TextField(blank=True) + name = models.CharField(max_length=100) created_at = models.DateTimeField(auto_now_add=True) - def save(self, *args, **kwargs): - if not self.slug: - self.slug = slugify(self.name) - super().save(*args, **kwargs) - def __str__(self): return self.name class TenantUserRole(models.Model): ROLE_CHOICES = [ - ('system_admin', 'System Administrator'), - ('campaign_admin', 'Campaign Administrator'), + ('admin', 'Admin'), + ('campaign_manager', 'Campaign Manager'), ('campaign_staff', 'Campaign Staff'), ] user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='tenant_roles') @@ -49,7 +41,7 @@ class InteractionType(models.Model): unique_together = ('tenant', 'name') def __str__(self): - return f"{self.name} ({self.tenant.name})" + return self.name class DonationMethod(models.Model): tenant = models.ForeignKey(Tenant, on_delete=models.CASCADE, related_name='donation_methods') @@ -60,7 +52,7 @@ class DonationMethod(models.Model): unique_together = ('tenant', 'name') def __str__(self): - return f"{self.name} ({self.tenant.name})" + return self.name class ElectionType(models.Model): tenant = models.ForeignKey(Tenant, on_delete=models.CASCADE, related_name='election_types') @@ -71,7 +63,7 @@ class ElectionType(models.Model): unique_together = ('tenant', 'name') def __str__(self): - return f"{self.name} ({self.tenant.name})" + return self.name class EventType(models.Model): tenant = models.ForeignKey(Tenant, on_delete=models.CASCADE, related_name='event_types') @@ -82,7 +74,7 @@ class EventType(models.Model): unique_together = ('tenant', 'name') def __str__(self): - return f"{self.name} ({self.tenant.name})" + return self.name class Voter(models.Model): SUPPORT_CHOICES = [ @@ -305,4 +297,4 @@ class CampaignSettings(models.Model): verbose_name_plural = 'Campaign Settings' def __str__(self): - return f'Settings for {self.tenant.name}' \ No newline at end of file + return f'Settings for {self.tenant.name}' diff --git a/core/templates/admin/import_mapping.html b/core/templates/admin/import_mapping.html index c07c7a1..b9ed1d7 100644 --- a/core/templates/admin/import_mapping.html +++ b/core/templates/admin/import_mapping.html @@ -41,8 +41,8 @@
- +
-{% endblock %} +{% endblock %} \ No newline at end of file diff --git a/core/templates/admin/import_preview.html b/core/templates/admin/import_preview.html new file mode 100644 index 0000000..1bf9ca8 --- /dev/null +++ b/core/templates/admin/import_preview.html @@ -0,0 +1,77 @@ +{% extends "admin/base_site.html" %} +{% load i18n admin_urls static %} + +{% block breadcrumbs %} +
+{% translate 'Home' %} +› {{ opts.app_config.verbose_name }} +› {{ opts.verbose_name_plural|capfirst }} +› {% translate 'Import Preview' %} +
+{% endblock %} + +{% block content %} +
+
+

{% translate "Import Preview" %}

+

+ {% blocktranslate with total=total_count created=create_count updated=update_count %} + Found {{ total }} records in the CSV file. +
+ - {{ created }} will be created. +
+ - {{ updated }} will be updated. + {% endblocktranslate %} +

+ + {% if preview_data %} +
+

{% translate "Sample Records" %}

+ + + + + + + + + + {% for row in preview_data %} + + + + + + {% endfor %} + +
{% translate "Action" %}{% translate "Identifyer" %}{% translate "Details" %}
+ {% if row.action == 'create' %} + {% translate "CREATE" %} + {% else %} + {% translate "UPDATE" %} + {% endif %} + {{ row.identifier }}{{ row.details }}
+ {% if total_count > preview_data|length %} +

... and {{ total_count|add:"-10" }} more records.

+ {% endif %} +
+ {% endif %} +
+ +
+ {% csrf_token %} + + + + {# Pass mapping as hidden fields #} + {% for field_name, csv_col in mapping.items %} + + {% endfor %} + +
+ + {% translate "Cancel and go back" %} +
+
+
+{% endblock %} diff --git a/core/templates/core/voter_detail.html b/core/templates/core/voter_detail.html index 5955dd9..2dab959 100644 --- a/core/templates/core/voter_detail.html +++ b/core/templates/core/voter_detail.html @@ -445,7 +445,7 @@
- {{ voter_form.voter_id }} + {{ voter_form.district }}
@@ -998,4 +998,4 @@ } }); -{% endblock %} \ No newline at end of file +{% endblock %}