diff --git a/core/__pycache__/admin.cpython-311.pyc b/core/__pycache__/admin.cpython-311.pyc index b188522..7a8a9d7 100644 Binary files a/core/__pycache__/admin.cpython-311.pyc and b/core/__pycache__/admin.cpython-311.pyc differ diff --git a/core/admin.py b/core/admin.py index 4cf45bb..19ac800 100644 --- a/core/admin.py +++ b/core/admin.py @@ -240,7 +240,7 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin): mapping[field_name] = request.POST.get(f"map_{field_name}") try: - with open(file_path, "r", encoding="UTF-8") as f: + with open(file_path, "r", encoding="utf-8-sig") as f: # Optimization: Fast count and partial preview total_count = sum(1 for line in f) - 1 f.seek(0) @@ -289,6 +289,7 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin): self.message_user(request, f"Error processing preview: {e}", level=messages.ERROR) return redirect("..") + elif "_import" in request.POST: file_path = request.POST.get("file_path") tenant_id = request.POST.get("tenant") @@ -298,22 +299,25 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin): try: count = 0 + created_count = 0 + updated_count = 0 + skipped_no_change = 0 + skipped_no_id = 0 errors = 0 failed_rows = [] - batch_size = 500 # Optimized batch size + batch_size = 500 - # Pre-calculate choice dicts and sets support_choices = dict(Voter.SUPPORT_CHOICES) yard_sign_choices = dict(Voter.YARD_SIGN_CHOICES) window_sticker_choices = dict(Voter.WINDOW_STICKER_CHOICES) phone_type_choices = dict(Voter.PHONE_TYPE_CHOICES) phone_type_reverse = {v.lower(): k for k, v in phone_type_choices.items()} - # Fields to fetch for change detection valid_fields = {f.name for f in Voter._meta.get_fields()} mapped_fields = {f for f in mapping.keys() if f in valid_fields} - fetch_fields = list(mapped_fields | {"voter_id", "address", "phone", "longitude"}) - update_fields = list(mapped_fields | {"address", "phone"}) + fetch_fields = list(mapped_fields | {"voter_id", "address", "phone", "longitude", "latitude"}) + # Ensure derived/special fields are in update_fields + update_fields = list(mapped_fields | {"address", "phone", "longitude", "latitude"}) if "voter_id" in update_fields: update_fields.remove("voter_id") def chunk_reader(reader, size): @@ -326,15 +330,18 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin): if chunk: yield chunk - with open(file_path, "r", encoding="UTF-8") as f: + with open(file_path, "r", encoding="utf-8-sig") as f: reader = csv.DictReader(f) v_id_col = mapping.get("voter_id") if not v_id_col: raise ValueError("Voter ID mapping is missing") + + print(f"DEBUG: Starting voter import. Tenant: {tenant.name}. Voter ID column: {v_id_col}") + total_processed = 0 for chunk_index, chunk in enumerate(chunk_reader(reader, batch_size)): with transaction.atomic(): - voter_ids = [row.get(v_id_col) for row in chunk if row.get(v_id_col)] + voter_ids = [str(row.get(v_id_col)).strip() for row in chunk if row.get(v_id_col)] existing_voters = {v.voter_id: v for v in Voter.objects.filter(tenant=tenant, voter_id__in=voter_ids).only(*fetch_fields)} to_create = [] @@ -342,9 +349,19 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin): processed_in_batch = set() for row in chunk: + total_processed += 1 try: - voter_id = row.get(v_id_col) - if not voter_id or voter_id in processed_in_batch: + raw_voter_id = row.get(v_id_col) + if raw_voter_id is None: + skipped_no_id += 1 + continue + + voter_id = str(raw_voter_id).strip() + if not voter_id: + skipped_no_id += 1 + continue + + if voter_id in processed_in_batch: continue processed_in_batch.add(voter_id) @@ -359,25 +376,37 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin): for field_name, csv_col in mapping.items(): if field_name == "voter_id": continue val = row.get(csv_col) - if val is None or str(val).strip() == "": continue + if val is None: continue + val = str(val).strip() + if val == "": continue - # Type-specific conversions if field_name == "is_targeted": val = str(val).lower() in ["true", "1", "yes"] elif field_name in ["birthdate", "registration_date"]: - try: - if isinstance(val, str): - val = datetime.strptime(val.strip(), "%Y-%m-%d").date() - except: - pass + orig_val = val + parsed_date = None + for fmt in ["%Y-%m-%d", "%m/%d/%Y", "%d/%m/%Y", "%Y/%m/%d"]: + try: + parsed_date = datetime.strptime(val, fmt).date() + break + except: + continue + if parsed_date: + val = parsed_date + else: + # If parsing fails, keep original or skip? Let's skip updating this field. + continue elif field_name == "candidate_support": + val = val.lower().replace(" ", "_") if val not in support_choices: val = "unknown" elif field_name == "yard_sign": + val = val.lower().replace(" ", "_") if val not in yard_sign_choices: val = "none" elif field_name == "window_sticker": + val = val.lower().replace(" ", "_") if val not in window_sticker_choices: val = "none" elif field_name == "phone_type": - val_lower = str(val).lower() + val_lower = val.lower() if val_lower in phone_type_choices: val = val_lower elif val_lower in phone_type_reverse: @@ -385,11 +414,11 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin): else: val = "cell" - if getattr(voter, field_name) != val: + current_val = getattr(voter, field_name) + if current_val != val: setattr(voter, field_name, val) changed = True - # Special fields old_phone = voter.phone voter.phone = format_phone_number(voter.phone) if voter.phone != old_phone: @@ -411,16 +440,19 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin): changed = True if not changed: + skipped_no_change += 1 continue if created: to_create.append(voter) + created_count += 1 else: to_update.append(voter) + updated_count += 1 count += 1 except Exception as e: - logger.error(f"Error importing row: {e}") + print(f"DEBUG: Error importing row {total_processed}: {e}") row["Import Error"] = str(e) failed_rows.append(row) errors += 1 @@ -430,11 +462,14 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin): if to_update: Voter.objects.bulk_update(to_update, update_fields, batch_size=250) - logger.info(f"Voter import progress: Processed batch {chunk_index + 1}. Total successes: {count}") + print(f"DEBUG: Voter import progress: {total_processed} processed. {count} created/updated. {skipped_no_change} skipped (no change). {skipped_no_id} skipped (no ID). {errors} errors.") if os.path.exists(file_path): os.remove(file_path) - self.message_user(request, f"Successfully imported {count} voters.") + + success_msg = f"Import complete: {count} voters created/updated. ({created_count} new, {updated_count} updated, {skipped_no_change} skipped with no changes, {skipped_no_id} skipped missing ID, {errors} errors)" + self.message_user(request, success_msg) + request.session[f"{self.model._meta.model_name}_import_errors"] = failed_rows request.session.modified = True if errors > 0: @@ -442,7 +477,7 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin): self.message_user(request, mark_safe(f"Failed to import {errors} rows. Download failed records"), level=messages.WARNING) return redirect("..") except Exception as e: - logger.exception("Voter import failed") + print(f"DEBUG: Voter import failed: {e}") self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) return redirect("..") else: @@ -460,7 +495,7 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin): tmp.write(chunk) file_path = tmp.name - with open(file_path, "r", encoding="UTF-8") as f: + with open(file_path, "r", encoding="utf-8-sig") as f: reader = csv.reader(f) headers = next(reader) @@ -475,6 +510,14 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin): "opts": self.model._meta, }) return render(request, "admin/import_mapping.html", context) + else: + form = VoterImportForm() + + context = self.admin_site.each_context(request) + context["form"] = form + context["title"] = "Import Voters" + context["opts"] = self.model._meta + return render(request, "admin/import_csv.html", context) class EventAdmin(BaseImportAdminMixin, admin.ModelAdmin): list_display = ('id', 'name', 'event_type', 'date', 'start_time', 'end_time', 'tenant') list_filter = ('tenant', 'date', 'event_type') @@ -626,6 +669,10 @@ class EventAdmin(BaseImportAdminMixin, admin.ModelAdmin): except Exception as e: self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) return redirect("..") + except Exception as e: + print(f"DEBUG: Voter import failed: {e}") + self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) + return redirect("..") else: form = EventImportForm(request.POST, request.FILES) if form.is_valid(): @@ -789,6 +836,10 @@ class VolunteerAdmin(BaseImportAdminMixin, admin.ModelAdmin): except Exception as e: self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) return redirect("..") + except Exception as e: + print(f"DEBUG: Voter import failed: {e}") + self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) + return redirect("..") else: form = VolunteerImportForm(request.POST, request.FILES) if form.is_valid(): @@ -1001,6 +1052,10 @@ class EventParticipationAdmin(BaseImportAdminMixin, admin.ModelAdmin): except Exception as e: self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) return redirect("..") + except Exception as e: + print(f"DEBUG: Voter import failed: {e}") + self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) + return redirect("..") else: form = EventParticipationImportForm(request.POST, request.FILES) if form.is_valid(): @@ -1190,6 +1245,10 @@ class DonationAdmin(BaseImportAdminMixin, admin.ModelAdmin): except Exception as e: self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) return redirect("..") + except Exception as e: + print(f"DEBUG: Voter import failed: {e}") + self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) + return redirect("..") else: form = DonationImportForm(request.POST, request.FILES) if form.is_valid(): @@ -1382,6 +1441,10 @@ class InteractionAdmin(BaseImportAdminMixin, admin.ModelAdmin): except Exception as e: self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) return redirect("..") + except Exception as e: + print(f"DEBUG: Voter import failed: {e}") + self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) + return redirect("..") else: form = InteractionImportForm(request.POST, request.FILES) if form.is_valid(): @@ -1585,6 +1648,10 @@ class VoterLikelihoodAdmin(BaseImportAdminMixin, admin.ModelAdmin): except Exception as e: self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) return redirect("..") + except Exception as e: + print(f"DEBUG: Voter import failed: {e}") + self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) + return redirect("..") else: form = VoterLikelihoodImportForm(request.POST, request.FILES) if form.is_valid():