admin/37769-vm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

.2

Browse Source
This commit is contained in:
Flatlogic Bot 2026-01-28 13:41:32 +00:00
parent 4056b17780
commit 9c3c5219c6
2 changed files with 92 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
core/__pycache__/admin.cpython-311.pyc
View File

Binary file not shown.

117
core/admin.py
View File

@ -240,7 +240,7 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin):
mapping[field_name] = request.POST.get(f"map_{field_name}") mapping[field_name] = request.POST.get(f"map_{field_name}")
try: try:
with open(file_path, "r", encoding="UTF-8") as f: with open(file_path, "r", encoding="utf-8-sig") as f:
# Optimization: Fast count and partial preview # Optimization: Fast count and partial preview
total_count = sum(1 for line in f) - 1 total_count = sum(1 for line in f) - 1
f.seek(0) f.seek(0)
@ -289,6 +289,7 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin):
self.message_user(request, f"Error processing preview: {e}", level=messages.ERROR) self.message_user(request, f"Error processing preview: {e}", level=messages.ERROR)
return redirect("..") return redirect("..")
elif "_import" in request.POST: elif "_import" in request.POST:
file_path = request.POST.get("file_path") file_path = request.POST.get("file_path")
tenant_id = request.POST.get("tenant") tenant_id = request.POST.get("tenant")
@ -298,22 +299,25 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin):
try: try:
count = 0 count = 0
created_count = 0
updated_count = 0
skipped_no_change = 0
skipped_no_id = 0
errors = 0 errors = 0
failed_rows = [] failed_rows = []
batch_size = 500 # Optimized batch size batch_size = 500
# Pre-calculate choice dicts and sets
support_choices = dict(Voter.SUPPORT_CHOICES) support_choices = dict(Voter.SUPPORT_CHOICES)
yard_sign_choices = dict(Voter.YARD_SIGN_CHOICES) yard_sign_choices = dict(Voter.YARD_SIGN_CHOICES)
window_sticker_choices = dict(Voter.WINDOW_STICKER_CHOICES) window_sticker_choices = dict(Voter.WINDOW_STICKER_CHOICES)
phone_type_choices = dict(Voter.PHONE_TYPE_CHOICES) phone_type_choices = dict(Voter.PHONE_TYPE_CHOICES)
phone_type_reverse = {v.lower(): k for k, v in phone_type_choices.items()} phone_type_reverse = {v.lower(): k for k, v in phone_type_choices.items()}
# Fields to fetch for change detection
valid_fields = {f.name for f in Voter._meta.get_fields()} valid_fields = {f.name for f in Voter._meta.get_fields()}
mapped_fields = {f for f in mapping.keys() if f in valid_fields} mapped_fields = {f for f in mapping.keys() if f in valid_fields}
fetch_fields = list(mapped_fields | {"voter_id", "address", "phone", "longitude"}) fetch_fields = list(mapped_fields | {"voter_id", "address", "phone", "longitude", "latitude"})
update_fields = list(mapped_fields | {"address", "phone"}) # Ensure derived/special fields are in update_fields
update_fields = list(mapped_fields | {"address", "phone", "longitude", "latitude"})
if "voter_id" in update_fields: update_fields.remove("voter_id") if "voter_id" in update_fields: update_fields.remove("voter_id")
def chunk_reader(reader, size): def chunk_reader(reader, size):
@ -326,15 +330,18 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin):
if chunk: if chunk:
yield chunk yield chunk
with open(file_path, "r", encoding="UTF-8") as f: with open(file_path, "r", encoding="utf-8-sig") as f:
reader = csv.DictReader(f) reader = csv.DictReader(f)
v_id_col = mapping.get("voter_id") v_id_col = mapping.get("voter_id")
if not v_id_col: if not v_id_col:
raise ValueError("Voter ID mapping is missing") raise ValueError("Voter ID mapping is missing")
print(f"DEBUG: Starting voter import. Tenant: {tenant.name}. Voter ID column: {v_id_col}")
total_processed = 0
for chunk_index, chunk in enumerate(chunk_reader(reader, batch_size)): for chunk_index, chunk in enumerate(chunk_reader(reader, batch_size)):
with transaction.atomic(): with transaction.atomic():
voter_ids = [row.get(v_id_col) for row in chunk if row.get(v_id_col)] voter_ids = [str(row.get(v_id_col)).strip() for row in chunk if row.get(v_id_col)]
existing_voters = {v.voter_id: v for v in Voter.objects.filter(tenant=tenant, voter_id__in=voter_ids).only(*fetch_fields)} existing_voters = {v.voter_id: v for v in Voter.objects.filter(tenant=tenant, voter_id__in=voter_ids).only(*fetch_fields)}
to_create = [] to_create = []
@ -342,9 +349,19 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin):
processed_in_batch = set() processed_in_batch = set()
for row in chunk: for row in chunk:
total_processed += 1
try: try:
voter_id = row.get(v_id_col) raw_voter_id = row.get(v_id_col)
if not voter_id or voter_id in processed_in_batch: if raw_voter_id is None:
skipped_no_id += 1
continue
voter_id = str(raw_voter_id).strip()
if not voter_id:
skipped_no_id += 1
continue
if voter_id in processed_in_batch:
continue continue
processed_in_batch.add(voter_id) processed_in_batch.add(voter_id)
@ -359,25 +376,37 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin):
for field_name, csv_col in mapping.items(): for field_name, csv_col in mapping.items():
if field_name == "voter_id": continue if field_name == "voter_id": continue
val = row.get(csv_col) val = row.get(csv_col)
if val is None or str(val).strip() == "": continue if val is None: continue
val = str(val).strip()
if val == "": continue
# Type-specific conversions
if field_name == "is_targeted": if field_name == "is_targeted":
val = str(val).lower() in ["true", "1", "yes"] val = str(val).lower() in ["true", "1", "yes"]
elif field_name in ["birthdate", "registration_date"]: elif field_name in ["birthdate", "registration_date"]:
try: orig_val = val
if isinstance(val, str): parsed_date = None
val = datetime.strptime(val.strip(), "%Y-%m-%d").date() for fmt in ["%Y-%m-%d", "%m/%d/%Y", "%d/%m/%Y", "%Y/%m/%d"]:
except: try:
pass parsed_date = datetime.strptime(val, fmt).date()
break
except:
continue
if parsed_date:
val = parsed_date
else:
# If parsing fails, keep original or skip? Let's skip updating this field.
continue
elif field_name == "candidate_support": elif field_name == "candidate_support":
val = val.lower().replace(" ", "_")
if val not in support_choices: val = "unknown" if val not in support_choices: val = "unknown"
elif field_name == "yard_sign": elif field_name == "yard_sign":
val = val.lower().replace(" ", "_")
if val not in yard_sign_choices: val = "none" if val not in yard_sign_choices: val = "none"
elif field_name == "window_sticker": elif field_name == "window_sticker":
val = val.lower().replace(" ", "_")
if val not in window_sticker_choices: val = "none" if val not in window_sticker_choices: val = "none"
elif field_name == "phone_type": elif field_name == "phone_type":
val_lower = str(val).lower() val_lower = val.lower()
if val_lower in phone_type_choices: if val_lower in phone_type_choices:
val = val_lower val = val_lower
elif val_lower in phone_type_reverse: elif val_lower in phone_type_reverse:
@ -385,11 +414,11 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin):
else: else:
val = "cell" val = "cell"
if getattr(voter, field_name) != val: current_val = getattr(voter, field_name)
if current_val != val:
setattr(voter, field_name, val) setattr(voter, field_name, val)
changed = True changed = True
# Special fields
old_phone = voter.phone old_phone = voter.phone
voter.phone = format_phone_number(voter.phone) voter.phone = format_phone_number(voter.phone)
if voter.phone != old_phone: if voter.phone != old_phone:
@ -411,16 +440,19 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin):
changed = True changed = True
if not changed: if not changed:
skipped_no_change += 1
continue continue
if created: if created:
to_create.append(voter) to_create.append(voter)
created_count += 1
else: else:
to_update.append(voter) to_update.append(voter)
updated_count += 1
count += 1 count += 1
except Exception as e: except Exception as e:
logger.error(f"Error importing row: {e}") print(f"DEBUG: Error importing row {total_processed}: {e}")
row["Import Error"] = str(e) row["Import Error"] = str(e)
failed_rows.append(row) failed_rows.append(row)
errors += 1 errors += 1
@ -430,11 +462,14 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin):
if to_update: if to_update:
Voter.objects.bulk_update(to_update, update_fields, batch_size=250) Voter.objects.bulk_update(to_update, update_fields, batch_size=250)
logger.info(f"Voter import progress: Processed batch {chunk_index + 1}. Total successes: {count}") print(f"DEBUG: Voter import progress: {total_processed} processed. {count} created/updated. {skipped_no_change} skipped (no change). {skipped_no_id} skipped (no ID). {errors} errors.")
if os.path.exists(file_path): if os.path.exists(file_path):
os.remove(file_path) os.remove(file_path)
self.message_user(request, f"Successfully imported {count} voters.")
success_msg = f"Import complete: {count} voters created/updated. ({created_count} new, {updated_count} updated, {skipped_no_change} skipped with no changes, {skipped_no_id} skipped missing ID, {errors} errors)"
self.message_user(request, success_msg)
request.session[f"{self.model._meta.model_name}_import_errors"] = failed_rows request.session[f"{self.model._meta.model_name}_import_errors"] = failed_rows
request.session.modified = True request.session.modified = True
if errors > 0: if errors > 0:
@ -442,7 +477,7 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin):
self.message_user(request, mark_safe(f"Failed to import {errors} rows. <a href='{error_url}' download>Download failed records</a>"), level=messages.WARNING) self.message_user(request, mark_safe(f"Failed to import {errors} rows. <a href='{error_url}' download>Download failed records</a>"), level=messages.WARNING)
return redirect("..") return redirect("..")
except Exception as e: except Exception as e:
logger.exception("Voter import failed") print(f"DEBUG: Voter import failed: {e}")
self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) self.message_user(request, f"Error processing file: {e}", level=messages.ERROR)
return redirect("..") return redirect("..")
else: else:
@ -460,7 +495,7 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin):
tmp.write(chunk) tmp.write(chunk)
file_path = tmp.name file_path = tmp.name
with open(file_path, "r", encoding="UTF-8") as f: with open(file_path, "r", encoding="utf-8-sig") as f:
reader = csv.reader(f) reader = csv.reader(f)
headers = next(reader) headers = next(reader)
@ -475,6 +510,14 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin):
"opts": self.model._meta, "opts": self.model._meta,
}) })
return render(request, "admin/import_mapping.html", context) return render(request, "admin/import_mapping.html", context)
else:
form = VoterImportForm()
context = self.admin_site.each_context(request)
context["form"] = form
context["title"] = "Import Voters"
context["opts"] = self.model._meta
return render(request, "admin/import_csv.html", context)
class EventAdmin(BaseImportAdminMixin, admin.ModelAdmin): class EventAdmin(BaseImportAdminMixin, admin.ModelAdmin):
list_display = ('id', 'name', 'event_type', 'date', 'start_time', 'end_time', 'tenant') list_display = ('id', 'name', 'event_type', 'date', 'start_time', 'end_time', 'tenant')
list_filter = ('tenant', 'date', 'event_type') list_filter = ('tenant', 'date', 'event_type')
@ -626,6 +669,10 @@ class EventAdmin(BaseImportAdminMixin, admin.ModelAdmin):
except Exception as e: except Exception as e:
self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) self.message_user(request, f"Error processing file: {e}", level=messages.ERROR)
return redirect("..") return redirect("..")
except Exception as e:
print(f"DEBUG: Voter import failed: {e}")
self.message_user(request, f"Error processing file: {e}", level=messages.ERROR)
return redirect("..")
else: else:
form = EventImportForm(request.POST, request.FILES) form = EventImportForm(request.POST, request.FILES)
if form.is_valid(): if form.is_valid():
@ -789,6 +836,10 @@ class VolunteerAdmin(BaseImportAdminMixin, admin.ModelAdmin):
except Exception as e: except Exception as e:
self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) self.message_user(request, f"Error processing file: {e}", level=messages.ERROR)
return redirect("..") return redirect("..")
except Exception as e:
print(f"DEBUG: Voter import failed: {e}")
self.message_user(request, f"Error processing file: {e}", level=messages.ERROR)
return redirect("..")
else: else:
form = VolunteerImportForm(request.POST, request.FILES) form = VolunteerImportForm(request.POST, request.FILES)
if form.is_valid(): if form.is_valid():
@ -1001,6 +1052,10 @@ class EventParticipationAdmin(BaseImportAdminMixin, admin.ModelAdmin):
except Exception as e: except Exception as e:
self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) self.message_user(request, f"Error processing file: {e}", level=messages.ERROR)
return redirect("..") return redirect("..")
except Exception as e:
print(f"DEBUG: Voter import failed: {e}")
self.message_user(request, f"Error processing file: {e}", level=messages.ERROR)
return redirect("..")
else: else:
form = EventParticipationImportForm(request.POST, request.FILES) form = EventParticipationImportForm(request.POST, request.FILES)
if form.is_valid(): if form.is_valid():
@ -1190,6 +1245,10 @@ class DonationAdmin(BaseImportAdminMixin, admin.ModelAdmin):
except Exception as e: except Exception as e:
self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) self.message_user(request, f"Error processing file: {e}", level=messages.ERROR)
return redirect("..") return redirect("..")
except Exception as e:
print(f"DEBUG: Voter import failed: {e}")
self.message_user(request, f"Error processing file: {e}", level=messages.ERROR)
return redirect("..")
else: else:
form = DonationImportForm(request.POST, request.FILES) form = DonationImportForm(request.POST, request.FILES)
if form.is_valid(): if form.is_valid():
@ -1382,6 +1441,10 @@ class InteractionAdmin(BaseImportAdminMixin, admin.ModelAdmin):
except Exception as e: except Exception as e:
self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) self.message_user(request, f"Error processing file: {e}", level=messages.ERROR)
return redirect("..") return redirect("..")
except Exception as e:
print(f"DEBUG: Voter import failed: {e}")
self.message_user(request, f"Error processing file: {e}", level=messages.ERROR)
return redirect("..")
else: else:
form = InteractionImportForm(request.POST, request.FILES) form = InteractionImportForm(request.POST, request.FILES)
if form.is_valid(): if form.is_valid():
@ -1585,6 +1648,10 @@ class VoterLikelihoodAdmin(BaseImportAdminMixin, admin.ModelAdmin):
except Exception as e: except Exception as e:
self.message_user(request, f"Error processing file: {e}", level=messages.ERROR) self.message_user(request, f"Error processing file: {e}", level=messages.ERROR)
return redirect("..") return redirect("..")
except Exception as e:
print(f"DEBUG: Voter import failed: {e}")
self.message_user(request, f"Error processing file: {e}", level=messages.ERROR)
return redirect("..")
else: else:
form = VoterLikelihoodImportForm(request.POST, request.FILES) form = VoterLikelihoodImportForm(request.POST, request.FILES)
if form.is_valid(): if form.is_valid():