.2
This commit is contained in:
Flatlogic Bot
parent
4056b17780
commit
9c3c5219c6
Binary file not shown.
117
core/admin.py
117
core/admin.py
@ -240,7 +240,7 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin):
|
||||
mapping[field_name] = request.POST.get(f"map_{field_name}")
|
||||
|
||||
try:
|
||||
with open(file_path, "r", encoding="UTF-8") as f:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
# Optimization: Fast count and partial preview
|
||||
total_count = sum(1 for line in f) - 1
|
||||
f.seek(0)
|
||||
@ -289,6 +289,7 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin):
|
||||
self.message_user(request, f"Error processing preview: {e}", level=messages.ERROR)
|
||||
return redirect("..")
|
||||
|
||||
|
||||
elif "_import" in request.POST:
|
||||
file_path = request.POST.get("file_path")
|
||||
tenant_id = request.POST.get("tenant")
|
||||
@ -298,22 +299,25 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin):
|
||||
|
||||
try:
|
||||
count = 0
|
||||
created_count = 0
|
||||
updated_count = 0
|
||||
skipped_no_change = 0
|
||||
skipped_no_id = 0
|
||||
errors = 0
|
||||
failed_rows = []
|
||||
batch_size = 500 # Optimized batch size
|
||||
batch_size = 500
|
||||
|
||||
# Pre-calculate choice dicts and sets
|
||||
support_choices = dict(Voter.SUPPORT_CHOICES)
|
||||
yard_sign_choices = dict(Voter.YARD_SIGN_CHOICES)
|
||||
window_sticker_choices = dict(Voter.WINDOW_STICKER_CHOICES)
|
||||
phone_type_choices = dict(Voter.PHONE_TYPE_CHOICES)
|
||||
phone_type_reverse = {v.lower(): k for k, v in phone_type_choices.items()}
|
||||
|
||||
# Fields to fetch for change detection
|
||||
valid_fields = {f.name for f in Voter._meta.get_fields()}
|
||||
mapped_fields = {f for f in mapping.keys() if f in valid_fields}
|
||||
fetch_fields = list(mapped_fields | {"voter_id", "address", "phone", "longitude"})
|
||||
update_fields = list(mapped_fields | {"address", "phone"})
|
||||
fetch_fields = list(mapped_fields | {"voter_id", "address", "phone", "longitude", "latitude"})
|
||||
# Ensure derived/special fields are in update_fields
|
||||
update_fields = list(mapped_fields | {"address", "phone", "longitude", "latitude"})
|
||||
if "voter_id" in update_fields: update_fields.remove("voter_id")
|
||||
|
||||
def chunk_reader(reader, size):
|
||||
@ -326,15 +330,18 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin):
|
||||
if chunk:
|
||||
yield chunk
|
||||
|
||||
with open(file_path, "r", encoding="UTF-8") as f:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
reader = csv.DictReader(f)
|
||||
v_id_col = mapping.get("voter_id")
|
||||
if not v_id_col:
|
||||
raise ValueError("Voter ID mapping is missing")
|
||||
|
||||
print(f"DEBUG: Starting voter import. Tenant: {tenant.name}. Voter ID column: {v_id_col}")
|
||||
|
||||
total_processed = 0
|
||||
for chunk_index, chunk in enumerate(chunk_reader(reader, batch_size)):
|
||||
with transaction.atomic():
|
||||
voter_ids = [row.get(v_id_col) for row in chunk if row.get(v_id_col)]
|
||||
voter_ids = [str(row.get(v_id_col)).strip() for row in chunk if row.get(v_id_col)]
|
||||
existing_voters = {v.voter_id: v for v in Voter.objects.filter(tenant=tenant, voter_id__in=voter_ids).only(*fetch_fields)}
|
||||
|
||||
to_create = []
|
||||
@ -342,9 +349,19 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin):
|
||||
processed_in_batch = set()
|
||||
|
||||
for row in chunk:
|
||||
total_processed += 1
|
||||
try:
|
||||
voter_id = row.get(v_id_col)
|
||||
if not voter_id or voter_id in processed_in_batch:
|
||||
raw_voter_id = row.get(v_id_col)
|
||||
if raw_voter_id is None:
|
||||
skipped_no_id += 1
|
||||
continue
|
||||
|
||||
voter_id = str(raw_voter_id).strip()
|
||||
if not voter_id:
|
||||
skipped_no_id += 1
|
||||
continue
|
||||
|
||||
if voter_id in processed_in_batch:
|
||||
continue
|
||||
processed_in_batch.add(voter_id)
|
||||
|
||||
@ -359,25 +376,37 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin):
|
||||
for field_name, csv_col in mapping.items():
|
||||
if field_name == "voter_id": continue
|
||||
val = row.get(csv_col)
|
||||
if val is None or str(val).strip() == "": continue
|
||||
if val is None: continue
|
||||
val = str(val).strip()
|
||||
if val == "": continue
|
||||
|
||||
# Type-specific conversions
|
||||
if field_name == "is_targeted":
|
||||
val = str(val).lower() in ["true", "1", "yes"]
|
||||
elif field_name in ["birthdate", "registration_date"]:
|
||||
try:
|
||||
if isinstance(val, str):
|
||||
val = datetime.strptime(val.strip(), "%Y-%m-%d").date()
|
||||
except:
|
||||
pass
|
||||
orig_val = val
|
||||
parsed_date = None
|
||||
for fmt in ["%Y-%m-%d", "%m/%d/%Y", "%d/%m/%Y", "%Y/%m/%d"]:
|
||||
try:
|
||||
parsed_date = datetime.strptime(val, fmt).date()
|
||||
break
|
||||
except:
|
||||
continue
|
||||
if parsed_date:
|
||||
val = parsed_date
|
||||
else:
|
||||
# If parsing fails, keep original or skip? Let's skip updating this field.
|
||||
continue
|
||||
elif field_name == "candidate_support":
|
||||
val = val.lower().replace(" ", "_")
|
||||
if val not in support_choices: val = "unknown"
|
||||
elif field_name == "yard_sign":
|
||||
val = val.lower().replace(" ", "_")
|
||||
if val not in yard_sign_choices: val = "none"
|
||||
elif field_name == "window_sticker":
|
||||
val = val.lower().replace(" ", "_")
|
||||
if val not in window_sticker_choices: val = "none"
|
||||
elif field_name == "phone_type":
|
||||
val_lower = str(val).lower()
|
||||
val_lower = val.lower()
|
||||
if val_lower in phone_type_choices:
|
||||
val = val_lower
|
||||
elif val_lower in phone_type_reverse:
|
||||
@ -385,11 +414,11 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin):
|
||||
else:
|
||||
val = "cell"
|
||||
|
||||
if getattr(voter, field_name) != val:
|
||||
current_val = getattr(voter, field_name)
|
||||
if current_val != val:
|
||||
setattr(voter, field_name, val)
|
||||
changed = True
|
||||
|
||||
# Special fields
|
||||
old_phone = voter.phone
|
||||
voter.phone = format_phone_number(voter.phone)
|
||||
if voter.phone != old_phone:
|
||||
@ -411,16 +440,19 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin):
|
||||
changed = True
|
||||
|
||||
if not changed:
|
||||
skipped_no_change += 1
|
||||
continue
|
||||
|
||||
if created:
|
||||
to_create.append(voter)
|
||||
created_count += 1
|
||||
else:
|
||||
to_update.append(voter)
|
||||
updated_count += 1
|
||||
|
||||
count += 1
|
||||
except Exception as e:
|
||||
logger.error(f"Error importing row: {e}")
|
||||
print(f"DEBUG: Error importing row {total_processed}: {e}")
|
||||
row["Import Error"] = str(e)
|
||||
failed_rows.append(row)
|
||||
errors += 1
|
||||
@ -430,11 +462,14 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin):
|
||||
if to_update:
|
||||
Voter.objects.bulk_update(to_update, update_fields, batch_size=250)
|
||||
|
||||
logger.info(f"Voter import progress: Processed batch {chunk_index + 1}. Total successes: {count}")
|
||||
print(f"DEBUG: Voter import progress: {total_processed} processed. {count} created/updated. {skipped_no_change} skipped (no change). {skipped_no_id} skipped (no ID). {errors} errors.")
|
||||
|
||||
if os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
self.message_user(request, f"Successfully imported {count} voters.")
|
||||
|
||||
success_msg = f"Import complete: {count} voters created/updated. ({created_count} new, {updated_count} updated, {skipped_no_change} skipped with no changes, {skipped_no_id} skipped missing ID, {errors} errors)"
|
||||
self.message_user(request, success_msg)
|
||||
|
||||
request.session[f"{self.model._meta.model_name}_import_errors"] = failed_rows
|
||||
request.session.modified = True
|
||||
if errors > 0:
|
||||
@ -442,7 +477,7 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin):
|
||||
self.message_user(request, mark_safe(f"Failed to import {errors} rows. <a href='{error_url}' download>Download failed records</a>"), level=messages.WARNING)
|
||||
return redirect("..")
|
||||
except Exception as e:
|
||||
logger.exception("Voter import failed")
|
||||
print(f"DEBUG: Voter import failed: {e}")
|
||||
self.message_user(request, f"Error processing file: {e}", level=messages.ERROR)
|
||||
return redirect("..")
|
||||
else:
|
||||
@ -460,7 +495,7 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin):
|
||||
tmp.write(chunk)
|
||||
file_path = tmp.name
|
||||
|
||||
with open(file_path, "r", encoding="UTF-8") as f:
|
||||
with open(file_path, "r", encoding="utf-8-sig") as f:
|
||||
reader = csv.reader(f)
|
||||
headers = next(reader)
|
||||
|
||||
@ -475,6 +510,14 @@ class VoterAdmin(BaseImportAdminMixin, admin.ModelAdmin):
|
||||
"opts": self.model._meta,
|
||||
})
|
||||
return render(request, "admin/import_mapping.html", context)
|
||||
else:
|
||||
form = VoterImportForm()
|
||||
|
||||
context = self.admin_site.each_context(request)
|
||||
context["form"] = form
|
||||
context["title"] = "Import Voters"
|
||||
context["opts"] = self.model._meta
|
||||
return render(request, "admin/import_csv.html", context)
|
||||
class EventAdmin(BaseImportAdminMixin, admin.ModelAdmin):
|
||||
list_display = ('id', 'name', 'event_type', 'date', 'start_time', 'end_time', 'tenant')
|
||||
list_filter = ('tenant', 'date', 'event_type')
|
||||
@ -626,6 +669,10 @@ class EventAdmin(BaseImportAdminMixin, admin.ModelAdmin):
|
||||
except Exception as e:
|
||||
self.message_user(request, f"Error processing file: {e}", level=messages.ERROR)
|
||||
return redirect("..")
|
||||
except Exception as e:
|
||||
print(f"DEBUG: Voter import failed: {e}")
|
||||
self.message_user(request, f"Error processing file: {e}", level=messages.ERROR)
|
||||
return redirect("..")
|
||||
else:
|
||||
form = EventImportForm(request.POST, request.FILES)
|
||||
if form.is_valid():
|
||||
@ -789,6 +836,10 @@ class VolunteerAdmin(BaseImportAdminMixin, admin.ModelAdmin):
|
||||
except Exception as e:
|
||||
self.message_user(request, f"Error processing file: {e}", level=messages.ERROR)
|
||||
return redirect("..")
|
||||
except Exception as e:
|
||||
print(f"DEBUG: Voter import failed: {e}")
|
||||
self.message_user(request, f"Error processing file: {e}", level=messages.ERROR)
|
||||
return redirect("..")
|
||||
else:
|
||||
form = VolunteerImportForm(request.POST, request.FILES)
|
||||
if form.is_valid():
|
||||
@ -1001,6 +1052,10 @@ class EventParticipationAdmin(BaseImportAdminMixin, admin.ModelAdmin):
|
||||
except Exception as e:
|
||||
self.message_user(request, f"Error processing file: {e}", level=messages.ERROR)
|
||||
return redirect("..")
|
||||
except Exception as e:
|
||||
print(f"DEBUG: Voter import failed: {e}")
|
||||
self.message_user(request, f"Error processing file: {e}", level=messages.ERROR)
|
||||
return redirect("..")
|
||||
else:
|
||||
form = EventParticipationImportForm(request.POST, request.FILES)
|
||||
if form.is_valid():
|
||||
@ -1190,6 +1245,10 @@ class DonationAdmin(BaseImportAdminMixin, admin.ModelAdmin):
|
||||
except Exception as e:
|
||||
self.message_user(request, f"Error processing file: {e}", level=messages.ERROR)
|
||||
return redirect("..")
|
||||
except Exception as e:
|
||||
print(f"DEBUG: Voter import failed: {e}")
|
||||
self.message_user(request, f"Error processing file: {e}", level=messages.ERROR)
|
||||
return redirect("..")
|
||||
else:
|
||||
form = DonationImportForm(request.POST, request.FILES)
|
||||
if form.is_valid():
|
||||
@ -1382,6 +1441,10 @@ class InteractionAdmin(BaseImportAdminMixin, admin.ModelAdmin):
|
||||
except Exception as e:
|
||||
self.message_user(request, f"Error processing file: {e}", level=messages.ERROR)
|
||||
return redirect("..")
|
||||
except Exception as e:
|
||||
print(f"DEBUG: Voter import failed: {e}")
|
||||
self.message_user(request, f"Error processing file: {e}", level=messages.ERROR)
|
||||
return redirect("..")
|
||||
else:
|
||||
form = InteractionImportForm(request.POST, request.FILES)
|
||||
if form.is_valid():
|
||||
@ -1585,6 +1648,10 @@ class VoterLikelihoodAdmin(BaseImportAdminMixin, admin.ModelAdmin):
|
||||
except Exception as e:
|
||||
self.message_user(request, f"Error processing file: {e}", level=messages.ERROR)
|
||||
return redirect("..")
|
||||
except Exception as e:
|
||||
print(f"DEBUG: Voter import failed: {e}")
|
||||
self.message_user(request, f"Error processing file: {e}", level=messages.ERROR)
|
||||
return redirect("..")
|
||||
else:
|
||||
form = VoterLikelihoodImportForm(request.POST, request.FILES)
|
||||
if form.is_valid():
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user