diff --git a/core/__pycache__/models.cpython-311.pyc b/core/__pycache__/models.cpython-311.pyc index 75858ff..4346ab3 100644 Binary files a/core/__pycache__/models.cpython-311.pyc and b/core/__pycache__/models.cpython-311.pyc differ diff --git a/core/__pycache__/urls.cpython-311.pyc b/core/__pycache__/urls.cpython-311.pyc index 483a4fe..9ff043e 100644 Binary files a/core/__pycache__/urls.cpython-311.pyc and b/core/__pycache__/urls.cpython-311.pyc differ diff --git a/core/__pycache__/views.cpython-311.pyc b/core/__pycache__/views.cpython-311.pyc index 3d61730..c4e6f28 100644 Binary files a/core/__pycache__/views.cpython-311.pyc and b/core/__pycache__/views.cpython-311.pyc differ diff --git a/core/migrations/0005_entity_last_seen_entity_metadata_and_more.py b/core/migrations/0005_entity_last_seen_entity_metadata_and_more.py new file mode 100644 index 0000000..a36eec5 --- /dev/null +++ b/core/migrations/0005_entity_last_seen_entity_metadata_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 5.2.7 on 2026-03-23 00:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0004_entity_identifier_code_entity_photo_url'), + ] + + operations = [ + migrations.AddField( + model_name='entity', + name='last_seen', + field=models.DateTimeField(auto_now=True), + ), + migrations.AddField( + model_name='entity', + name='metadata', + field=models.JSONField(blank=True, default=dict), + ), + migrations.AddField( + model_name='identityprofile', + name='last_seen', + field=models.DateTimeField(auto_now=True), + ), + migrations.AddField( + model_name='identityprofile', + name='metadata', + field=models.JSONField(blank=True, default=dict), + ), + ] diff --git a/core/migrations/__pycache__/0005_entity_last_seen_entity_metadata_and_more.cpython-311.pyc b/core/migrations/__pycache__/0005_entity_last_seen_entity_metadata_and_more.cpython-311.pyc new file mode 100644 index 0000000..398d0fc Binary files /dev/null and b/core/migrations/__pycache__/0005_entity_last_seen_entity_metadata_and_more.cpython-311.pyc differ diff --git a/core/models.py b/core/models.py index 1b1c389..3d9badc 100644 --- a/core/models.py +++ b/core/models.py @@ -17,6 +17,8 @@ class IdentityProfile(models.Model): profile_image_url = models.URLField(blank=True) created_at = models.DateTimeField(auto_now_add=True) updated_at = models.DateTimeField(auto_now=True) + last_seen = models.DateTimeField(auto_now=True) + metadata = models.JSONField(default=dict, blank=True) def __str__(self): return self.full_name @@ -39,6 +41,8 @@ class Entity(models.Model): confidence_score = models.FloatField(default=1.0) created_at = models.DateTimeField(auto_now_add=True) updated_at = models.DateTimeField(auto_now=True) + last_seen = models.DateTimeField(auto_now=True) + metadata = models.JSONField(default=dict, blank=True) class Meta: verbose_name_plural = "Entities" diff --git a/core/services/__pycache__/resolution.cpython-311.pyc b/core/services/__pycache__/resolution.cpython-311.pyc index 6ec4cfb..dd312bc 100644 Binary files a/core/services/__pycache__/resolution.cpython-311.pyc and b/core/services/__pycache__/resolution.cpython-311.pyc differ diff --git a/core/services/resolution.py b/core/services/resolution.py index 4d44eb0..9b0396b 100644 --- a/core/services/resolution.py +++ b/core/services/resolution.py @@ -3,6 +3,7 @@ import logging from bs4 import BeautifulSoup from core.models import Entity, Relationship, Source from urllib.parse import urljoin, quote, unquote +from django.utils import timezone logger = logging.getLogger(__name__) @@ -66,35 +67,51 @@ class WebCrawler: class NetworkDiscoveryService: @staticmethod def perform_osint_search(query): - """Perform discovery using Web Crawling, extracting metadata and images.""" + """Perform deep discovery, creating retroactive relationships.""" crawler = WebCrawler() search_results = crawler.search(query) source, _ = Source.objects.get_or_create(name='Web Crawler Engine') person, _ = Entity.objects.get_or_create(entity_type='PERSON', value=query, source=source) + person.last_seen = timezone.now() - # Use first valid image found among search results if available - found_photo = None - - # Increase search limit to 6 and improve crawling logic + # Deep discovery: fetch related entities and link them for res in search_results[:6]: meta, images = crawler.fetch_url(res['url']) - - if images and not found_photo: - found_photo = images[0] - if meta: + # Store metadata in the entity + person.metadata.update({res['url']: meta['description']}) + associate_val = meta['title'] or res['title'] if associate_val and associate_val.lower() != query.lower(): + # Create associate entity associate, _ = Entity.objects.get_or_create( entity_type='PERSON', value=associate_val[:100], source=source ) + associate.last_seen = timezone.now() + associate.save() + + # Create relationship Relationship.objects.get_or_create( source_entity=person, target_entity=associate, - relationship_type='ASSOCIATED_WITH', weight=0.5 + relationship_type='ASSOCIATED_WITH', weight=0.7 ) + + # Retroactive check: search associates to find further connections (level 2) + second_degree = crawler.search(associate_val) + for sec in second_degree[:2]: + s_meta, _ = crawler.fetch_url(sec['url']) + if s_meta and s_meta['title']: + target_val = s_meta['title'][:100] + if target_val.lower() != associate_val.lower(): + target, _ = Entity.objects.get_or_create( + entity_type='PERSON', value=target_val, source=source + ) + Relationship.objects.get_or_create( + source_entity=associate, target_entity=target, + relationship_type='ASSOCIATED_WITH', weight=0.3 + ) - person.photo_url = found_photo or f"https://api.dicebear.com/7.x/initials/svg?seed={quote(query)}" person.save() return person @@ -104,4 +121,4 @@ class EntityResolutionService: query = data.get('query') if query: return NetworkDiscoveryService.perform_osint_search(query) - return None + return None \ No newline at end of file diff --git a/core/templates/core/dashboard.html b/core/templates/core/dashboard.html index 47f4595..304eaed 100644 --- a/core/templates/core/dashboard.html +++ b/core/templates/core/dashboard.html @@ -5,8 +5,10 @@
@@ -34,9 +36,8 @@
-
-
Network Visualization
-
+
+
@@ -53,22 +54,19 @@ document.getElementById('searchForm').addEventListener('submit', function(e) { const btnText = document.getElementById('btnText'); const loader = document.getElementById('loader'); - // UI Loading state searchBtn.disabled = true; - btnText.textContent = "Searching..."; + btnText.textContent = "Processing..."; loader.style.display = "inline-block"; - graphContainer.html('

Discovering network, please wait...

'); - - fetch(`{% url 'core:search_api' %}?q=${encodeURIComponent(query)}`, { method: 'GET' }) - .then(response => { - if (!response.ok) throw new Error("Search failed"); - return response.json(); - }) + + // Fetch from new graph endpoint + fetch(`{% url 'core:get_graph_data' %}?q=${encodeURIComponent(query)}`, { method: 'GET' }) + .then(response => response.json()) .then(data => { - graphContainer.html(''); // clear + graphContainer.html(''); renderGraph(data); }) .catch(err => { + console.error(err); graphContainer.html(`

Error: ${err.message}

`); }) .finally(() => { @@ -79,29 +77,27 @@ document.getElementById('searchForm').addEventListener('submit', function(e) { }); function renderGraph(data) { - const width = 800; - const height = 600; + const container = document.getElementById('graphContainer'); + const width = container.clientWidth; + const height = container.clientHeight; - // Create SVG and enforce bounds const svg = d3.select("#graphContainer") .append("svg") - .attr("width", "100%") - .attr("height", "100%") - .attr("viewBox", [0, 0, width, height]); + .attr("width", width) + .attr("height", height); const simulation = d3.forceSimulation(data.nodes) - .force("link", d3.forceLink(data.links).id(d => d.id).distance(100)) - .force("charge", d3.forceManyBody().strength(-300)) + .force("link", d3.forceLink(data.links).id(d => d.id).distance(150)) + .force("charge", d3.forceManyBody().strength(-400)) .force("center", d3.forceCenter(width / 2, height / 2)) - .force("x", d3.forceX(width / 2).strength(0.05)) - .force("y", d3.forceY(height / 2).strength(0.05)); + .force("collide", d3.forceCollide().radius(30)); const link = svg.append("g") .selectAll("line") .data(data.links) .join("line") - .attr("stroke", "#999") - .attr("stroke-width", 1); + .attr("class", "link") + .attr("stroke-width", d => Math.sqrt(d.value || 1)); const node = svg.append("g") .selectAll("g") @@ -113,36 +109,22 @@ function renderGraph(data) { .on("drag", dragged) .on("end", dragended)); - // Reduce node sizes significantly - const radius = 12; - node.append("circle") - .attr("r", radius) - .attr("class", "node-circle") - .attr("fill", d => d.type === 'PERSON' ? '#e74c3c' : '#3498db'); + // Colors for types + const color = d3.scaleOrdinal() + .domain(["PERSON", "EMAIL", "ENTITY", "ORG"]) + .range(["#e74c3c", "#3498db", "#2ecc71", "#f1c40f"]); - // Add Image if available, scaled down - node.filter(d => d.photo) - .append("image") - .attr("xlink:href", d => d.photo) - .attr("x", -radius) - .attr("y", -radius) - .attr("width", radius * 2) - .attr("height", radius * 2) - .attr("clip-path", `circle(${radius}px)`); + node.append("circle") + .attr("r", 15) + .attr("fill", d => color(d.type)); node.append("text") - .attr("dy", radius + 15) + .attr("dy", 25) .attr("text-anchor", "middle") .attr("class", "node-text") .text(d => d.name); simulation.on("tick", () => { - // Enforce boundary constraints - node.each(d => { - d.x = Math.max(radius, Math.min(width - radius, d.x)); - d.y = Math.max(radius, Math.min(height - radius, d.y)); - }); - link.attr("x1", d => d.source.x).attr("y1", d => d.source.y) .attr("x2", d => d.target.x).attr("y2", d => d.target.y); node.attr("transform", d => `translate(${d.x},${d.y})`); @@ -153,12 +135,10 @@ function renderGraph(data) { event.subject.fx = event.subject.x; event.subject.fy = event.subject.y; } - function dragged(event) { event.subject.fx = event.x; event.subject.fy = event.y; } - function dragended(event) { if (!event.active) simulation.alphaTarget(0); event.subject.fx = null; @@ -166,4 +146,4 @@ function renderGraph(data) { } } -{% endblock %} +{% endblock %} \ No newline at end of file diff --git a/core/urls.py b/core/urls.py index e069edd..b8916c9 100644 --- a/core/urls.py +++ b/core/urls.py @@ -1,7 +1,7 @@ from django.urls import path from django.contrib.auth import views as auth_views -from .views import home, ingest_data, resolve_entities, login_view, dashboard_view, logout_view +from .views import home, ingest_data, resolve_entities, login_view, dashboard_view, logout_view, get_graph_data from .api_views import search_api app_name = 'core' @@ -10,8 +10,9 @@ urlpatterns = [ path("", home, name="home"), path("api/ingest/", ingest_data, name="ingest_data"), path("api/resolve/", resolve_entities, name="resolve_entities"), + path("api/graph/", get_graph_data, name="get_graph_data"), path("api/search/", search_api, name="search_api"), path("login/", login_view, name="login"), path("dashboard/", dashboard_view, name="dashboard"), path("logout/", logout_view, name="logout"), -] +] \ No newline at end of file diff --git a/core/views.py b/core/views.py index 5fc1bea..6bd09f5 100644 --- a/core/views.py +++ b/core/views.py @@ -3,6 +3,7 @@ from django.http import JsonResponse from django.views.decorators.csrf import csrf_exempt from core.services.ingestion import IngestionService from core.services.resolution import EntityResolutionService +from core.models import Entity, Relationship from django.shortcuts import render, redirect from django.contrib.auth import authenticate, login, logout from django.contrib.auth.decorators import login_required @@ -62,5 +63,15 @@ def resolve_entities(request): data = json.loads(request.body) # Using EntityResolutionService as a placeholder for actual processing result = EntityResolutionService.resolve(data) - return JsonResponse({'status': 'success', 'result': result}) - return JsonResponse({'error': 'Invalid request'}, status=400) \ No newline at end of file + return JsonResponse({'status': 'success', 'result': str(result)}) + return JsonResponse({'error': 'Invalid request'}, status=400) + +@login_required +def get_graph_data(request): + entities = Entity.objects.all()[:50] + relationships = Relationship.objects.filter(source_entity__in=entities, target_entity__in=entities) + + nodes = [{'id': e.value, 'name': e.value, 'type': e.entity_type, 'size': 5} for e in entities] + links = [{'source': r.source_entity.value, 'target': r.target_entity.value, 'value': r.weight} for r in relationships] + + return JsonResponse({'nodes': nodes, 'links': links}) \ No newline at end of file