RIPLEY

core/migrations/0005_entity_last_seen_entity_metadata_and_more.py

@@ -0,0 +1,33 @@
+# Generated by Django 5.2.7 on 2026-03-23 00:28
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0004_entity_identifier_code_entity_photo_url'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='entity',
+            name='last_seen',
+            field=models.DateTimeField(auto_now=True),
+        ),
+        migrations.AddField(
+            model_name='entity',
+            name='metadata',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+        migrations.AddField(
+            model_name='identityprofile',
+            name='last_seen',
+            field=models.DateTimeField(auto_now=True),
+        ),
+        migrations.AddField(
+            model_name='identityprofile',
+            name='metadata',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+    ]

core/models.py

@@ -17,6 +17,8 @@ class IdentityProfile(models.Model):
     profile_image_url = models.URLField(blank=True)
     created_at = models.DateTimeField(auto_now_add=True)
     updated_at = models.DateTimeField(auto_now=True)
+    last_seen = models.DateTimeField(auto_now=True)
+    metadata = models.JSONField(default=dict, blank=True)
 
     def __str__(self):
         return self.full_name
@@ -39,6 +41,8 @@ class Entity(models.Model):
     confidence_score = models.FloatField(default=1.0)
     created_at = models.DateTimeField(auto_now_add=True)
     updated_at = models.DateTimeField(auto_now=True)
+    last_seen = models.DateTimeField(auto_now=True)
+    metadata = models.JSONField(default=dict, blank=True)
 
     class Meta:
         verbose_name_plural = "Entities"

core/services/resolution.py

@@ -3,6 +3,7 @@ import logging
 from bs4 import BeautifulSoup
 from core.models import Entity, Relationship, Source
 from urllib.parse import urljoin, quote, unquote
+from django.utils import timezone
 
 logger = logging.getLogger(__name__)
 
@@ -66,35 +67,51 @@ class WebCrawler:
 class NetworkDiscoveryService:
     @staticmethod
     def perform_osint_search(query):
-        """Perform discovery using Web Crawling, extracting metadata and images."""
+        """Perform deep discovery, creating retroactive relationships."""
         crawler = WebCrawler()
         search_results = crawler.search(query)
         
         source, _ = Source.objects.get_or_create(name='Web Crawler Engine')
         person, _ = Entity.objects.get_or_create(entity_type='PERSON', value=query, source=source)
+        person.last_seen = timezone.now()
         
-        # Use first valid image found among search results if available
-        found_photo = None
-        
-        # Increase search limit to 6 and improve crawling logic
+        # Deep discovery: fetch related entities and link them
         for res in search_results[:6]:
             meta, images = crawler.fetch_url(res['url'])
-            
-            if images and not found_photo:
-                found_photo = images[0]
-            
             if meta:
+                # Store metadata in the entity
+                person.metadata.update({res['url']: meta['description']})
+                
                 associate_val = meta['title'] or res['title']
                 if associate_val and associate_val.lower() != query.lower():
+                    # Create associate entity
                     associate, _ = Entity.objects.get_or_create(
                         entity_type='PERSON', value=associate_val[:100], source=source
                     )
+                    associate.last_seen = timezone.now()
+                    associate.save()
+                    
+                    # Create relationship
                     Relationship.objects.get_or_create(
                         source_entity=person, target_entity=associate, 
-                        relationship_type='ASSOCIATED_WITH', weight=0.5
+                        relationship_type='ASSOCIATED_WITH', weight=0.7
                     )
                     
-        person.photo_url = found_photo or f"https://api.dicebear.com/7.x/initials/svg?seed={quote(query)}"
+                    # Retroactive check: search associates to find further connections (level 2)
+                    second_degree = crawler.search(associate_val)
+                    for sec in second_degree[:2]:
+                        s_meta, _ = crawler.fetch_url(sec['url'])
+                        if s_meta and s_meta['title']:
+                            target_val = s_meta['title'][:100]
+                            if target_val.lower() != associate_val.lower():
+                                target, _ = Entity.objects.get_or_create(
+                                    entity_type='PERSON', value=target_val, source=source
+                                )
+                                Relationship.objects.get_or_create(
+                                    source_entity=associate, target_entity=target, 
+                                    relationship_type='ASSOCIATED_WITH', weight=0.3
+                                )
+        
         person.save()
         return person
 

core/templates/core/dashboard.html

@@ -5,8 +5,10 @@
 <style>
     .node-group { cursor: pointer; }
     .node-circle { stroke: #fff; stroke-width: 2px; }
-    .node-text { font-size: 10px; pointer-events: none; }
+    .node-text { font-size: 10px; pointer-events: none; fill: #555; }
     #loader { display: none; }
+    #graphContainer { background: #1a1a1a; border-radius: 8px; }
+    .link { stroke: #444; stroke-opacity: 0.6; }
 </style>
 
 <div class="container mt-5">
@@ -34,9 +36,8 @@
     <div class="row">
         <div class="col-md-12">
             <div class="card shadow-sm">
-                <div class="card-body">
-                    <h5 class="card-title">Network Visualization</h5>
-                    <div id="graphContainer" style="width: 100%; height: 600px; background: #f8f9fa; border: 1px solid #ddd; overflow: hidden;"></div>
+                <div class="card-body p-0">
+                    <div id="graphContainer" style="width: 100%; height: 700px; overflow: hidden;"></div>
                 </div>
             </div>
         </div>
@@ -53,22 +54,19 @@ document.getElementById('searchForm').addEventListener('submit', function(e) {
     const btnText = document.getElementById('btnText');
     const loader = document.getElementById('loader');
 
-    // UI Loading state
     searchBtn.disabled = true;
-    btnText.textContent = "Searching...";
+    btnText.textContent = "Processing...";
     loader.style.display = "inline-block";
-    graphContainer.html('<p class="p-3 text-muted">Discovering network, please wait...</p>');
     
-    fetch(`{% url 'core:search_api' %}?q=${encodeURIComponent(query)}`, { method: 'GET' })
-        .then(response => {
-            if (!response.ok) throw new Error("Search failed");
-            return response.json();
-        })
+    // Fetch from new graph endpoint
+    fetch(`{% url 'core:get_graph_data' %}?q=${encodeURIComponent(query)}`, { method: 'GET' })
+        .then(response => response.json())
         .then(data => {
-            graphContainer.html(''); // clear
+            graphContainer.html('');
             renderGraph(data);
         })
         .catch(err => {
+            console.error(err);
             graphContainer.html(`<p class="p-3 text-danger">Error: ${err.message}</p>`);
         })
         .finally(() => {
@@ -79,29 +77,27 @@ document.getElementById('searchForm').addEventListener('submit', function(e) {
 });
 
 function renderGraph(data) {
-    const width = 800;
-    const height = 600;
+    const container = document.getElementById('graphContainer');
+    const width = container.clientWidth;
+    const height = container.clientHeight;
     
-    // Create SVG and enforce bounds
     const svg = d3.select("#graphContainer")
                   .append("svg")
-                  .attr("width", "100%")
-                  .attr("height", "100%")
-                  .attr("viewBox", [0, 0, width, height]);
+                  .attr("width", width)
+                  .attr("height", height);
 
     const simulation = d3.forceSimulation(data.nodes)
-        .force("link", d3.forceLink(data.links).id(d => d.id).distance(100))
-        .force("charge", d3.forceManyBody().strength(-300))
+        .force("link", d3.forceLink(data.links).id(d => d.id).distance(150))
+        .force("charge", d3.forceManyBody().strength(-400))
         .force("center", d3.forceCenter(width / 2, height / 2))
-        .force("x", d3.forceX(width / 2).strength(0.05))
-        .force("y", d3.forceY(height / 2).strength(0.05));
+        .force("collide", d3.forceCollide().radius(30));
 
     const link = svg.append("g")
         .selectAll("line")
         .data(data.links)
         .join("line")
-        .attr("stroke", "#999")
-        .attr("stroke-width", 1);
+        .attr("class", "link")
+        .attr("stroke-width", d => Math.sqrt(d.value || 1));
 
     const node = svg.append("g")
         .selectAll("g")
@@ -113,36 +109,22 @@ function renderGraph(data) {
             .on("drag", dragged)
             .on("end", dragended));
 
-    // Reduce node sizes significantly
-    const radius = 12;
-    node.append("circle")
-        .attr("r", radius)
-        .attr("class", "node-circle")
-        .attr("fill", d => d.type === 'PERSON' ? '#e74c3c' : '#3498db');
+    // Colors for types
+    const color = d3.scaleOrdinal()
+        .domain(["PERSON", "EMAIL", "ENTITY", "ORG"])
+        .range(["#e74c3c", "#3498db", "#2ecc71", "#f1c40f"]);
 
-    // Add Image if available, scaled down
-    node.filter(d => d.photo)
-        .append("image")
-        .attr("xlink:href", d => d.photo)
-        .attr("x", -radius)
-        .attr("y", -radius)
-        .attr("width", radius * 2)
-        .attr("height", radius * 2)
-        .attr("clip-path", `circle(${radius}px)`);
+    node.append("circle")
+        .attr("r", 15)
+        .attr("fill", d => color(d.type));
 
     node.append("text")
-        .attr("dy", radius + 15)
+        .attr("dy", 25)
         .attr("text-anchor", "middle")
         .attr("class", "node-text")
         .text(d => d.name);
 
     simulation.on("tick", () => {
-        // Enforce boundary constraints
-        node.each(d => {
-            d.x = Math.max(radius, Math.min(width - radius, d.x));
-            d.y = Math.max(radius, Math.min(height - radius, d.y));
-        });
-
         link.attr("x1", d => d.source.x).attr("y1", d => d.source.y)
             .attr("x2", d => d.target.x).attr("y2", d => d.target.y);
         node.attr("transform", d => `translate(${d.x},${d.y})`);
@@ -153,12 +135,10 @@ function renderGraph(data) {
         event.subject.fx = event.subject.x;
         event.subject.fy = event.subject.y;
     }
-
     function dragged(event) {
         event.subject.fx = event.x;
         event.subject.fy = event.y;
     }
-
     function dragended(event) {
         if (!event.active) simulation.alphaTarget(0);
         event.subject.fx = null;

core/urls.py

@@ -1,7 +1,7 @@
 from django.urls import path
 from django.contrib.auth import views as auth_views
 
-from .views import home, ingest_data, resolve_entities, login_view, dashboard_view, logout_view
+from .views import home, ingest_data, resolve_entities, login_view, dashboard_view, logout_view, get_graph_data
 from .api_views import search_api
 
 app_name = 'core'
@@ -10,6 +10,7 @@ urlpatterns = [
     path("", home, name="home"),
     path("api/ingest/", ingest_data, name="ingest_data"),
     path("api/resolve/", resolve_entities, name="resolve_entities"),
+    path("api/graph/", get_graph_data, name="get_graph_data"),
     path("api/search/", search_api, name="search_api"),
     path("login/", login_view, name="login"),
     path("dashboard/", dashboard_view, name="dashboard"),

core/views.py

@@ -3,6 +3,7 @@ from django.http import JsonResponse
 from django.views.decorators.csrf import csrf_exempt
 from core.services.ingestion import IngestionService
 from core.services.resolution import EntityResolutionService
+from core.models import Entity, Relationship
 from django.shortcuts import render, redirect
 from django.contrib.auth import authenticate, login, logout
 from django.contrib.auth.decorators import login_required
@@ -62,5 +63,15 @@ def resolve_entities(request):
         data = json.loads(request.body)
         # Using EntityResolutionService as a placeholder for actual processing
         result = EntityResolutionService.resolve(data)
-        return JsonResponse({'status': 'success', 'result': result})
+        return JsonResponse({'status': 'success', 'result': str(result)})
     return JsonResponse({'error': 'Invalid request'}, status=400)
+
+@login_required
+def get_graph_data(request):
+    entities = Entity.objects.all()[:50]
+    relationships = Relationship.objects.filter(source_entity__in=entities, target_entity__in=entities)
+    
+    nodes = [{'id': e.value, 'name': e.value, 'type': e.entity_type, 'size': 5} for e in entities]
+    links = [{'source': r.source_entity.value, 'target': r.target_entity.value, 'value': r.weight} for r in relationships]
+    
+    return JsonResponse({'nodes': nodes, 'links': links})