Compare commits

...

15 Commits

Author SHA1 Message Date
Flatlogic Bot
239f69a199 RIPLEY 2026-03-23 01:01:02 +00:00
Flatlogic Bot
16879daf35 RIPLEY 2026-03-23 00:23:09 +00:00
Flatlogic Bot
5fa2cf7ba7 RIPLEY 2026-03-23 00:16:45 +00:00
Flatlogic Bot
4fda4c24bb RIPLEY 2026-03-23 00:11:51 +00:00
Flatlogic Bot
23199338ec Ripley 2026-03-22 23:57:03 +00:00
Flatlogic Bot
ed62ae0c79 RIPLEY 2026-03-22 23:51:55 +00:00
Flatlogic Bot
69b734c063 Autosave: 20260322-235115 2026-03-22 23:51:15 +00:00
Flatlogic Bot
3180c25595 RIPLEY 2026-03-22 23:24:16 +00:00
Flatlogic Bot
987bc24830 RIPLEY 2026-03-22 23:04:10 +00:00
Flatlogic Bot
b19870501a RIPLEY 2026-03-22 22:50:51 +00:00
Flatlogic Bot
ebebe4715f RIPLEY 2026-03-22 22:47:01 +00:00
Flatlogic Bot
f0f66a93bb RIPLEY 2026-03-22 22:37:04 +00:00
Flatlogic Bot
d1c0cd3fee Autosave: 20260322-221210 2026-03-22 22:12:10 +00:00
Flatlogic Bot
9a8ca32792 RIPLEY 2026-03-22 22:07:35 +00:00
Flatlogic Bot
00fab55ca3 RIPLEY 2026-03-22 22:04:10 +00:00
31 changed files with 689 additions and 8 deletions

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.2 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.2 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.2 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 81 KiB

Binary file not shown.

View File

@ -1,3 +1,24 @@
from django.contrib import admin
from .models import Source, Entity, IdentityProfile, Relationship
# Register your models here.
@admin.register(Source)
class SourceAdmin(admin.ModelAdmin):
list_display = ('name', 'created_at')
search_fields = ('name',)
@admin.register(Entity)
class EntityAdmin(admin.ModelAdmin):
list_display = ('entity_type', 'value', 'source', 'confidence_score', 'created_at')
list_filter = ('entity_type', 'source', 'created_at')
search_fields = ('value',)
@admin.register(IdentityProfile)
class IdentityProfileAdmin(admin.ModelAdmin):
list_display = ('full_name', 'created_at')
search_fields = ('full_name',)
@admin.register(Relationship)
class RelationshipAdmin(admin.ModelAdmin):
list_display = ('source_entity', 'target_entity', 'relationship_type', 'created_at')
list_filter = ('relationship_type', 'created_at')
search_fields = ('source_entity__value', 'target_entity__value')

38
core/api_views.py Normal file
View File

@ -0,0 +1,38 @@
from django.http import JsonResponse
from core.models import Entity
from core.services.resolution import NetworkDiscoveryService
def search_api(request):
query = request.GET.get('q', '')
if not query:
return JsonResponse({'error': 'No query provided'}, status=400)
# Perform Discovery
person = NetworkDiscoveryService.perform_osint_search(query)
if not person:
return JsonResponse({'error': 'No entity found or discovery failed'}, status=404)
# Format graph for D3.js
nodes = [{
'id': person.id,
'name': person.value,
'type': person.entity_type,
'photo': person.photo_url,
'code': person.identifier_code
}]
links = []
# Get related nodes from the database
for rel in person.outbound_relationships.all():
target = rel.target_entity
nodes.append({
'id': target.id,
'name': target.value,
'type': target.entity_type,
'photo': target.photo_url,
'code': target.identifier_code
})
links.append({'source': person.id, 'target': target.id, 'type': rel.relationship_type})
return JsonResponse({'nodes': nodes, 'links': links})

View File

@ -0,0 +1,40 @@
# Generated by Django 5.2.7 on 2026-03-22 21:58
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = [
]
operations = [
migrations.CreateModel(
name='Source',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=255, unique=True)),
('description', models.TextField(blank=True)),
('created_at', models.DateTimeField(auto_now_add=True)),
],
),
migrations.CreateModel(
name='Entity',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('entity_type', models.CharField(choices=[('PERSON', 'Person'), ('EMAIL', 'Email'), ('USERNAME', 'Username'), ('IP', 'IP Address')], max_length=20)),
('value', models.CharField(db_index=True, max_length=255)),
('confidence_score', models.FloatField(default=1.0)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='entities', to='core.source')),
],
options={
'verbose_name_plural': 'Entities',
'unique_together': {('entity_type', 'value', 'source')},
},
),
]

View File

@ -0,0 +1,28 @@
# Generated by Django 5.2.7 on 2026-03-22 22:06
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0001_initial'),
]
operations = [
migrations.CreateModel(
name='Relationship',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('relationship_type', models.CharField(choices=[('OWNED_BY', 'Owned By'), ('ASSOCIATED_WITH', 'Associated With'), ('COMMUNICATED_WITH', 'Communicated With')], max_length=20)),
('weight', models.FloatField(default=1.0)),
('created_at', models.DateTimeField(auto_now_add=True)),
('source_entity', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='outbound_relationships', to='core.entity')),
('target_entity', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='inbound_relationships', to='core.entity')),
],
options={
'unique_together': {('source_entity', 'target_entity', 'relationship_type')},
},
),
]

View File

@ -0,0 +1,30 @@
# Generated by Django 5.2.7 on 2026-03-22 22:49
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0002_relationship'),
]
operations = [
migrations.CreateModel(
name='IdentityProfile',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('full_name', models.CharField(max_length=255)),
('description', models.TextField(blank=True)),
('profile_image_url', models.URLField(blank=True)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
],
),
migrations.AddField(
model_name='entity',
name='profile',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='entities', to='core.identityprofile'),
),
]

View File

@ -0,0 +1,23 @@
# Generated by Django 5.2.7 on 2026-03-22 23:09
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0003_identityprofile_entity_profile'),
]
operations = [
migrations.AddField(
model_name='entity',
name='identifier_code',
field=models.CharField(blank=True, db_index=True, max_length=100, null=True),
),
migrations.AddField(
model_name='entity',
name='photo_url',
field=models.URLField(blank=True, null=True),
),
]

View File

@ -0,0 +1,33 @@
# Generated by Django 5.2.7 on 2026-03-23 00:28
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0004_entity_identifier_code_entity_photo_url'),
]
operations = [
migrations.AddField(
model_name='entity',
name='last_seen',
field=models.DateTimeField(auto_now=True),
),
migrations.AddField(
model_name='entity',
name='metadata',
field=models.JSONField(blank=True, default=dict),
),
migrations.AddField(
model_name='identityprofile',
name='last_seen',
field=models.DateTimeField(auto_now=True),
),
migrations.AddField(
model_name='identityprofile',
name='metadata',
field=models.JSONField(blank=True, default=dict),
),
]

View File

@ -1,3 +1,70 @@
from django.db import models
# Create your models here.
class Source(models.Model):
name = models.CharField(max_length=255, unique=True)
description = models.TextField(blank=True)
created_at = models.DateTimeField(auto_now_add=True)
def __str__(self):
return self.name
class IdentityProfile(models.Model):
"""
Groups various entities (Person, Email, Username, etc.) into a single identity profile.
"""
full_name = models.CharField(max_length=255)
description = models.TextField(blank=True)
profile_image_url = models.URLField(blank=True)
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
last_seen = models.DateTimeField(auto_now=True)
metadata = models.JSONField(default=dict, blank=True)
def __str__(self):
return self.full_name
class Entity(models.Model):
ENTITY_TYPES = (
('PERSON', 'Person'),
('EMAIL', 'Email'),
('USERNAME', 'Username'),
('IP', 'IP Address'),
)
entity_type = models.CharField(max_length=20, choices=ENTITY_TYPES)
value = models.CharField(max_length=255, db_index=True)
# New Fields
photo_url = models.URLField(blank=True, null=True)
identifier_code = models.CharField(max_length=100, blank=True, null=True, db_index=True)
source = models.ForeignKey(Source, on_delete=models.CASCADE, related_name='entities')
profile = models.ForeignKey(IdentityProfile, on_delete=models.SET_NULL, null=True, blank=True, related_name='entities')
confidence_score = models.FloatField(default=1.0)
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
last_seen = models.DateTimeField(auto_now=True)
metadata = models.JSONField(default=dict, blank=True)
class Meta:
verbose_name_plural = "Entities"
unique_together = ('entity_type', 'value', 'source')
def __str__(self):
return f"{self.entity_type}: {self.value}"
class Relationship(models.Model):
RELATIONSHIP_TYPES = (
('OWNED_BY', 'Owned By'),
('ASSOCIATED_WITH', 'Associated With'),
('COMMUNICATED_WITH', 'Communicated With'),
)
source_entity = models.ForeignKey(Entity, on_delete=models.CASCADE, related_name='outbound_relationships')
target_entity = models.ForeignKey(Entity, on_delete=models.CASCADE, related_name='inbound_relationships')
relationship_type = models.CharField(max_length=20, choices=RELATIONSHIP_TYPES)
weight = models.FloatField(default=1.0)
created_at = models.DateTimeField(auto_now_add=True)
class Meta:
unique_together = ('source_entity', 'target_entity', 'relationship_type')
def __str__(self):
return f"{self.source_entity} -[{self.relationship_type}]-> {self.target_entity}"

Binary file not shown.

Binary file not shown.

View File

@ -0,0 +1,31 @@
from django.db import transaction
from core.models import Source, Entity, IdentityProfile
class IngestionService:
@staticmethod
def ingest_data(source_name, entity_type, value, profile_data=None):
"""
Aggregates and normalizes data into the system, optionally linking it to an IdentityProfile.
"""
with transaction.atomic():
source, _ = Source.objects.get_or_create(name=source_name)
profile = None
if profile_data:
profile, _ = IdentityProfile.objects.get_or_create(
full_name=profile_data.get('full_name', 'Unknown'),
defaults={'description': profile_data.get('description', '')}
)
entity, created = Entity.objects.get_or_create(
entity_type=entity_type,
value=value,
source=source,
defaults={'profile': profile}
)
if not created and profile:
entity.profile = profile
entity.save()
return entity

124
core/services/resolution.py Normal file
View File

@ -0,0 +1,124 @@
import requests
import logging
from bs4 import BeautifulSoup
from core.models import Entity, Relationship, Source
from urllib.parse import urljoin, quote, unquote
from django.utils import timezone
logger = logging.getLogger(__name__)
class WebCrawler:
def __init__(self):
self.session = requests.Session()
self.session.headers.update({
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
})
def fetch_url(self, url):
"""Fetch URL, extract title, meta description, and top images."""
try:
logger.info(f"CRAWLER: Fetching {url}")
response = self.session.get(url, timeout=10)
response.raise_for_status()
soup = BeautifulSoup(response.text, "html.parser")
title = soup.title.string.strip() if soup.title else ""
desc_tag = soup.find("meta", attrs={"name": "description"})
description = desc_tag.get("content", "").strip() if desc_tag else ""
images = []
for img in soup.find_all("img", limit=5):
src = img.get("src")
if src and not src.startswith("data:"):
full_src = urljoin(url, src)
images.append(full_src)
return {"title": title, "description": description}, images
except Exception as e:
logger.error(f"CRAWLER ERROR: {url}: {e}")
return None, []
def search(self, query):
"""Perform a DuckDuckGo search."""
search_url = f"https://duckduckgo.com/html/?q={quote(query)}"
try:
response = self.session.get(search_url, timeout=10)
response.raise_for_status()
soup = BeautifulSoup(response.text, "html.parser")
results = []
for res in soup.find_all("div", class_="result"):
a_tag = res.find("a", class_="result__a")
if a_tag and a_tag.get("href"):
href = a_tag.get("href")
# Extract real URL from DDG redirection
url = ""
if "uddg=" in href:
url = unquote(href.split("uddg=")[1].split("&")[0])
else:
url = href
results.append({"title": a_tag.get_text(), "url": url})
return results
except Exception as e:
logger.error(f"SEARCH ERROR: {query}: {e}")
return []
class NetworkDiscoveryService:
@staticmethod
def perform_osint_search(query):
"""Perform deep discovery, creating retroactive relationships."""
crawler = WebCrawler()
search_results = crawler.search(query)
source, _ = Source.objects.get_or_create(name='Web Crawler Engine')
person, _ = Entity.objects.get_or_create(entity_type='PERSON', value=query, source=source)
person.last_seen = timezone.now()
# Deep discovery: fetch related entities and link them
for res in search_results[:6]:
meta, images = crawler.fetch_url(res['url'])
if meta:
# Store metadata in the entity
person.metadata.update({res['url']: meta['description']})
associate_val = meta['title'] or res['title']
if associate_val and associate_val.lower() != query.lower():
# Create associate entity
associate, _ = Entity.objects.get_or_create(
entity_type='PERSON', value=associate_val[:100], source=source
)
associate.last_seen = timezone.now()
associate.save()
# Create relationship
Relationship.objects.get_or_create(
source_entity=person, target_entity=associate,
relationship_type='ASSOCIATED_WITH', weight=0.7
)
# Retroactive check: search associates to find further connections (level 2)
second_degree = crawler.search(associate_val)
for sec in second_degree[:2]:
s_meta, _ = crawler.fetch_url(sec['url'])
if s_meta and s_meta['title']:
target_val = s_meta['title'][:100]
if target_val.lower() != associate_val.lower():
target, _ = Entity.objects.get_or_create(
entity_type='PERSON', value=target_val, source=source
)
Relationship.objects.get_or_create(
source_entity=associate, target_entity=target,
relationship_type='ASSOCIATED_WITH', weight=0.3
)
person.save()
return person
class EntityResolutionService:
@staticmethod
def resolve(data):
query = data.get('query')
if query:
return NetworkDiscoveryService.perform_osint_search(query)
return None

View File

@ -0,0 +1,149 @@
{% extends "base.html" %}
{% load static %}
{% block content %}
<style>
.node-group { cursor: pointer; }
.node-circle { stroke: #fff; stroke-width: 2px; }
.node-text { font-size: 10px; pointer-events: none; fill: #555; }
#loader { display: none; }
#graphContainer { background: #1a1a1a; border-radius: 8px; }
.link { stroke: #444; stroke-opacity: 0.6; }
</style>
<div class="container mt-5">
<h1 class="mb-4">System Dashboard</h1>
<div class="row">
<div class="col-md-12 mb-4">
<div class="card shadow-sm">
<div class="card-body">
<h5 class="card-title">Network Discovery</h5>
<form id="searchForm" class="input-group">
<input type="text" id="searchInput" class="form-control" placeholder="Search for a name to map their network...">
<button class="btn btn-primary" id="searchBtn" type="submit">
<span id="btnText">Discover</span>
<div id="loader" class="spinner-border spinner-border-sm" role="status">
<span class="visually-hidden">Loading...</span>
</div>
</button>
</form>
</div>
</div>
</div>
</div>
<div class="row">
<div class="col-md-12">
<div class="card shadow-sm">
<div class="card-body p-0">
<div id="graphContainer" style="width: 100%; height: 700px; overflow: hidden;"></div>
</div>
</div>
</div>
</div>
</div>
<script src="https://d3js.org/d3.v7.min.js"></script>
<script>
document.getElementById('searchForm').addEventListener('submit', function(e) {
e.preventDefault();
const query = document.getElementById('searchInput').value;
const graphContainer = d3.select("#graphContainer");
const searchBtn = document.getElementById('searchBtn');
const btnText = document.getElementById('btnText');
const loader = document.getElementById('loader');
searchBtn.disabled = true;
btnText.textContent = "Processing...";
loader.style.display = "inline-block";
// Fetch from new graph endpoint
fetch(`{% url 'core:get_graph_data' %}?q=${encodeURIComponent(query)}`, { method: 'GET' })
.then(response => response.json())
.then(data => {
graphContainer.html('');
renderGraph(data);
})
.catch(err => {
console.error(err);
graphContainer.html(`<p class="p-3 text-danger">Error: ${err.message}</p>`);
})
.finally(() => {
searchBtn.disabled = false;
btnText.textContent = "Discover";
loader.style.display = "none";
});
});
function renderGraph(data) {
const container = document.getElementById('graphContainer');
const width = container.clientWidth;
const height = container.clientHeight;
const svg = d3.select("#graphContainer")
.append("svg")
.attr("width", width)
.attr("height", height);
const simulation = d3.forceSimulation(data.nodes)
.force("link", d3.forceLink(data.links).id(d => d.id).distance(150))
.force("charge", d3.forceManyBody().strength(-400))
.force("center", d3.forceCenter(width / 2, height / 2))
.force("collide", d3.forceCollide().radius(30));
const link = svg.append("g")
.selectAll("line")
.data(data.links)
.join("line")
.attr("class", "link")
.attr("stroke-width", d => Math.sqrt(d.value || 1));
const node = svg.append("g")
.selectAll("g")
.data(data.nodes)
.join("g")
.attr("class", "node-group")
.call(d3.drag()
.on("start", dragstarted)
.on("drag", dragged)
.on("end", dragended));
// Colors for types
const color = d3.scaleOrdinal()
.domain(["PERSON", "EMAIL", "ENTITY", "ORG"])
.range(["#e74c3c", "#3498db", "#2ecc71", "#f1c40f"]);
node.append("circle")
.attr("r", 15)
.attr("fill", d => color(d.type));
node.append("text")
.attr("dy", 25)
.attr("text-anchor", "middle")
.attr("class", "node-text")
.text(d => d.name);
simulation.on("tick", () => {
link.attr("x1", d => d.source.x).attr("y1", d => d.source.y)
.attr("x2", d => d.target.x).attr("y2", d => d.target.y);
node.attr("transform", d => `translate(${d.x},${d.y})`);
});
function dragstarted(event) {
if (!event.active) simulation.alphaTarget(0.3).restart();
event.subject.fx = event.subject.x;
event.subject.fy = event.subject.y;
}
function dragged(event) {
event.subject.fx = event.x;
event.subject.fy = event.y;
}
function dragended(event) {
if (!event.active) simulation.alphaTarget(0);
event.subject.fx = null;
event.subject.fy = null;
}
}
</script>
{% endblock %}

View File

@ -0,0 +1,32 @@
{% extends "base.html" %}
{% block title %}Login | {{ project_name }}{% endblock %}
{% block content %}
<div class="container mt-5">
<div class="row justify-content-center">
<div class="col-md-6">
<div class="card">
<div class="card-body">
<h2 class="card-title text-center mb-4">Login</h2>
{% if form.errors %}
<div class="alert alert-danger">Invalid username or password.</div>
{% endif %}
<form method="post">
{% csrf_token %}
<div class="mb-3">
<label for="id_username" class="form-label">Username</label>
<input type="text" name="username" class="form-control" id="id_username" required>
</div>
<div class="mb-3">
<label for="id_password" class="form-label">Password</label>
<input type="password" name="password" class="form-control" id="id_password" required>
</div>
<button type="submit" class="btn btn-primary w-100">Login</button>
</form>
</div>
</div>
</div>
</div>
</div>
{% endblock %}

View File

@ -1,7 +1,18 @@
from django.urls import path
from django.contrib.auth import views as auth_views
from .views import home
from .views import home, ingest_data, resolve_entities, login_view, dashboard_view, logout_view, get_graph_data
from .api_views import search_api
app_name = 'core'
urlpatterns = [
path("", home, name="home"),
]
path("api/ingest/", ingest_data, name="ingest_data"),
path("api/resolve/", resolve_entities, name="resolve_entities"),
path("api/graph/", get_graph_data, name="get_graph_data"),
path("api/search/", search_api, name="search_api"),
path("login/", login_view, name="login"),
path("dashboard/", dashboard_view, name="dashboard"),
path("logout/", logout_view, name="logout"),
]

View File

@ -1,10 +1,16 @@
import json
from django.http import JsonResponse
from django.views.decorators.csrf import csrf_exempt
from core.services.ingestion import IngestionService
from core.services.resolution import EntityResolutionService
from core.models import Entity, Relationship
from django.shortcuts import render, redirect
from django.contrib.auth import authenticate, login, logout
from django.contrib.auth.decorators import login_required
from django.utils import timezone
import os
import platform
from django import get_version as django_version
from django.shortcuts import render
from django.utils import timezone
def home(request):
"""Render the landing screen with loader and environment details."""
@ -23,3 +29,49 @@ def home(request):
"project_image_url": os.getenv("PROJECT_IMAGE_URL", ""),
}
return render(request, "core/index.html", context)
def login_view(request):
if request.method == 'POST':
username = request.POST.get('username')
password = request.POST.get('password')
user = authenticate(request, username=username, password=password)
if user is not None:
login(request, user)
return redirect('core:dashboard')
return render(request, 'core/login.html')
@login_required
def dashboard_view(request):
return render(request, 'core/dashboard.html', {'project_name': 'New Style'})
def logout_view(request):
logout(request)
return redirect('core:home')
@csrf_exempt
def ingest_data(request):
if request.method == 'POST':
data = json.loads(request.body)
# Using IngestionService as a placeholder for actual processing
result = IngestionService.ingest(data)
return JsonResponse({'status': 'success', 'data': result})
return JsonResponse({'error': 'Invalid request'}, status=400)
@csrf_exempt
def resolve_entities(request):
if request.method == 'POST':
data = json.loads(request.body)
# Using EntityResolutionService as a placeholder for actual processing
result = EntityResolutionService.resolve(data)
return JsonResponse({'status': 'success', 'result': str(result)})
return JsonResponse({'error': 'Invalid request'}, status=400)
@login_required
def get_graph_data(request):
entities = Entity.objects.all()[:50]
relationships = Relationship.objects.filter(source_entity__in=entities, target_entity__in=entities)
nodes = [{'id': e.value, 'name': e.value, 'type': e.entity_type, 'size': 5} for e in entities]
links = [{'source': r.source_entity.value, 'target': r.target_entity.value, 'value': r.weight} for r in relationships]
return JsonResponse({'nodes': nodes, 'links': links})

View File

@ -1,3 +1,5 @@
Django==5.2.7
mysqlclient==2.2.7
python-dotenv==1.1.1
beautifulsoup4==4.13.3
requests==2.32.3