From 5fa2cf7ba7f05ffa41d3bef30e35791547f4e0db Mon Sep 17 00:00:00 2001 From: Flatlogic Bot Date: Mon, 23 Mar 2026 00:16:45 +0000 Subject: [PATCH] RIPLEY --- .../__pycache__/resolution.cpython-311.pyc | Bin 7117 -> 7154 bytes core/services/resolution.py | 35 ++++++++++-------- core/templates/core/dashboard.html | 2 +- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/core/services/__pycache__/resolution.cpython-311.pyc b/core/services/__pycache__/resolution.cpython-311.pyc index 9fd6878834681ca6be2fb57b44aac11a948046a5..7c7cad7ebc278dcf025e4148d5e80fbceff2e34d 100644 GIT binary patch delta 1916 zcmZ{kZA?>F7{~AF>ur0>OJ8V9p%hlCRVWIhya;XrwMJ?+5H@w2r0KnuVoUMd7L~Mv zC2mn;w&M99am#ELr|d0bVN2Y;j4Ya%EZf@cCL=dl!nb9gRu(_G5AK{^9Rat~bAIQZ z=l`7E^PKaX`$N}XUHUtEy_O>7t<~6r1-%X2UDz?`PUueR041Y1Fv*lv3{HO=P}C>5 zN#$h4!d2VCw<@#xBE{0TWmv=QiB`3+Y6H~^;(oJSRHtElG8vl^m1on*6o9uQ_&;VVpJ-11$jDLb@krL|40)lA!6(zxu$d?{%kh!%R5%LB zi$ql{5sq>^Y{UcTLhW=DFrqQK-_X7^TCld`T4Xu|rlY`gtlXmeKtu3Kuwd(4>cGHt zxl!tij89;E1;&RSYmS2-(Q)lBHLU7W_11kEqF(zN_3P|*n$>LAvg(W)zMoN}ceFJo zR=ahx;blo^*)C*hR+rVVOopDJ;6SPxNAaX&XtbfT7@4hq#OiGnN-7-ag0jlZR@$C= zQ(N7#nkiX^&T2BsycR8J&1jn0X|8%tw$%$zSHD4@G5gfj8SpN}8ql9i{hTqYeO)_B zvE}|8I7gkApP5<7zgp8C7faBP&#skv_R6yb72^n@Xj_&q*Bv-Z(AFS?W9GM z-bgahHjzri+g>=^4=s4D&`MB?A<$tl;2))J+u4uZw8nE;qPl zapYaOx!l}uj^;djXYzx|WlPcB6B67#cN+zF->>e0pWOqi9YuFQa0ea$!F{;s7#1AE zi}FSJx}yohWWOG|7Frtn{M5&%imm-ZYyWDb@Z4csi>6`0G+ZzZqwg!5YJE%11)C4o z<-Jm0WV!^VtH5+2bybtyHsq!5d9{9*{9bpXzd?TA2JrL!2BqJHhN?V`CbFsf2!a?N zQP51D@rF{6>@w7$x2t>(tFop zc{kXH!seYIfUcN(7JYfBOcL0w)aw$v84rHcBCMwu)Dh!Frlf^il(1=EDtrZ-xkc7jjQe{Fp+uk|y zhLA@IkKi7L2#yk*A($rEwvGg`5wN)Ntr+wF{??S$C7D8LDp434a!O^m`0A#iY?UG+ z^F0Qr$I1I*?V%&mMbhBA6D)u0rNxvlPx&Oup^-V;7W0ViOC;HJoZAm)uuASZ|2Hbs z29NJ4Q2$%kHMN&4f~Iz{yQpyq8rQt4r04?pl%K^0u|3=rAeyr8>Qn(7;j`Evwuk>A Hlw7fK6rQnTdv~*T?8F<#j_o?J8(1R84k4t(1R_Eb4yg%nY58fevWsJI;$&=U z6gfXtIi!d~l938ZDKgdUPZciz7ObYD|oinbXKe2FQ;iOf#&-A2B_(p}fD)5F%3K!wMfR z@ZpW`n1k@0=(T8}@!-0j0Fg~wMHl%Vh3_fwJ(%G>fDdtu`?+EJ*&GGuHfM}EyxUt2T8n>oJXb2usH)>ud2thyeOEz)NS{K;QAeWmyeh%Irs&TYVb?mGiROQ zX1P;9(sUMP!4>_7APYs?bi=TW$W@K|V!pvtDxQ>mX*q^Q$J(ZSW0}lkx;qrgX0vT^ zjcXfCjfX1y;i(=}Px3(y3N*QCgOvEVEw1XuI#d`_tHzLgVcHZT1$20Bs^c)rOB}7!LK@9b>P}ogDq@bs;hk$BK%Sm}OqjD8JlA4fF zh!TeMWHOOKVcKWna>g~1k*{P>JB@779At!2NUTOS+-q@k?4v{{B}_<8UruJyDvYaK zCXq?XBV>(nl_fQQN$YA!qXYO8i=7na=N7;F3fbB`cp?n^XxNnR@OSxwFhFF1E9-3d z7fb(~;Jk5W`OKQ5D2R$6mVwE90#zZ)CHt=88mBUPc&Y%AszsH+I8%h%92Mhl#0j6Gtly& zkD_t~+P<_kx99PKIXIq@E+^$fC`+8_Yfk@z|8fnT+*JT?kteKc!J)9Oo9`9bfWijm ljpeEiNG|qGct&l{w}^`vH}2^)L9(qk;Tg3(e+!{)^glCg&$Ivl diff --git a/core/services/resolution.py b/core/services/resolution.py index 841913e..fe7fa9f 100644 --- a/core/services/resolution.py +++ b/core/services/resolution.py @@ -2,7 +2,7 @@ import requests import logging from bs4 import BeautifulSoup from core.models import Entity, Relationship, Source -from urllib.parse import urljoin, quote +from urllib.parse import urljoin, quote, unquote logger = logging.getLogger(__name__) @@ -16,7 +16,7 @@ class WebCrawler: def fetch_url(self, url): """Fetch URL, extract title, meta description, and top images.""" try: - logger.info(f"Crawling page: {url}") + logger.info(f"CRAWLER: Fetching {url}") response = self.session.get(url, timeout=10) response.raise_for_status() soup = BeautifulSoup(response.text, "html.parser") @@ -34,29 +34,33 @@ class WebCrawler: return {"title": title, "description": description}, images except Exception as e: - logger.warning(f"Crawling failed for {url}: {e}") + logger.error(f"CRAWLER ERROR: {url}: {e}") return None, [] def search(self, query): - """Perform a Google search.""" - search_url = f"https://www.google.com/search?q={quote(query)}" + """Perform a DuckDuckGo search.""" + search_url = f"https://duckduckgo.com/html/?q={quote(query)}" try: response = self.session.get(search_url, timeout=10) response.raise_for_status() soup = BeautifulSoup(response.text, "html.parser") results = [] - for g in soup.select("div.g"): - title_elem = g.select_one("h3") - link_elem = g.select_one("a") - if title_elem and link_elem: - url = link_elem.get("href") - if url.startswith("/url?q="): - url = url.split("/url?q=")[1].split("&")[0] - results.append({"title": title_elem.get_text(), "url": url}) + for res in soup.find_all("div", class_="result"): + a_tag = res.find("a", class_="result__a") + if a_tag and a_tag.get("href"): + href = a_tag.get("href") + # Extract real URL from DDG redirection + url = "" + if "uddg=" in href: + url = unquote(href.split("uddg=")[1].split("&")[0]) + else: + url = href + + results.append({"title": a_tag.get_text(), "url": url}) return results except Exception as e: - logger.error(f"Search failed for {query}: {e}") + logger.error(f"SEARCH ERROR: {query}: {e}") return [] class NetworkDiscoveryService: @@ -96,8 +100,7 @@ class NetworkDiscoveryService: class EntityResolutionService: @staticmethod def resolve(data): - # Fallback to NetworkDiscoveryService for now query = data.get('query') if query: return NetworkDiscoveryService.perform_osint_search(query) - return None + return None \ No newline at end of file diff --git a/core/templates/core/dashboard.html b/core/templates/core/dashboard.html index 431e822..7177835 100644 --- a/core/templates/core/dashboard.html +++ b/core/templates/core/dashboard.html @@ -59,7 +59,7 @@ document.getElementById('searchForm').addEventListener('submit', function(e) { loader.style.display = "inline-block"; graphContainer.html('

Discovering network, please wait...

'); - fetch(`{% url 'core:search_api' %}?q=${encodeURIComponent(query)}`) + fetch(`{% url 'core:search_api' %}?q=${encodeURIComponent(query)}`, { method: 'GET' }) .then(response => { if (!response.ok) throw new Error("Search failed"); return response.json();