diff --git a/celery.log b/celery.log new file mode 100644 index 0000000..4cefb20 --- /dev/null +++ b/celery.log @@ -0,0 +1,154 @@ + + -------------- celery@pool-python-d07be985 v5.4.0 (opalescent) +--- ***** ----- +-- ******* ---- Linux-6.1.0-42-cloud-amd64-x86_64-with-glibc2.36 2026-02-08 16:57:58 +- *** --- * --- +- ** ---------- [config] +- ** ---------- .> app: config:0x7f0365fc7010 +- ** ---------- .> transport: redis://localhost:6379/0 +- ** ---------- .> results: redis://localhost:6379/0 +- *** --- * --- .> concurrency: 2 (prefork) +-- ******* ---- .> task events: OFF (enable -E to monitor tasks in this worker) +--- ***** ----- + -------------- [queues] + .> celery exchange=celery(direct) key=celery + + +[tasks] + . config.celery.debug_task + . core.tasks.generate_summary + . core.tasks.process_bookmark + +[2026-02-08 16:57:58,648: WARNING/MainProcess] /home/ubuntu/.local/lib/python3.11/site-packages/celery/worker/consumer/consumer.py:508: CPendingDeprecationWarning: The broker_connection_retry configuration setting will no longer determine +whether broker connection retries are made during startup in Celery 6.0 and above. +If you wish to retain the existing behavior for retrying connections on startup, +you should set broker_connection_retry_on_startup to True. + warnings.warn( + +[2026-02-08 16:57:58,669: INFO/MainProcess] Connected to redis://localhost:6379/0 +[2026-02-08 16:57:58,672: WARNING/MainProcess] /home/ubuntu/.local/lib/python3.11/site-packages/celery/worker/consumer/consumer.py:508: CPendingDeprecationWarning: The broker_connection_retry configuration setting will no longer determine +whether broker connection retries are made during startup in Celery 6.0 and above. +If you wish to retain the existing behavior for retrying connections on startup, +you should set broker_connection_retry_on_startup to True. + warnings.warn( + +[2026-02-08 16:57:58,676: INFO/MainProcess] mingle: searching for neighbors +[2026-02-08 16:57:59,699: INFO/MainProcess] mingle: all alone +[2026-02-08 16:57:59,722: INFO/MainProcess] celery@pool-python-d07be985 ready. +[2026-02-08 16:58:54,807: INFO/MainProcess] Task core.tasks.process_bookmark[6aeefd1e-df28-42d3-bb17-d6836c389361] received +[2026-02-08 16:58:55,106: INFO/ForkPoolWorker-2] HTTP Request: GET https://openai.com "HTTP/1.1 403 Forbidden" +[2026-02-08 16:58:55,121: WARNING/ForkPoolWorker-2] Error fetching bookmark 9 (https://openai.com): Client error '403 Forbidden' for url 'https://openai.com' +For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/403. Trying base domain backup. +[2026-02-08 16:58:55,121: ERROR/ForkPoolWorker-2] Error fetching base domain for bookmark 9: Client error '403 Forbidden' for url 'https://openai.com' +For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/403 +[2026-02-08 16:58:55,222: INFO/MainProcess] Task core.tasks.generate_summary[b7f3ce50-3949-4d4b-8124-986fc6fed32d] received +[2026-02-08 16:58:55,230: INFO/ForkPoolWorker-2] Task core.tasks.process_bookmark[6aeefd1e-df28-42d3-bb17-d6836c389361] succeeded in 0.41664800100261346s: 'Processed bookmark 9' +[2026-02-08 16:59:06,405: INFO/ForkPoolWorker-1] Task core.tasks.generate_summary[b7f3ce50-3949-4d4b-8124-986fc6fed32d] succeeded in 11.157703020959161s: 'Generated summary for bookmark 9' +[2026-02-08 17:49:01,106: INFO/MainProcess] Task core.tasks.process_bookmark[480f8119-0f34-4c8d-af74-8223f6d86777] received +[2026-02-08 17:49:01,495: INFO/ForkPoolWorker-2] HTTP Request: GET https://aimlapi.com/ "HTTP/1.1 200 OK" +[2026-02-08 17:49:01,833: INFO/MainProcess] Task core.tasks.generate_summary[5f9a1a29-9b56-4219-b073-86aa584fa574] received +[2026-02-08 17:49:01,840: INFO/ForkPoolWorker-2] Task core.tasks.process_bookmark[480f8119-0f34-4c8d-af74-8223f6d86777] succeeded in 0.7128028109436855s: 'Processed bookmark 10' +[2026-02-08 17:49:12,749: INFO/ForkPoolWorker-1] Task core.tasks.generate_summary[5f9a1a29-9b56-4219-b073-86aa584fa574] succeeded in 10.906358937965706s: 'Generated summary for bookmark 10' +[2026-02-08 17:50:13,199: INFO/MainProcess] Task core.tasks.process_bookmark[38a7f2f4-6b5e-48ab-8ce5-b973aecf0829] received +[2026-02-08 17:50:13,401: INFO/ForkPoolWorker-2] HTTP Request: GET https://dropoverapp.com/ "HTTP/1.1 200 OK" +[2026-02-08 17:50:13,476: INFO/MainProcess] Task core.tasks.generate_summary[f6142f4c-c377-44cf-8921-4de37564f845] received +[2026-02-08 17:50:13,486: INFO/ForkPoolWorker-2] Task core.tasks.process_bookmark[38a7f2f4-6b5e-48ab-8ce5-b973aecf0829] succeeded in 0.28576223901472986s: 'Processed bookmark 11' +[2026-02-08 17:50:19,030: INFO/ForkPoolWorker-1] Task core.tasks.generate_summary[f6142f4c-c377-44cf-8921-4de37564f845] succeeded in 5.548579726018943s: 'Generated summary for bookmark 11' +[2026-02-08 17:54:37,900: INFO/MainProcess] Task core.tasks.process_bookmark[49edb552-63b2-44e3-b6bf-29ea3921cbc0] received +[2026-02-08 17:54:39,151: INFO/ForkPoolWorker-2] HTTP Request: GET https://www.wikipedia.org/ "HTTP/1.1 200 OK" +[2026-02-08 17:54:39,938: INFO/MainProcess] Task core.tasks.generate_summary[62fbb6ba-951b-469a-ac79-19663a33d9a4] received +[2026-02-08 17:54:39,982: INFO/ForkPoolWorker-2] Task core.tasks.process_bookmark[49edb552-63b2-44e3-b6bf-29ea3921cbc0] succeeded in 1.9269755689892918s: 'Processed bookmark 12' +[2026-02-08 17:54:46,341: INFO/ForkPoolWorker-1] Task core.tasks.generate_summary[62fbb6ba-951b-469a-ac79-19663a33d9a4] succeeded in 6.309643072017934s: 'Generated summary for bookmark 12' +[2026-02-08 17:55:04,551: INFO/MainProcess] Task core.tasks.process_bookmark[20ea2b59-ed0f-4970-ad43-cea2bc3597e0] received +[2026-02-08 17:55:04,768: INFO/ForkPoolWorker-2] HTTP Request: GET https://www.wikipedia.org/ "HTTP/1.1 200 OK" +[2026-02-08 17:55:05,030: INFO/MainProcess] Task core.tasks.generate_summary[e29ad0ec-2683-44f9-964a-6883351d3e53] received +[2026-02-08 17:55:05,034: INFO/ForkPoolWorker-2] Task core.tasks.process_bookmark[20ea2b59-ed0f-4970-ad43-cea2bc3597e0] succeeded in 0.4778669200022705s: 'Processed bookmark 12' +[2026-02-08 17:55:15,878: INFO/ForkPoolWorker-1] Task core.tasks.generate_summary[e29ad0ec-2683-44f9-964a-6883351d3e53] succeeded in 10.846816856996156s: 'Generated summary for bookmark 12' +[2026-02-08 17:55:38,808: INFO/MainProcess] Task core.tasks.process_bookmark[631857db-83e4-448e-91bc-5964ff822b82] received +[2026-02-08 17:55:38,955: INFO/ForkPoolWorker-2] HTTP Request: GET https://openai.com "HTTP/1.1 403 Forbidden" +[2026-02-08 17:55:38,975: WARNING/ForkPoolWorker-2] Error fetching bookmark 9 (https://openai.com): Client error '403 Forbidden' for url 'https://openai.com' +For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/403. Trying base domain backup. +[2026-02-08 17:55:38,976: ERROR/ForkPoolWorker-2] Error fetching base domain for bookmark 9: Client error '403 Forbidden' for url 'https://openai.com' +For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/403 +[2026-02-08 17:55:38,992: INFO/MainProcess] Task core.tasks.generate_summary[ac5da805-5c8e-4641-ba7d-594b569f9472] received +[2026-02-08 17:55:38,996: INFO/ForkPoolWorker-2] Task core.tasks.process_bookmark[631857db-83e4-448e-91bc-5964ff822b82] succeeded in 0.18439359701005742s: 'Processed bookmark 9' +[2026-02-08 17:55:49,704: INFO/ForkPoolWorker-1] Task core.tasks.generate_summary[ac5da805-5c8e-4641-ba7d-594b569f9472] succeeded in 10.710596179007553s: 'Generated summary for bookmark 9' +[2026-02-08 17:56:04,496: INFO/MainProcess] Task core.tasks.process_bookmark[8e44cfbd-da5d-4f2c-a670-205d1f35397c] received +[2026-02-08 17:56:06,477: INFO/ForkPoolWorker-2] HTTP Request: GET https://www.wikipedia.org/ "HTTP/1.1 200 OK" +[2026-02-08 17:56:10,002: INFO/MainProcess] Task core.tasks.generate_summary[d89d853b-545c-47ef-8d88-b3645948b928] received +[2026-02-08 17:56:10,231: INFO/ForkPoolWorker-2] Task core.tasks.process_bookmark[8e44cfbd-da5d-4f2c-a670-205d1f35397c] succeeded in 5.731945501989685s: 'Processed bookmark 12' +[2026-02-08 17:56:23,118: INFO/ForkPoolWorker-1] Task core.tasks.generate_summary[d89d853b-545c-47ef-8d88-b3645948b928] succeeded in 13.114061183994636s: 'Generated summary for bookmark 12' + + -------------- celery@pool-python-d07be985 v5.4.0 (opalescent) +--- ***** ----- +-- ******* ---- Linux-6.1.0-42-cloud-amd64-x86_64-with-glibc2.36 2026-02-08 19:37:54 +- *** --- * --- +- ** ---------- [config] +- ** ---------- .> app: config:0x7f7767509510 +- ** ---------- .> transport: redis://localhost:6379/0 +- ** ---------- .> results: redis://localhost:6379/0 +- *** --- * --- .> concurrency: 2 (prefork) +-- ******* ---- .> task events: OFF (enable -E to monitor tasks in this worker) +--- ***** ----- + -------------- [queues] + .> celery exchange=celery(direct) key=celery + + +[tasks] + . config.celery.debug_task + . core.tasks.generate_summary + . core.tasks.process_bookmark + +[2026-02-08 19:37:55,136: WARNING/MainProcess] /home/ubuntu/.local/lib/python3.11/site-packages/celery/worker/consumer/consumer.py:508: CPendingDeprecationWarning: The broker_connection_retry configuration setting will no longer determine +whether broker connection retries are made during startup in Celery 6.0 and above. +If you wish to retain the existing behavior for retrying connections on startup, +you should set broker_connection_retry_on_startup to True. + warnings.warn( + +[2026-02-08 19:37:55,230: INFO/MainProcess] Connected to redis://localhost:6379/0 +[2026-02-08 19:37:55,233: WARNING/MainProcess] /home/ubuntu/.local/lib/python3.11/site-packages/celery/worker/consumer/consumer.py:508: CPendingDeprecationWarning: The broker_connection_retry configuration setting will no longer determine +whether broker connection retries are made during startup in Celery 6.0 and above. +If you wish to retain the existing behavior for retrying connections on startup, +you should set broker_connection_retry_on_startup to True. + warnings.warn( + +[2026-02-08 19:37:55,266: INFO/MainProcess] mingle: searching for neighbors +[2026-02-08 19:37:56,302: INFO/MainProcess] mingle: all alone +[2026-02-08 19:37:56,334: INFO/MainProcess] celery@pool-python-d07be985 ready. +[2026-02-08 19:37:56,351: INFO/MainProcess] Task core.tasks.process_bookmark[49276cb8-da4c-4e3b-a10c-1eff3e4358fc] received +[2026-02-08 19:37:56,364: INFO/MainProcess] Task core.tasks.process_bookmark[4cc4cf11-e28f-4a72-9d09-4da9b24bce6e] received +[2026-02-08 19:37:56,819: INFO/ForkPoolWorker-2] HTTP Request: GET https://dropoverapp.com/ "HTTP/1.1 200 OK" +[2026-02-08 19:37:56,955: INFO/ForkPoolWorker-1] HTTP Request: GET https://www.strella.io/ "HTTP/1.1 200 OK" +[2026-02-08 19:37:57,117: INFO/MainProcess] Task core.tasks.generate_summary[7ca1dbb6-005a-4fcf-b5d0-5fd6a2ae8ac5] received +[2026-02-08 19:37:57,130: INFO/ForkPoolWorker-2] Task core.tasks.process_bookmark[49276cb8-da4c-4e3b-a10c-1eff3e4358fc] succeeded in 0.7769331529852934s: 'Processed bookmark 13' +[2026-02-08 19:37:57,193: INFO/ForkPoolWorker-2] Generating summary/tags for bookmark 13... +[2026-02-08 19:37:57,361: INFO/MainProcess] Task core.tasks.generate_summary[71b0f9cb-36dc-4eb4-91f7-f6e8bba9c54d] received +[2026-02-08 19:37:57,389: INFO/ForkPoolWorker-1] Task core.tasks.process_bookmark[4cc4cf11-e28f-4a72-9d09-4da9b24bce6e] succeeded in 1.0219957570079714s: 'Processed bookmark 14' +[2026-02-08 19:37:57,419: INFO/ForkPoolWorker-1] Generating summary/tags for bookmark 14... +[2026-02-08 19:37:58,896: INFO/MainProcess] Events of group {task} enabled by remote. +[2026-02-08 19:38:07,952: INFO/ForkPoolWorker-2] AI Raw Response for 13: { + "summary": "Dropover is a macOS utility that provides floating shelves to collect, organize, and batch-move dragged items (files, folders, images, URLs, text) for streamlined file management. It offers built-in file actions, cloud uploads and sharing, plus extensive customization and automation features (custom actions, scripts, keyboard shortcuts, Siri Shortcuts) for power users.", + "tags": ["filesharing", "sharing", "documents", "productivity"] +} +[2026-02-08 19:38:07,953: INFO/ForkPoolWorker-2] Decoded JSON for 13: summary=True, tags=['filesharing', 'sharing', 'documents', 'productivity'] +[2026-02-08 19:38:08,022: INFO/ForkPoolWorker-2] Successfully added tags ['filesharing', 'sharing', 'documents', 'productivity'] to bookmark 13 +[2026-02-08 19:38:08,024: INFO/ForkPoolWorker-2] Task core.tasks.generate_summary[7ca1dbb6-005a-4fcf-b5d0-5fd6a2ae8ac5] succeeded in 10.876905062003061s: 'Generated summary and tags for bookmark 13' +[2026-02-08 19:38:08,341: INFO/ForkPoolWorker-1] AI Raw Response for 14: { + "summary": "Strella is an AI-powered customer research platform that runs AI-moderated interviews, recruits participants, generates discussion guides, and analyzes responses to deliver actionable insights within hours. It’s built for teams (UX, product, consumer insights, marketing) to accelerate market research, usability testing, and concept validation and to share findings with stakeholders.", + "tags": ["marketing", "company", "sharing", "development"] +} +[2026-02-08 19:38:08,342: INFO/ForkPoolWorker-1] Decoded JSON for 14: summary=True, tags=['marketing', 'company', 'sharing', 'development'] +[2026-02-08 19:38:08,422: INFO/ForkPoolWorker-1] Successfully added tags ['marketing', 'company', 'sharing', 'development'] to bookmark 14 +[2026-02-08 19:38:08,426: INFO/ForkPoolWorker-1] Task core.tasks.generate_summary[71b0f9cb-36dc-4eb4-91f7-f6e8bba9c54d] succeeded in 11.034683802979998s: 'Generated summary and tags for bookmark 14' +[2026-02-09 04:08:42,072: INFO/MainProcess] Task core.tasks.process_bookmark[cd622357-234e-4dc9-9fab-eefa651ea2a4] received +[2026-02-09 04:08:43,746: INFO/ForkPoolWorker-2] HTTP Request: GET https://www.strella.io/ "HTTP/1.1 200 OK" +[2026-02-09 04:08:45,048: INFO/MainProcess] Task core.tasks.generate_summary[ba68ccad-d413-41cc-8b1c-65384d2792ab] received +[2026-02-09 04:08:45,073: INFO/ForkPoolWorker-2] Task core.tasks.process_bookmark[cd622357-234e-4dc9-9fab-eefa651ea2a4] succeeded in 2.6813507160404697s: 'Processed bookmark 15' +[2026-02-09 04:08:45,647: INFO/ForkPoolWorker-1] Generating summary/tags for bookmark 15... +[2026-02-09 04:08:51,671: INFO/ForkPoolWorker-1] AI Raw Response for 15: { + "summary": "Strella is an AI-powered customer research platform that runs AI-moderated interviews, recruits targeted participants, and analyzes responses to generate actionable insights in hours. It’s designed for teams (UX, product, consumer insights, marketing) to accelerate research workflows, produce unbiased discussion guides, and share findings with stakeholders.", + "tags": ["marketing", "company", "productivity", "sharing"] +} +[2026-02-09 04:08:51,673: INFO/ForkPoolWorker-1] Decoded JSON for 15: summary=True, tags=['marketing', 'company', 'productivity', 'sharing'] +[2026-02-09 04:08:51,759: INFO/ForkPoolWorker-1] Successfully added tags ['marketing', 'company', 'productivity', 'sharing'] to bookmark 15 +[2026-02-09 04:08:51,894: INFO/ForkPoolWorker-1] Task core.tasks.generate_summary[ba68ccad-d413-41cc-8b1c-65384d2792ab] succeeded in 6.7007439360022545s: 'Generated summary and tags for bookmark 15' diff --git a/check_import.py b/check_import.py new file mode 100644 index 0000000..10a9ae9 --- /dev/null +++ b/check_import.py @@ -0,0 +1,6 @@ + +try: + from taggit.serializers import TagListSerializerField, TaggitSerializer + print("Import SUCCESS") +except ImportError as e: + print(f"Import FAILED: {e}") diff --git a/config/__pycache__/settings.cpython-311.pyc b/config/__pycache__/settings.cpython-311.pyc index 7eeb2e3..1dd5fe8 100644 Binary files a/config/__pycache__/settings.cpython-311.pyc and b/config/__pycache__/settings.cpython-311.pyc differ diff --git a/config/__pycache__/urls.cpython-311.pyc b/config/__pycache__/urls.cpython-311.pyc index 5d3b77d..a62b69c 100644 Binary files a/config/__pycache__/urls.cpython-311.pyc and b/config/__pycache__/urls.cpython-311.pyc differ diff --git a/config/settings.py b/config/settings.py index 9c8820b..19e2b83 100644 --- a/config/settings.py +++ b/config/settings.py @@ -207,7 +207,7 @@ CELERY_RESULT_SERIALIZER = 'json' CELERY_TIMEZONE = TIME_ZONE # Run tasks synchronously in development (no Redis required) -CELERY_TASK_ALWAYS_EAGER = True +CELERY_TASK_ALWAYS_EAGER = False CELERY_TASK_EAGER_PROPAGATES = True # Login/Logout Redirects diff --git a/config/urls.py b/config/urls.py index 4da058d..ff44bea 100644 --- a/config/urls.py +++ b/config/urls.py @@ -1,10 +1,13 @@ from django.contrib import admin -from django.urls import path, include +from django.urls import path, include, re_path from django.contrib.auth import views as auth_views +from revproxy.views import ProxyView +from django.contrib.admin.views.decorators import staff_member_required urlpatterns = [ path('admin/', admin.site.urls), path('accounts/login/', auth_views.LoginView.as_view(), name='login'), path('accounts/logout/', auth_views.LogoutView.as_view(), name='logout'), + re_path(r'^flower/(?P.*)$', staff_member_required(ProxyView.as_view(upstream='http://127.0.0.1:5555/flower/'))), path('', include('core.urls')), -] +] \ No newline at end of file diff --git a/core/__pycache__/api_views.cpython-311.pyc b/core/__pycache__/api_views.cpython-311.pyc index 4245aaf..d518a20 100644 Binary files a/core/__pycache__/api_views.cpython-311.pyc and b/core/__pycache__/api_views.cpython-311.pyc differ diff --git a/core/__pycache__/serializers.cpython-311.pyc b/core/__pycache__/serializers.cpython-311.pyc index 593f032..198adfc 100644 Binary files a/core/__pycache__/serializers.cpython-311.pyc and b/core/__pycache__/serializers.cpython-311.pyc differ diff --git a/core/__pycache__/tasks.cpython-311.pyc b/core/__pycache__/tasks.cpython-311.pyc index 3eba445..0a6bed3 100644 Binary files a/core/__pycache__/tasks.cpython-311.pyc and b/core/__pycache__/tasks.cpython-311.pyc differ diff --git a/core/__pycache__/tests.cpython-311.pyc b/core/__pycache__/tests.cpython-311.pyc new file mode 100644 index 0000000..4c29352 Binary files /dev/null and b/core/__pycache__/tests.cpython-311.pyc differ diff --git a/core/__pycache__/views.cpython-311.pyc b/core/__pycache__/views.cpython-311.pyc index 9c74dc9..8790f16 100644 Binary files a/core/__pycache__/views.cpython-311.pyc and b/core/__pycache__/views.cpython-311.pyc differ diff --git a/core/api_views.py b/core/api_views.py index 8f062f2..89c66f3 100644 --- a/core/api_views.py +++ b/core/api_views.py @@ -2,6 +2,7 @@ from rest_framework import viewsets, permissions, filters from rest_framework.views import APIView from rest_framework.response import Response from django_filters.rest_framework import DjangoFilterBackend +from django.db.models import Q from core.models import Bookmark, Team from core.serializers import BookmarkSerializer, BookmarkDetailSerializer, TeamSerializer from core.tasks import process_bookmark @@ -24,7 +25,11 @@ class BookmarkViewSet(viewsets.ModelViewSet): ordering = ['-created_at'] def get_queryset(self): - return Bookmark.objects.filter(user=self.request.user).select_related('extraction') + user_teams = self.request.user.teams.all() + return Bookmark.objects.filter( + Q(user=self.request.user) | + Q(shares__team__in=user_teams) + ).distinct().select_related('extraction', 'summary') def get_serializer_class(self): if self.action == 'retrieve': @@ -40,4 +45,4 @@ class TeamViewSet(viewsets.ModelViewSet): serializer_class = TeamSerializer def get_queryset(self): - return self.request.user.teams.all() \ No newline at end of file + return self.request.user.teams.all() diff --git a/core/serializers.py b/core/serializers.py index bd60b84..32a5784 100644 --- a/core/serializers.py +++ b/core/serializers.py @@ -13,12 +13,23 @@ class TeamSerializer(serializers.ModelSerializer): model = Team fields = ['id', 'name', 'description', 'created_at'] +class ExtractionSerializer(serializers.ModelSerializer): + class Meta: + model = Extraction + fields = ['content_text', 'extracted_at'] + +class SummarySerializer(serializers.ModelSerializer): + class Meta: + model = Summary + fields = ['content', 'generated_at'] + class BookmarkSerializer(TaggitSerializer, serializers.ModelSerializer): tags = TagListSerializerField(required=False) + summary = SummarySerializer(read_only=True) class Meta: model = Bookmark - fields = ['id', 'url', 'title', 'notes', 'is_favorite', 'tags', 'created_at', 'updated_at'] + fields = ['id', 'url', 'title', 'notes', 'is_favorite', 'tags', 'summary', 'created_at', 'updated_at'] read_only_fields = ['id', 'created_at', 'updated_at'] def create(self, validated_data): @@ -28,11 +39,6 @@ class BookmarkSerializer(TaggitSerializer, serializers.ModelSerializer): bookmark.tags.set(tags) return bookmark -class ExtractionSerializer(serializers.ModelSerializer): - class Meta: - model = Extraction - fields = ['content_text', 'extracted_at'] - class BookmarkDetailSerializer(BookmarkSerializer): extraction = ExtractionSerializer(read_only=True) diff --git a/core/tasks.py b/core/tasks.py index dc1829b..24cc8d1 100644 --- a/core/tasks.py +++ b/core/tasks.py @@ -7,6 +7,8 @@ from bs4 import BeautifulSoup import html2text import logging from urllib.parse import urlparse +from taggit.models import Tag +import json logger = logging.getLogger(__name__) @@ -100,12 +102,16 @@ def process_bookmark(self, bookmark_id): def generate_summary(bookmark_id): try: bookmark = Bookmark.objects.get(id=bookmark_id) - extraction = bookmark.extraction except Bookmark.DoesNotExist: return + + try: + extraction = bookmark.extraction except Extraction.DoesNotExist: - # If extraction doesn't exist yet, we might want to wait or just return - # But in EAGER mode it should be there. + Summary.objects.update_or_create( + bookmark=bookmark, + defaults={'content': "Content extraction failed or is still in progress. AI summary cannot be generated."} + ) return content_to_summarize = extraction.content_text.strip() @@ -118,29 +124,71 @@ def generate_summary(bookmark_id): ) return + # Check if we should generate tags (only if bookmark has no tags) + should_generate_tags = bookmark.tags.count() == 0 + existing_tags = list(Tag.objects.values_list('name', flat=True).distinct()[:50]) + existing_tags_str = ", ".join(existing_tags) + # Prepare prompt for AI - if used_backup: - prompt = f"The specific page '{bookmark.url}' could not be reached. Summarize the main domain front page content instead to describe what this website is about.\n\nContent:\n{content_to_summarize[:4000]}" + system_prompt = "You are a helpful assistant that summarizes web content and suggests tags for researchers. Be concise and professional. Always return response in JSON format." + + user_prompt = f"Analyze the following content from the webpage '{bookmark.title or bookmark.url}'.\n\n" + user_prompt += "1. Provide a summary in 2-3 concise sentences.\n" + + if should_generate_tags: + user_prompt += "2. Suggest 3-5 short and concise tags for this content.\n" + if existing_tags: + user_prompt += f"Prioritize these existing tags if they match: {existing_tags_str}\n" + + user_prompt += "\nReturn your response in valid JSON format:\n" + user_prompt += "{\n \"summary\": \"your summary here\"" + if should_generate_tags: + user_prompt += ",\n \"tags\": [\"tag1\", \"tag2\", \"tag3\"]\n" else: - prompt = f"Summarize the following content from the webpage '{bookmark.title or bookmark.url}' in 2-3 concise sentences. Focus on the main points for a researcher.\n\nContent:\n{content_to_summarize[:4000]}" + user_prompt += "\n" + user_prompt += "}\n\n" + user_prompt += f"Content:\n{content_to_summarize[:4000]}" try: + logger.info(f"Generating summary/tags for bookmark {bookmark_id}...") response = LocalAIApi.create_response({ "input": [ - {"role": "system", "content": "You are a helpful assistant that summarizes web content for researchers and knowledge workers. Be concise and professional."}, - {"role": "user", "content": prompt}, + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt}, ], + # "response_format": {"type": "json_object"} # Some proxies might not like this }) summary_text = None + suggested_tags = [] + if response.get("success"): - summary_text = LocalAIApi.extract_text(response) + raw_text = LocalAIApi.extract_text(response) + logger.info(f"AI Raw Response for {bookmark_id}: {raw_text}") + data = LocalAIApi.decode_json_from_response(response) + if data: + summary_text = data.get("summary") + suggested_tags = data.get("tags", []) + logger.info(f"Decoded JSON for {bookmark_id}: summary={bool(summary_text)}, tags={suggested_tags}") + else: + logger.warning(f"JSON decoding failed for {bookmark_id}. Fallback to text.") + summary_text = raw_text if summary_text and len(summary_text.strip()) > 10: Summary.objects.update_or_create( bookmark=bookmark, defaults={'content': summary_text.strip()} ) + + # Add tags if we should + if should_generate_tags and suggested_tags: + # Limit to 5 tags and ensure they are strings + valid_tags = [str(t)[:50] for t in suggested_tags if t][:5] + if valid_tags: + bookmark.tags.add(*valid_tags) + logger.info(f"Successfully added tags {valid_tags} to bookmark {bookmark_id}") + return f"Generated summary and tags for bookmark {bookmark_id}" + return f"Generated summary for bookmark {bookmark_id}" else: error_msg = response.get('error') or "Empty response from AI" @@ -161,7 +209,7 @@ def generate_summary(bookmark_id): ) return f"Failed to generate summary for bookmark {bookmark_id}, created fallback." except Exception as e: - logger.exception(f"Unexpected error in generate_summary for bookmark {bookmark_id}: {e}") + logger.error(f"Unexpected error in generate_summary for bookmark {bookmark_id}: {e}") Summary.objects.update_or_create( bookmark=bookmark, defaults={'content': "An unexpected error occurred while generating the AI summary."} diff --git a/core/templates/base.html b/core/templates/base.html index 4b6a3fd..293d348 100644 --- a/core/templates/base.html +++ b/core/templates/base.html @@ -136,7 +136,10 @@ {{ user.username }}