diff --git a/ai/__pycache__/__init__.cpython-311.pyc b/ai/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000..9beeae7 Binary files /dev/null and b/ai/__pycache__/__init__.cpython-311.pyc differ diff --git a/ai/__pycache__/local_ai_api.cpython-311.pyc b/ai/__pycache__/local_ai_api.cpython-311.pyc new file mode 100644 index 0000000..ae12bda Binary files /dev/null and b/ai/__pycache__/local_ai_api.cpython-311.pyc differ diff --git a/config/__init__.py b/config/__init__.py index e69de29..fb989c4 100644 --- a/config/__init__.py +++ b/config/__init__.py @@ -0,0 +1,3 @@ +from .celery import app as celery_app + +__all__ = ('celery_app',) diff --git a/config/__pycache__/__init__.cpython-311.pyc b/config/__pycache__/__init__.cpython-311.pyc index 423a636..ac17703 100644 Binary files a/config/__pycache__/__init__.cpython-311.pyc and b/config/__pycache__/__init__.cpython-311.pyc differ diff --git a/config/__pycache__/celery.cpython-311.pyc b/config/__pycache__/celery.cpython-311.pyc new file mode 100644 index 0000000..8983b9d Binary files /dev/null and b/config/__pycache__/celery.cpython-311.pyc differ diff --git a/config/__pycache__/settings.cpython-311.pyc b/config/__pycache__/settings.cpython-311.pyc index 96bce55..7eeb2e3 100644 Binary files a/config/__pycache__/settings.cpython-311.pyc and b/config/__pycache__/settings.cpython-311.pyc differ diff --git a/config/__pycache__/urls.cpython-311.pyc b/config/__pycache__/urls.cpython-311.pyc index 0b85e94..5d3b77d 100644 Binary files a/config/__pycache__/urls.cpython-311.pyc and b/config/__pycache__/urls.cpython-311.pyc differ diff --git a/config/celery.py b/config/celery.py new file mode 100644 index 0000000..19539c5 --- /dev/null +++ b/config/celery.py @@ -0,0 +1,20 @@ +import os +from celery import Celery + +# Set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings') + +app = Celery('config') + +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +app.config_from_object('django.conf:settings', namespace='CELERY') + +# Load task modules from all registered Django app configs. +app.autodiscover_tasks() + +@app.task(bind=True, ignore_result=True) +def debug_task(self): + print(f'Request: {self.request!r}') diff --git a/config/settings.py b/config/settings.py index 291d043..9c8820b 100644 --- a/config/settings.py +++ b/config/settings.py @@ -55,6 +55,9 @@ INSTALLED_APPS = [ 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', + 'rest_framework', + 'taggit', + 'django_filters', 'core', ] @@ -180,3 +183,33 @@ if EMAIL_USE_SSL: # https://docs.djangoproject.com/en/5.2/ref/settings/#default-auto-field DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' + +# DRF Settings +REST_FRAMEWORK = { + 'DEFAULT_PERMISSION_CLASSES': [ + 'rest_framework.permissions.IsAuthenticated', + ], + 'DEFAULT_AUTHENTICATION_CLASSES': [ + 'rest_framework.authentication.SessionAuthentication', + 'rest_framework.authentication.BasicAuthentication', + ], + 'DEFAULT_FILTER_BACKENDS': ( + 'django_filters.rest_framework.DjangoFilterBackend', + ), +} + +# Celery Settings +CELERY_BROKER_URL = os.getenv('CELERY_BROKER_URL', 'redis://localhost:6379/0') +CELERY_RESULT_BACKEND = os.getenv('CELERY_RESULT_BACKEND', 'redis://localhost:6379/0') +CELERY_ACCEPT_CONTENT = ['json'] +CELERY_TASK_SERIALIZER = 'json' +CELERY_RESULT_SERIALIZER = 'json' +CELERY_TIMEZONE = TIME_ZONE + +# Run tasks synchronously in development (no Redis required) +CELERY_TASK_ALWAYS_EAGER = True +CELERY_TASK_EAGER_PROPAGATES = True + +# Login/Logout Redirects +LOGIN_REDIRECT_URL = 'home' +LOGOUT_REDIRECT_URL = 'home' \ No newline at end of file diff --git a/config/urls.py b/config/urls.py index bcfc074..4da058d 100644 --- a/config/urls.py +++ b/config/urls.py @@ -1,29 +1,10 @@ -""" -URL configuration for config project. - -The `urlpatterns` list routes URLs to views. For more information please see: - https://docs.djangoproject.com/en/5.2/topics/http/urls/ -Examples: -Function views - 1. Add an import: from my_app import views - 2. Add a URL to urlpatterns: path('', views.home, name='home') -Class-based views - 1. Add an import: from other_app.views import Home - 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') -Including another URLconf - 1. Import the include() function: from django.urls import include, path - 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) -""" from django.contrib import admin -from django.urls import include, path -from django.conf import settings -from django.conf.urls.static import static +from django.urls import path, include +from django.contrib.auth import views as auth_views urlpatterns = [ - path("admin/", admin.site.urls), - path("", include("core.urls")), + path('admin/', admin.site.urls), + path('accounts/login/', auth_views.LoginView.as_view(), name='login'), + path('accounts/logout/', auth_views.LogoutView.as_view(), name='logout'), + path('', include('core.urls')), ] - -if settings.DEBUG: - urlpatterns += static("/assets/", document_root=settings.BASE_DIR / "assets") - urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) diff --git a/core/__pycache__/admin.cpython-311.pyc b/core/__pycache__/admin.cpython-311.pyc index a5ed392..9e20c27 100644 Binary files a/core/__pycache__/admin.cpython-311.pyc and b/core/__pycache__/admin.cpython-311.pyc differ diff --git a/core/__pycache__/api_views.cpython-311.pyc b/core/__pycache__/api_views.cpython-311.pyc new file mode 100644 index 0000000..5e92c52 Binary files /dev/null and b/core/__pycache__/api_views.cpython-311.pyc differ diff --git a/core/__pycache__/models.cpython-311.pyc b/core/__pycache__/models.cpython-311.pyc index e061640..991a04b 100644 Binary files a/core/__pycache__/models.cpython-311.pyc and b/core/__pycache__/models.cpython-311.pyc differ diff --git a/core/__pycache__/serializers.cpython-311.pyc b/core/__pycache__/serializers.cpython-311.pyc new file mode 100644 index 0000000..593f032 Binary files /dev/null and b/core/__pycache__/serializers.cpython-311.pyc differ diff --git a/core/__pycache__/tasks.cpython-311.pyc b/core/__pycache__/tasks.cpython-311.pyc new file mode 100644 index 0000000..7074f20 Binary files /dev/null and b/core/__pycache__/tasks.cpython-311.pyc differ diff --git a/core/__pycache__/urls.cpython-311.pyc b/core/__pycache__/urls.cpython-311.pyc index 5a69659..82edf1b 100644 Binary files a/core/__pycache__/urls.cpython-311.pyc and b/core/__pycache__/urls.cpython-311.pyc differ diff --git a/core/__pycache__/views.cpython-311.pyc b/core/__pycache__/views.cpython-311.pyc index 2a36fd6..61d1f79 100644 Binary files a/core/__pycache__/views.cpython-311.pyc and b/core/__pycache__/views.cpython-311.pyc differ diff --git a/core/admin.py b/core/admin.py index 8c38f3f..0c091ca 100644 --- a/core/admin.py +++ b/core/admin.py @@ -1,3 +1,30 @@ from django.contrib import admin +from .models import Team, TeamMembership, Bookmark, BookmarkShare, Extraction, Summary -# Register your models here. +class TeamMembershipInline(admin.TabularInline): + model = TeamMembership + extra = 1 + +@admin.register(Team) +class TeamAdmin(admin.ModelAdmin): + list_display = ('name', 'created_at', 'updated_at') + inlines = [TeamMembershipInline] + +@admin.register(Bookmark) +class BookmarkAdmin(admin.ModelAdmin): + list_display = ('title', 'url', 'user', 'is_favorite', 'created_at') + list_filter = ('is_favorite', 'created_at', 'user') + search_fields = ('title', 'url', 'notes') + +@admin.register(BookmarkShare) +class BookmarkShareAdmin(admin.ModelAdmin): + list_display = ('bookmark', 'team', 'shared_by', 'shared_at') + list_filter = ('team', 'shared_at') + +@admin.register(Extraction) +class ExtractionAdmin(admin.ModelAdmin): + list_display = ('bookmark', 'extracted_at') + +@admin.register(Summary) +class SummaryAdmin(admin.ModelAdmin): + list_display = ('bookmark', 'generated_at') \ No newline at end of file diff --git a/core/api_views.py b/core/api_views.py new file mode 100644 index 0000000..b98de26 --- /dev/null +++ b/core/api_views.py @@ -0,0 +1,32 @@ +from rest_framework import viewsets, permissions, filters +from django_filters.rest_framework import DjangoFilterBackend +from core.models import Bookmark, Team +from core.serializers import BookmarkSerializer, BookmarkDetailSerializer, TeamSerializer +from core.tasks import process_bookmark + +class BookmarkViewSet(viewsets.ModelViewSet): + permission_classes = [permissions.IsAuthenticated] + filter_backends = [DjangoFilterBackend, filters.SearchFilter, filters.OrderingFilter] + filterset_fields = ['is_favorite'] + search_fields = ['title', 'url', 'notes', 'extraction__content_text'] + ordering_fields = ['created_at', 'updated_at', 'title'] + ordering = ['-created_at'] + + def get_queryset(self): + return Bookmark.objects.filter(user=self.request.user).select_related('extraction') + + def get_serializer_class(self): + if self.action == 'retrieve': + return BookmarkDetailSerializer + return BookmarkSerializer + + def perform_create(self, serializer): + bookmark = serializer.save() + process_bookmark.delay(bookmark.id) + +class TeamViewSet(viewsets.ModelViewSet): + permission_classes = [permissions.IsAuthenticated] + serializer_class = TeamSerializer + + def get_queryset(self): + return self.request.user.teams.all() diff --git a/core/migrations/0001_initial.py b/core/migrations/0001_initial.py new file mode 100644 index 0000000..74c3f4e --- /dev/null +++ b/core/migrations/0001_initial.py @@ -0,0 +1,94 @@ +# Generated by Django 5.2.7 on 2026-02-04 16:55 + +import django.db.models.deletion +import taggit.managers +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('taggit', '0006_rename_taggeditem_content_type_object_id_taggit_tagg_content_8fc721_idx'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='Team', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255)), + ('description', models.TextField(blank=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ], + ), + migrations.CreateModel( + name='Bookmark', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('url', models.URLField(max_length=1000)), + ('title', models.CharField(blank=True, max_length=255)), + ('notes', models.TextField(blank=True)), + ('is_favorite', models.BooleanField(default=False)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('tags', taggit.managers.TaggableManager(help_text='A comma-separated list of tags.', through='taggit.TaggedItem', to='taggit.Tag', verbose_name='Tags')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='bookmarks', to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.CreateModel( + name='Extraction', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('content_html', models.TextField(blank=True)), + ('content_text', models.TextField(blank=True)), + ('metadata', models.JSONField(blank=True, default=dict)), + ('extracted_at', models.DateTimeField(auto_now_add=True)), + ('bookmark', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='extraction', to='core.bookmark')), + ], + ), + migrations.CreateModel( + name='Summary', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('content', models.TextField()), + ('generated_at', models.DateTimeField(auto_now_add=True)), + ('bookmark', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='summary', to='core.bookmark')), + ], + ), + migrations.CreateModel( + name='TeamMembership', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('role', models.CharField(choices=[('OWNER', 'Owner'), ('ADMIN', 'Admin'), ('MEMBER', 'Member')], default='MEMBER', max_length=10)), + ('joined_at', models.DateTimeField(auto_now_add=True)), + ('team', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='core.team')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'unique_together': {('user', 'team')}, + }, + ), + migrations.AddField( + model_name='team', + name='members', + field=models.ManyToManyField(related_name='teams', through='core.TeamMembership', to=settings.AUTH_USER_MODEL), + ), + migrations.CreateModel( + name='BookmarkShare', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('shared_at', models.DateTimeField(auto_now_add=True)), + ('bookmark', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='shares', to='core.bookmark')), + ('shared_by', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ('team', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='shared_bookmarks', to='core.team')), + ], + options={ + 'unique_together': {('bookmark', 'team')}, + }, + ), + ] diff --git a/core/migrations/__pycache__/0001_initial.cpython-311.pyc b/core/migrations/__pycache__/0001_initial.cpython-311.pyc new file mode 100644 index 0000000..a1fca15 Binary files /dev/null and b/core/migrations/__pycache__/0001_initial.cpython-311.pyc differ diff --git a/core/models.py b/core/models.py index 71a8362..901c8e1 100644 --- a/core/models.py +++ b/core/models.py @@ -1,3 +1,68 @@ from django.db import models +from django.contrib.auth.models import User +from taggit.managers import TaggableManager -# Create your models here. +class Team(models.Model): + name = models.CharField(max_length=255) + description = models.TextField(blank=True) + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + members = models.ManyToManyField(User, through='TeamMembership', related_name='teams') + + def __str__(self): + return self.name + +class TeamMembership(models.Model): + ROLE_CHOICES = [ + ('OWNER', 'Owner'), + ('ADMIN', 'Admin'), + ('MEMBER', 'Member'), + ] + user = models.ForeignKey(User, on_delete=models.CASCADE) + team = models.ForeignKey(Team, on_delete=models.CASCADE) + role = models.CharField(max_length=10, choices=ROLE_CHOICES, default='MEMBER') + joined_at = models.DateTimeField(auto_now_add=True) + + class Meta: + unique_together = ('user', 'team') + +class Bookmark(models.Model): + user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='bookmarks') + url = models.URLField(max_length=1000) + title = models.CharField(max_length=255, blank=True) + notes = models.TextField(blank=True) + is_favorite = models.BooleanField(default=False) + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + tags = TaggableManager() + + def __str__(self): + return self.title or self.url + +class BookmarkShare(models.Model): + bookmark = models.ForeignKey(Bookmark, on_delete=models.CASCADE, related_name='shares') + team = models.ForeignKey(Team, on_delete=models.CASCADE, related_name='shared_bookmarks') + shared_by = models.ForeignKey(User, on_delete=models.CASCADE) + shared_at = models.DateTimeField(auto_now_add=True) + + class Meta: + unique_together = ('bookmark', 'team') + +class Extraction(models.Model): + bookmark = models.OneToOneField(Bookmark, on_delete=models.CASCADE, related_name='extraction') + content_html = models.TextField(blank=True) + content_text = models.TextField(blank=True) + metadata = models.JSONField(default=dict, blank=True) + extracted_at = models.DateTimeField(auto_now_add=True) + + def __str__(self): + return f"Extraction for {self.bookmark}" + +class Summary(models.Model): + bookmark = models.OneToOneField(Bookmark, on_delete=models.CASCADE, related_name='summary') + content = models.TextField() + generated_at = models.DateTimeField(auto_now_add=True) + + def __str__(self): + return f"Summary for {self.bookmark}" diff --git a/core/serializers.py b/core/serializers.py new file mode 100644 index 0000000..bd60b84 --- /dev/null +++ b/core/serializers.py @@ -0,0 +1,40 @@ +from rest_framework import serializers +from django.contrib.auth.models import User +from core.models import Bookmark, Team, TeamMembership, BookmarkShare, Extraction, Summary +from taggit.serializers import TagListSerializerField, TaggitSerializer + +class UserSerializer(serializers.ModelSerializer): + class Meta: + model = User + fields = ['id', 'username', 'email'] + +class TeamSerializer(serializers.ModelSerializer): + class Meta: + model = Team + fields = ['id', 'name', 'description', 'created_at'] + +class BookmarkSerializer(TaggitSerializer, serializers.ModelSerializer): + tags = TagListSerializerField(required=False) + + class Meta: + model = Bookmark + fields = ['id', 'url', 'title', 'notes', 'is_favorite', 'tags', 'created_at', 'updated_at'] + read_only_fields = ['id', 'created_at', 'updated_at'] + + def create(self, validated_data): + user = self.context['request'].user + tags = validated_data.pop('tags', []) + bookmark = Bookmark.objects.create(user=user, **validated_data) + bookmark.tags.set(tags) + return bookmark + +class ExtractionSerializer(serializers.ModelSerializer): + class Meta: + model = Extraction + fields = ['content_text', 'extracted_at'] + +class BookmarkDetailSerializer(BookmarkSerializer): + extraction = ExtractionSerializer(read_only=True) + + class Meta(BookmarkSerializer.Meta): + fields = BookmarkSerializer.Meta.fields + ['extraction'] diff --git a/core/tasks.py b/core/tasks.py new file mode 100644 index 0000000..1de05d3 --- /dev/null +++ b/core/tasks.py @@ -0,0 +1,91 @@ +import httpx +from celery import shared_task +from django.utils import timezone +from core.models import Bookmark, Extraction, Summary +from ai.local_ai_api import LocalAIApi +from bs4 import BeautifulSoup +import html2text +import logging + +logger = logging.getLogger(__name__) + +@shared_task(bind=True, max_retries=3) +def process_bookmark(self, bookmark_id): + try: + bookmark = Bookmark.objects.get(id=bookmark_id) + except Bookmark.DoesNotExist: + return + + try: + with httpx.Client(follow_redirects=True, timeout=30.0) as client: + response = client.get(bookmark.url) + response.raise_for_status() + html_content = response.text + except Exception as exc: + logger.error(f"Error fetching bookmark {bookmark_id}: {exc}") + raise self.retry(exc=exc, countdown=60) + + soup = BeautifulSoup(html_content, 'html.parser') + + # Simple title extraction if not already set + if not bookmark.title: + title_tag = soup.find('title') + if title_tag: + bookmark.title = title_tag.string.strip()[:255] + bookmark.save() + + # Readability extraction + h = html2text.HTML2Text() + h.ignore_links = False + h.ignore_images = True + text_content = h.handle(html_content) + + extraction, created = Extraction.objects.update_or_create( + bookmark=bookmark, + defaults={ + 'content_html': html_content, + 'content_text': text_content, + 'metadata': { + 'status_code': response.status_code, + 'content_type': response.headers.get('content-type'), + } + } + ) + + # AI Summary generation + generate_summary.delay(bookmark_id) + + return f"Processed bookmark {bookmark_id}" + +@shared_task +def generate_summary(bookmark_id): + try: + bookmark = Bookmark.objects.get(id=bookmark_id) + extraction = bookmark.extraction + except (Bookmark.DoesNotExist, Extraction.DoesNotExist): + return + + if not extraction.content_text: + return + + # Prepare prompt for AI + prompt = f"Summarize the following content from the webpage '{bookmark.title or bookmark.url}' in 2-3 concise sentences. Focus on the main points for a researcher.\n\nContent:\n{extraction.content_text[:4000]}" + + response = LocalAIApi.create_response({ + "input": [ + {"role": "system", "content": "You are a helpful assistant that summarizes web content for researchers and knowledge workers. Be concise and professional."}, + {"role": "user", "content": prompt}, + ], + }) + + if response.get("success"): + summary_text = LocalAIApi.extract_text(response) + if summary_text: + Summary.objects.update_or_create( + bookmark=bookmark, + defaults={'content': summary_text} + ) + return f"Generated summary for bookmark {bookmark_id}" + + logger.error(f"Failed to generate summary for bookmark {bookmark_id}: {response.get('error')}") + return f"Failed to generate summary for bookmark {bookmark_id}" \ No newline at end of file diff --git a/core/templates/base.html b/core/templates/base.html index 1e7e5fb..4b6a3fd 100644 --- a/core/templates/base.html +++ b/core/templates/base.html @@ -1,25 +1,169 @@ +{% load static %} -
- -