Bump version

* generate fallback web archive URL if none exists

* remove fallback web archive snapshot creation

* fix test

CHANGELOG.md

@@ -1,5 +1,25 @@
 # Changelog
 
+## v1.31.0 (16/06/2024)
+
+### What's Changed
+* Add support for bookmark thumbnails by @vslinko in https://github.com/sissbruecker/linkding/pull/721
+* Automatically add tags to bookmarks based on URL pattern by @vslinko in https://github.com/sissbruecker/linkding/pull/736
+* Load bookmark thumbnails after import by @vslinko in https://github.com/sissbruecker/linkding/pull/724
+* Load missing thumbnails after enabling the feature by @sissbruecker in https://github.com/sissbruecker/linkding/pull/725
+* Thumbnails lazy loading by @vslinko in https://github.com/sissbruecker/linkding/pull/734
+* Add option for disabling tag grouping by @vslinko in https://github.com/sissbruecker/linkding/pull/735
+* Preview auto tags in bookmark form by @sissbruecker in https://github.com/sissbruecker/linkding/pull/737
+* Hide tooltip on mobile by @vslinko in https://github.com/sissbruecker/linkding/pull/733
+* Bump requests from 2.31.0 to 2.32.0 by @dependabot in https://github.com/sissbruecker/linkding/pull/740
+
+### New Contributors
+* @vslinko made their first contribution in https://github.com/sissbruecker/linkding/pull/721
+
+**Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.30.0...v1.31.0
+
+---
+
 ## v1.30.0 (20/04/2024)
 
 ### What's Changed

Makefile

@@ -7,7 +7,7 @@ tasks:
 	python manage.py process_tasks
 
 test:
-	pytest
+	pytest -n auto
 
 format:
 	black bookmarks

README.md

@@ -237,6 +237,7 @@ This section lists community projects around using linkding, in alphabetical ord
 - [Linkdy](https://github.com/JGeek00/linkdy): An open source mobile and desktop (not yet) client created with Flutter. Available at the [Google Play Store](https://play.google.com/store/apps/details?id=com.jgeek00.linkdy). By [JGeek00](https://github.com/JGeek00).
 - [LinkThing](https://apps.apple.com/us/app/linkthing/id1666031776) An iOS client for linkding. By [amoscardino](https://github.com/amoscardino)
 - [Open all links bookmarklet](https://gist.github.com/ukcuddlyguy/336dd7339e6d35fc64a75ccfc9323c66) A browser bookmarklet to open all links on the current Linkding page in new tabs. By [ukcuddlyguy](https://github.com/ukcuddlyguy)
+- [Pinkt](https://github.com/fibelatti/pinboard-kotlin) An Android client for linkding. By [fibelatti](https://github.com/fibelatti)
 - [Postman collection](https://gist.github.com/gingerbeardman/f0b42502f3bc9344e92ce63afd4360d3) a group of saved request templates for API testing. By [gingerbeardman](https://github.com/gingerbeardman)
 
 ## Acknowledgements + Donations

assets/logo.svg

@@ -0,0 +1 @@
+<svg clip-rule="evenodd" fill-rule="evenodd" height="512" stroke-linejoin="round" stroke-miterlimit="1.5" viewBox="0 0 512 512" width="512" xmlns="http://www.w3.org/2000/svg"><circle cx="255.0164" cy="254.9236" fill="#5856e0" r="224.78528" stroke-width="1.18"/><g fill="none" stroke="#fff" stroke-width="31.25"><path d="m1244.39 1293.95v199.64s-.81 67.89 74.9 68.88c75.98.99 74.88-68.88 74.88-68.88v-199.64" transform="matrix(.70710678 .70710678 -.70710678 .70710678 284.139117 -1684.198509)"/><path d="m1244.39 1293.95v199.64s-.81 67.89 74.9 68.88c75.98.99 74.88-68.88 74.88-68.88v-199.64" transform="matrix(-.70957074 -.70463421 .70463421 -.70957074 235.113139 2195.434643)"/></g></svg>

bookmarks/api/routes.py

@@ -52,7 +52,7 @@ class BookmarkViewSet(
         return Bookmark.objects.all().filter(owner=user)
 
     def get_serializer_context(self):
-        return {"user": self.request.user}
+        return {"request": self.request, "user": self.request.user}
 
     @action(methods=["get"], detail=False)
     def archived(self, request):
@@ -60,8 +60,8 @@ class BookmarkViewSet(
         search = BookmarkSearch.from_request(request.GET)
         query_set = queries.query_archived_bookmarks(user, user.profile, search)
         page = self.paginate_queryset(query_set)
-        serializer = self.get_serializer_class()
-        data = serializer(page, many=True).data
+        serializer = self.get_serializer(page, many=True)
+        data = serializer.data
         return self.get_paginated_response(data)
 
     @action(methods=["get"], detail=False)
@@ -73,8 +73,8 @@ class BookmarkViewSet(
             user, request.user_profile, search, public_only
         )
         page = self.paginate_queryset(query_set)
-        serializer = self.get_serializer_class()
-        data = serializer(page, many=True).data
+        serializer = self.get_serializer(page, many=True)
+        data = serializer.data
         return self.get_paginated_response(data)
 
     @action(methods=["post"], detail=True)

bookmarks/api/serializers.py

@@ -1,4 +1,5 @@
 from django.db.models import prefetch_related_objects
+from django.templatetags.static import static
 from rest_framework import serializers
 from rest_framework.serializers import ListSerializer
 
@@ -31,6 +32,8 @@ class BookmarkSerializer(serializers.ModelSerializer):
             "website_title",
             "website_description",
             "web_archive_snapshot_url",
+            "favicon_url",
+            "preview_image_url",
             "is_archived",
             "unread",
             "shared",
@@ -42,6 +45,8 @@ class BookmarkSerializer(serializers.ModelSerializer):
             "website_title",
             "website_description",
             "web_archive_snapshot_url",
+            "favicon_url",
+            "preview_image_url",
             "date_added",
             "date_modified",
         ]
@@ -56,6 +61,24 @@ class BookmarkSerializer(serializers.ModelSerializer):
     shared = serializers.BooleanField(required=False, default=False)
     # Override readonly tag_names property to allow passing a list of tag names to create/update
     tag_names = TagListField(required=False, default=[])
+    favicon_url = serializers.SerializerMethodField()
+    preview_image_url = serializers.SerializerMethodField()
+
+    def get_favicon_url(self, obj: Bookmark):
+        if not obj.favicon_file:
+            return None
+        request = self.context.get("request")
+        favicon_file_path = static(obj.favicon_file)
+        favicon_url = request.build_absolute_uri(favicon_file_path)
+        return favicon_url
+
+    def get_preview_image_url(self, obj: Bookmark):
+        if not obj.preview_image_file:
+            return None
+        request = self.context.get("request")
+        preview_image_file_path = static(obj.preview_image_file)
+        preview_image_url = request.build_absolute_uri(preview_image_file_path)
+        return preview_image_url
 
     def create(self, validated_data):
         bookmark = Bookmark()

bookmarks/services/importer.py

@@ -79,8 +79,6 @@ def import_netscape_html(
     for batch in batches:
         _import_batch(batch, user, options, tag_cache, result)
 
-    # Create snapshots for newly imported bookmarks
-    tasks.schedule_bookmarks_without_snapshots(user)
     # Load favicons for newly imported bookmarks
     tasks.schedule_bookmarks_without_favicons(user)
     # Load previews for newly imported bookmarks

bookmarks/services/tasks.py

@@ -12,9 +12,8 @@ from django.utils import timezone, formats
 from huey import crontab
 from huey.contrib.djhuey import HUEY as huey
 from huey.exceptions import TaskLockedException
-from waybackpy.exceptions import WaybackError, TooManyRequestsError, NoCDXRecordFound
+from waybackpy.exceptions import WaybackError, TooManyRequestsError
 
-import bookmarks.services.wayback
 from bookmarks.models import Bookmark, BookmarkAsset, UserProfile
 from bookmarks.services import favicon_loader, singlefile, preview_image_loader
 from bookmarks.services.website_loader import DEFAULT_USER_AGENT
@@ -66,29 +65,6 @@ def create_web_archive_snapshot(user: User, bookmark: Bookmark, force_update: bo
         _create_web_archive_snapshot_task(bookmark.id, force_update)
 
 
-def _load_newest_snapshot(bookmark: Bookmark):
-    try:
-        logger.info(f"Load existing snapshot for bookmark. url={bookmark.url}")
-        cdx_api = bookmarks.services.wayback.CustomWaybackMachineCDXServerAPI(
-            bookmark.url
-        )
-        existing_snapshot = cdx_api.newest()
-
-        if existing_snapshot:
-            bookmark.web_archive_snapshot_url = existing_snapshot.archive_url
-            bookmark.save(update_fields=["web_archive_snapshot_url"])
-            logger.info(
-                f"Using newest snapshot. url={bookmark.url} from={existing_snapshot.datetime_timestamp}"
-            )
-
-    except NoCDXRecordFound:
-        logger.info(f"Could not find any snapshots for bookmark. url={bookmark.url}")
-    except WaybackError as error:
-        logger.error(
-            f"Failed to load existing snapshot. url={bookmark.url}", exc_info=error
-        )
-
-
 def _create_snapshot(bookmark: Bookmark):
     logger.info(f"Create new snapshot for bookmark. url={bookmark.url}...")
     archive = waybackpy.WaybackMachineSaveAPI(
@@ -117,48 +93,27 @@ def _create_web_archive_snapshot_task(bookmark_id: int, force_update: bool):
         return
     except TooManyRequestsError:
         logger.error(
-            f"Failed to create snapshot due to rate limiting, trying to load newest snapshot as fallback. url={bookmark.url}"
+            f"Failed to create snapshot due to rate limiting. url={bookmark.url}"
         )
     except WaybackError as error:
         logger.error(
-            f"Failed to create snapshot, trying to load newest snapshot as fallback. url={bookmark.url}",
+            f"Failed to create snapshot. url={bookmark.url}",
             exc_info=error,
         )
 
-    # Load the newest snapshot as fallback
-    _load_newest_snapshot(bookmark)
-
 
 @task()
 def _load_web_archive_snapshot_task(bookmark_id: int):
-    try:
-        bookmark = Bookmark.objects.get(id=bookmark_id)
-    except Bookmark.DoesNotExist:
-        return
-    # Skip if snapshot exists
-    if bookmark.web_archive_snapshot_url:
-        return
-    # Load the newest snapshot
-    _load_newest_snapshot(bookmark)
-
-
-def schedule_bookmarks_without_snapshots(user: User):
-    if is_web_archive_integration_active(user):
-        _schedule_bookmarks_without_snapshots_task(user.id)
+    # Loading snapshots from CDX API has been removed, keeping the task function
+    # for now to prevent errors when huey tries to run the task
+    pass
 
 
 @task()
 def _schedule_bookmarks_without_snapshots_task(user_id: int):
-    user = get_user_model().objects.get(id=user_id)
-    bookmarks_without_snapshots = Bookmark.objects.filter(
-        web_archive_snapshot_url__exact="", owner=user
-    )
-
-    # TODO: Implement bulk task creation
-    for bookmark in bookmarks_without_snapshots:
-        # To prevent rate limit errors from the Wayback API only try to load the latest snapshots instead of creating
-        # new ones when processing bookmarks in bulk
-        _load_web_archive_snapshot_task(bookmark.id)
+    # Loading snapshots from CDX API has been removed, keeping the task function
+    # for now to prevent errors when huey tries to run the task
+    pass
 
 
 def is_favicon_feature_active(user: User) -> bool:

bookmarks/services/wayback.py

@@ -1,42 +1,20 @@
-import time
-from typing import Dict
+import datetime
 
-import waybackpy
-import waybackpy.utils
-from waybackpy.exceptions import NoCDXRecordFound
+from django.utils import timezone
 
 
-class CustomWaybackMachineCDXServerAPI(waybackpy.WaybackMachineCDXServerAPI):
+def generate_fallback_webarchive_url(
+    url: str, timestamp: datetime.datetime
+) -> str | None:
     """
-    Customized WaybackMachineCDXServerAPI to work around some issues with retrieving the newest snapshot.
-    See https://github.com/akamhy/waybackpy/issues/176
+    Generate a URL to the web archive for the given URL and timestamp.
+    A snapshot for the specific timestamp might not exist, in which case the
+    web archive will show the closest snapshot to the given timestamp.
+    If there is no snapshot at all the URL will be invalid.
     """
+    if not url:
+        return None
+    if not timestamp:
+        timestamp = timezone.now()
 
-    def newest(self):
-        unix_timestamp = int(time.time())
-        self.closest = waybackpy.utils.unix_timestamp_to_wayback_timestamp(
-            unix_timestamp
-        )
-        self.sort = "closest"
-        self.limit = -5
-
-        newest_snapshot = None
-        for snapshot in self.snapshots():
-            newest_snapshot = snapshot
-            break
-
-        if not newest_snapshot:
-            raise NoCDXRecordFound(
-                "Wayback Machine's CDX server did not return any records "
-                + "for the query. The URL may not have any archives "
-                + " on the Wayback Machine or the URL may have been recently "
-                + "archived and is still not available on the CDX server."
-            )
-
-        return newest_snapshot
-
-    def add_payload(self, payload: Dict[str, str]) -> None:
-        super().add_payload(payload)
-        # Set fastLatest query param, as we are only using this API to get the latest snapshot and using fastLatest
-        # makes searching for latest snapshots faster
-        payload["fastLatest"] = "true"
+    return f"https://web.archive.org/web/{timestamp.strftime('%Y%m%d%H%M%S')}/{url}"

bookmarks/signals.py

@@ -1,15 +1,7 @@
 from django.conf import settings
-from django.contrib.auth import user_logged_in
 from django.db.backends.signals import connection_created
 from django.dispatch import receiver
 
-from bookmarks.services import tasks
-
-
-@receiver(user_logged_in)
-def user_logged_in(sender, request, user, **kwargs):
-    tasks.schedule_bookmarks_without_snapshots(user)
-
 
 @receiver(connection_created)
 def extend_sqlite(connection=None, **kwargs):

bookmarks/styles/settings.scss

@@ -7,8 +7,9 @@
     }
   }
 
-  textarea.custom-css {
+  textarea.monospace {
     font-family: monospace;
+    box-sizing: border-box;
   }
 
   .input-group > input[type=submit] {

bookmarks/styles/spectre.scss

@@ -195,3 +195,10 @@ ul.menu li:first-child {
     font-size: 16px;
   }
 }
+
+// Hide tooltips on mobile
+@media (pointer:coarse) {
+  .tooltip::after {
+    display: none;
+  }
+}

bookmarks/templates/bookmarks/details/form.html

@@ -23,8 +23,8 @@
         <span>Reader mode</span>
       </a>
     {% endif %}
-    {% if details.bookmark.web_archive_snapshot_url %}
-      <a class="weblink" href="{{ details.bookmark.web_archive_snapshot_url }}"
+    {% if details.web_archive_snapshot_url %}
+      <a class="weblink" href="{{ details.web_archive_snapshot_url }}"
          target="{{ details.profile.bookmark_link_target }}">
         {% if details.show_link_icons %}
           <svg class="favicon" viewBox="0 0 76 86" xmlns="http://www.w3.org/2000/svg">

bookmarks/templates/settings/general.html

@@ -123,7 +123,7 @@
             <summary>Auto Tagging</summary>
             <label for="{{ form.auto_tagging_rules.id_for_label }}" class="text-assistive">Auto Tagging</label>
             <div class="mt-2">
-              {{ form.auto_tagging_rules|add_class:"form-input custom-css"|attr:"rows:6" }}
+              {{ form.auto_tagging_rules|add_class:"form-input monospace"|attr:"rows:6" }}
             </div>
           </details>
           <div class="form-input-hint">
@@ -225,7 +225,7 @@ reddit.com/r/Music music reddit</pre>
             <summary>Custom CSS</summary>
             <label for="{{ form.custom_css.id_for_label }}" class="text-assistive">Custom CSS</label>
             <div class="mt-2">
-              {{ form.custom_css|add_class:"form-input custom-css"|attr:"rows:6" }}
+              {{ form.custom_css|add_class:"form-input monospace"|attr:"rows:6" }}
             </div>
           </details>
           <div class="form-input-hint">

bookmarks/tests/helpers.py

@@ -87,6 +87,8 @@ class BookmarkFactoryMixin:
         shared: bool = False,
         with_tags: bool = False,
         with_web_archive_snapshot_url: bool = False,
+        with_favicon_file: bool = False,
+        with_preview_image_file: bool = False,
         user: User = None,
     ):
         user = user or self.get_or_create_test_user()
@@ -118,6 +120,12 @@ class BookmarkFactoryMixin:
             web_archive_snapshot_url = ""
             if with_web_archive_snapshot_url:
                 web_archive_snapshot_url = f"https://web.archive.org/web/{i}"
+            favicon_file = ""
+            if with_favicon_file:
+                favicon_file = f"favicon_{i}.png"
+            preview_image_file = ""
+            if with_preview_image_file:
+                preview_image_file = f"preview_image_{i}.png"
             bookmark = self.setup_bookmark(
                 url=url,
                 title=title,
@@ -126,6 +134,8 @@ class BookmarkFactoryMixin:
                 shared=shared,
                 tags=tags,
                 web_archive_snapshot_url=web_archive_snapshot_url,
+                favicon_file=favicon_file,
+                preview_image_file=preview_image_file,
                 user=user,
             )
             bookmarks.append(bookmark)

bookmarks/tests/test_bookmark_details_modal.py

@@ -1,10 +1,11 @@
+import datetime
 import re
 from unittest.mock import patch
 
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.test import TestCase, override_settings
 from django.urls import reverse
-from django.utils import formats
+from django.utils import formats, timezone
 
 from bookmarks.models import BookmarkAsset, UserProfile
 from bookmarks.services import bookmarks, tasks
@@ -180,7 +181,7 @@ class BookmarkDetailsModalTestCase(TestCase, BookmarkFactoryMixin, HtmlTestMixin
         # no latest snapshot
         bookmark = self.setup_bookmark()
         soup = self.get_details(bookmark)
-        self.assertEqual(self.count_weblinks(soup), 1)
+        self.assertEqual(self.count_weblinks(soup), 2)
 
         # snapshot is not complete
         self.setup_asset(
@@ -194,7 +195,7 @@ class BookmarkDetailsModalTestCase(TestCase, BookmarkFactoryMixin, HtmlTestMixin
             status=BookmarkAsset.STATUS_FAILURE,
         )
         soup = self.get_details(bookmark)
-        self.assertEqual(self.count_weblinks(soup), 1)
+        self.assertEqual(self.count_weblinks(soup), 2)
 
         # not a snapshot
         self.setup_asset(
@@ -203,7 +204,7 @@ class BookmarkDetailsModalTestCase(TestCase, BookmarkFactoryMixin, HtmlTestMixin
             status=BookmarkAsset.STATUS_COMPLETE,
         )
         soup = self.get_details(bookmark)
-        self.assertEqual(self.count_weblinks(soup), 1)
+        self.assertEqual(self.count_weblinks(soup), 2)
 
         # snapshot is complete
         asset = self.setup_asset(
@@ -212,20 +213,13 @@ class BookmarkDetailsModalTestCase(TestCase, BookmarkFactoryMixin, HtmlTestMixin
             status=BookmarkAsset.STATUS_COMPLETE,
         )
         soup = self.get_details(bookmark)
-        self.assertEqual(self.count_weblinks(soup), 2)
+        self.assertEqual(self.count_weblinks(soup), 3)
 
         reader_mode_url = reverse("bookmarks:assets.read", args=[asset.id])
         link = self.find_weblink(soup, reader_mode_url)
         self.assertIsNotNone(link)
 
-    def test_internet_archive_link(self):
-        # without snapshot url
-        bookmark = self.setup_bookmark()
-        soup = self.get_details(bookmark)
-        link = self.find_weblink(soup, bookmark.web_archive_snapshot_url)
-        self.assertIsNone(link)
-
-        # with snapshot url
+    def test_internet_archive_link_with_snapshot_url(self):
         bookmark = self.setup_bookmark(web_archive_snapshot_url="https://example.com/")
         soup = self.get_details(bookmark)
         link = self.find_weblink(soup, bookmark.web_archive_snapshot_url)
@@ -264,6 +258,21 @@ class BookmarkDetailsModalTestCase(TestCase, BookmarkFactoryMixin, HtmlTestMixin
         image = link.select_one("svg")
         self.assertIsNotNone(image)
 
+    def test_internet_archive_link_with_fallback_url(self):
+        date_added = timezone.datetime(
+            2023, 8, 11, 21, 45, 11, tzinfo=datetime.timezone.utc
+        )
+        bookmark = self.setup_bookmark(url="https://example.com/", added=date_added)
+        fallback_web_archive_url = (
+            "https://web.archive.org/web/20230811214511/https://example.com/"
+        )
+
+        soup = self.get_details(bookmark)
+        link = self.find_weblink(soup, fallback_web_archive_url)
+        self.assertIsNotNone(link)
+        self.assertEqual(link["href"], fallback_web_archive_url)
+        self.assertEqual(link.text.strip(), "Internet Archive")
+
     def test_weblinks_respect_target_setting(self):
         bookmark = self.setup_bookmark(web_archive_snapshot_url="https://example.com/")
 

bookmarks/tests/test_bookmark_index_view.py

@@ -6,11 +6,7 @@ from django.test import TestCase
 from django.urls import reverse
 
 from bookmarks.models import Bookmark, BookmarkSearch, Tag, UserProfile
-from bookmarks.tests.helpers import (
-    BookmarkFactoryMixin,
-    HtmlTestMixin,
-    collapse_whitespace,
-)
+from bookmarks.tests.helpers import BookmarkFactoryMixin, HtmlTestMixin
 
 
 class BookmarkIndexViewTestCase(TestCase, BookmarkFactoryMixin, HtmlTestMixin):

bookmarks/tests/test_bookmarks_api.py

@@ -36,6 +36,16 @@ class BookmarksApiTestCase(LinkdingApiTestCase, BookmarkFactoryMixin):
             expectation["website_title"] = bookmark.website_title
             expectation["website_description"] = bookmark.website_description
             expectation["web_archive_snapshot_url"] = bookmark.web_archive_snapshot_url
+            expectation["favicon_url"] = (
+                f"http://testserver/static/{bookmark.favicon_file}"
+                if bookmark.favicon_file
+                else None
+            )
+            expectation["preview_image_url"] = (
+                f"http://testserver/static/{bookmark.preview_image_file}"
+                if bookmark.preview_image_file
+                else None
+            )
             expectation["is_archived"] = bookmark.is_archived
             expectation["unread"] = bookmark.unread
             expectation["shared"] = bookmark.shared
@@ -65,7 +75,11 @@ class BookmarksApiTestCase(LinkdingApiTestCase, BookmarkFactoryMixin):
     def test_list_bookmarks_with_more_details(self):
         self.authenticate()
         bookmarks = self.setup_numbered_bookmarks(
-            5, with_tags=True, with_web_archive_snapshot_url=True
+            5,
+            with_tags=True,
+            with_web_archive_snapshot_url=True,
+            with_favicon_file=True,
+            with_preview_image_file=True,
         )
 
         response = self.get(
@@ -171,6 +185,23 @@ class BookmarksApiTestCase(LinkdingApiTestCase, BookmarkFactoryMixin):
         )
         self.assertBookmarkListEqual(response.data["results"], archived_bookmarks)
 
+    def test_list_archived_bookmarks_with_more_details(self):
+        self.authenticate()
+        archived_bookmarks = self.setup_numbered_bookmarks(
+            5,
+            archived=True,
+            with_tags=True,
+            with_web_archive_snapshot_url=True,
+            with_favicon_file=True,
+            with_preview_image_file=True,
+        )
+
+        response = self.get(
+            reverse("bookmarks:bookmark-archived"),
+            expected_status_code=status.HTTP_200_OK,
+        )
+        self.assertBookmarkListEqual(response.data["results"], archived_bookmarks)
+
     def test_list_archived_bookmarks_should_filter_by_query(self):
         self.authenticate()
         search_value = self.get_random_string()
@@ -220,6 +251,26 @@ class BookmarksApiTestCase(LinkdingApiTestCase, BookmarkFactoryMixin):
         )
         self.assertBookmarkListEqual(response.data["results"], shared_bookmarks)
 
+    def test_list_shared_bookmarks_with_more_details(self):
+        self.authenticate()
+
+        other_user = self.setup_user(enable_sharing=True)
+        shared_bookmarks = self.setup_numbered_bookmarks(
+            5,
+            shared=True,
+            user=other_user,
+            with_tags=True,
+            with_web_archive_snapshot_url=True,
+            with_favicon_file=True,
+            with_preview_image_file=True,
+        )
+
+        response = self.get(
+            reverse("bookmarks:bookmark-shared"),
+            expected_status_code=status.HTTP_200_OK,
+        )
+        self.assertBookmarkListEqual(response.data["results"], shared_bookmarks)
+
     def test_list_only_publicly_shared_bookmarks_when_not_logged_in(self):
         user1 = self.setup_user(enable_sharing=True, enable_public_sharing=True)
         user2 = self.setup_user(enable_sharing=True)
@@ -701,6 +752,8 @@ class BookmarksApiTestCase(LinkdingApiTestCase, BookmarkFactoryMixin):
             url="https://example.com",
             title="Example title",
             description="Example description",
+            favicon_file="favicon.png",
+            preview_image_file="preview.png",
         )
 
         url = reverse("bookmarks:bookmark-check")
@@ -715,6 +768,12 @@ class BookmarksApiTestCase(LinkdingApiTestCase, BookmarkFactoryMixin):
         self.assertEqual(bookmark.url, bookmark_data["url"])
         self.assertEqual(bookmark.title, bookmark_data["title"])
         self.assertEqual(bookmark.description, bookmark_data["description"])
+        self.assertEqual(
+            "http://testserver/static/favicon.png", bookmark_data["favicon_url"]
+        )
+        self.assertEqual(
+            "http://testserver/static/preview.png", bookmark_data["preview_image_url"]
+        )
 
     def test_check_returns_existing_metadata_if_url_is_bookmarked(self):
         self.authenticate()

bookmarks/tests/test_bookmarks_list_template.py

@@ -1,3 +1,4 @@
+import datetime
 from typing import Type
 
 from dateutil.relativedelta import relativedelta
@@ -36,15 +37,6 @@ class BookmarkListTemplateTest(TestCase, BookmarkFactoryMixin, HtmlTestMixin):
             html,
         )
 
-    def assertDateLabel(self, html: str, label_content: str):
-        self.assertInHTML(
-            f"""
-        <span>{label_content}</span>
-        <span>|</span>
-        """,
-            html,
-        )
-
     def assertWebArchiveLink(
         self, html: str, label_content: str, url: str, link_target: str = "_blank"
     ):
@@ -465,15 +457,6 @@ class BookmarkListTemplateTest(TestCase, BookmarkFactoryMixin, HtmlTestMixin):
         style = bookmark_list["style"]
         self.assertIn("--ld-bookmark-description-max-lines:3;", style)
 
-    def test_should_respect_absolute_date_setting(self):
-        bookmark = self.setup_date_format_test(
-            UserProfile.BOOKMARK_DATE_DISPLAY_ABSOLUTE
-        )
-        html = self.render_template()
-        formatted_date = formats.date_format(bookmark.date_added, "SHORT_DATE_FORMAT")
-
-        self.assertDateLabel(html, formatted_date)
-
     def test_should_render_web_archive_link_with_absolute_date_setting(self):
         bookmark = self.setup_date_format_test(
             UserProfile.BOOKMARK_DATE_DISPLAY_ABSOLUTE,
@@ -486,12 +469,6 @@ class BookmarkListTemplateTest(TestCase, BookmarkFactoryMixin, HtmlTestMixin):
             html, formatted_date, bookmark.web_archive_snapshot_url
         )
 
-    def test_should_respect_relative_date_setting(self):
-        self.setup_date_format_test(UserProfile.BOOKMARK_DATE_DISPLAY_RELATIVE)
-        html = self.render_template()
-
-        self.assertDateLabel(html, "1 week ago")
-
     def test_should_render_web_archive_link_with_relative_date_setting(self):
         bookmark = self.setup_date_format_test(
             UserProfile.BOOKMARK_DATE_DISPLAY_RELATIVE,
@@ -501,6 +478,27 @@ class BookmarkListTemplateTest(TestCase, BookmarkFactoryMixin, HtmlTestMixin):
 
         self.assertWebArchiveLink(html, "1 week ago", bookmark.web_archive_snapshot_url)
 
+    def test_should_render_generated_web_archive_link_without_saved_snapshot_url(self):
+        user = self.get_or_create_test_user()
+        user.profile.bookmark_date_display = UserProfile.BOOKMARK_DATE_DISPLAY_ABSOLUTE
+        user.profile.save()
+
+        date_added = timezone.datetime(
+            2023, 8, 11, 21, 45, 11, tzinfo=datetime.timezone.utc
+        )
+        bookmark = self.setup_bookmark(
+            url="https://example.com/article", added=date_added
+        )
+
+        html = self.render_template()
+        formatted_date = formats.date_format(bookmark.date_added, "SHORT_DATE_FORMAT")
+
+        self.assertWebArchiveLink(
+            html,
+            formatted_date,
+            "https://web.archive.org/web/20230811214511/https://example.com/article",
+        )
+
     def test_bookmark_link_target_should_be_blank_by_default(self):
         bookmark = self.setup_bookmark()
         html = self.render_template()

bookmarks/tests/test_bookmarks_tasks.py

@@ -1,6 +1,4 @@
-import datetime
 import os.path
-from dataclasses import dataclass
 from unittest import mock
 
 import waybackpy
@@ -10,8 +8,6 @@ from django.test import TestCase, override_settings
 from huey.contrib.djhuey import HUEY as huey
 from waybackpy.exceptions import WaybackError
 
-import bookmarks.services.favicon_loader
-import bookmarks.services.wayback
 from bookmarks.models import BookmarkAsset, UserProfile
 from bookmarks.services import tasks, singlefile
 from bookmarks.tests.helpers import BookmarkFactoryMixin
@@ -29,12 +25,6 @@ def create_wayback_machine_save_api_mock(
     return mock_api
 
 
-@dataclass
-class MockCdxSnapshot:
-    archive_url: str
-    datetime_timestamp: datetime.datetime
-
-
 class BookmarkTasksTestCase(TestCase, BookmarkFactoryMixin):
 
     def setUp(self):
@@ -50,17 +40,6 @@ class BookmarkTasksTestCase(TestCase, BookmarkFactoryMixin):
         )
         self.mock_save_api_patcher.start()
 
-        self.mock_cdx_api = mock.Mock()
-        self.mock_cdx_api.newest.return_value = MockCdxSnapshot(
-            "https://example.com/newest_snapshot", datetime.datetime.now()
-        )
-        self.mock_cdx_api_patcher = mock.patch.object(
-            bookmarks.services.wayback,
-            "CustomWaybackMachineCDXServerAPI",
-            return_value=self.mock_cdx_api,
-        )
-        self.mock_cdx_api_patcher.start()
-
         self.mock_load_favicon_patcher = mock.patch(
             "bookmarks.services.favicon_loader.load_favicon"
         )
@@ -90,7 +69,6 @@ class BookmarkTasksTestCase(TestCase, BookmarkFactoryMixin):
 
     def tearDown(self):
         self.mock_save_api_patcher.stop()
-        self.mock_cdx_api_patcher.stop()
         self.mock_load_favicon_patcher.stop()
         self.mock_singlefile_create_snapshot_patcher.stop()
         self.mock_load_preview_image_patcher.stop()
@@ -142,45 +120,6 @@ class BookmarkTasksTestCase(TestCase, BookmarkFactoryMixin):
 
         self.assertEqual(bookmark.web_archive_snapshot_url, "https://other.com")
 
-    def test_create_web_archive_snapshot_should_use_newest_snapshot_as_fallback(self):
-        bookmark = self.setup_bookmark()
-        self.mock_save_api.save.side_effect = WaybackError
-
-        tasks.create_web_archive_snapshot(
-            self.get_or_create_test_user(), bookmark, False
-        )
-
-        bookmark.refresh_from_db()
-        self.mock_cdx_api.newest.assert_called_once()
-        self.assertEqual(
-            "https://example.com/newest_snapshot",
-            bookmark.web_archive_snapshot_url,
-        )
-
-    def test_create_web_archive_snapshot_should_ignore_missing_newest_snapshot(self):
-        bookmark = self.setup_bookmark()
-        self.mock_save_api.save.side_effect = WaybackError
-        self.mock_cdx_api.newest.return_value = None
-
-        tasks.create_web_archive_snapshot(
-            self.get_or_create_test_user(), bookmark, False
-        )
-
-        bookmark.refresh_from_db()
-        self.assertEqual("", bookmark.web_archive_snapshot_url)
-
-    def test_create_web_archive_snapshot_should_ignore_newest_snapshot_errors(self):
-        bookmark = self.setup_bookmark()
-        self.mock_save_api.save.side_effect = WaybackError
-        self.mock_cdx_api.newest.side_effect = WaybackError
-
-        tasks.create_web_archive_snapshot(
-            self.get_or_create_test_user(), bookmark, False
-        )
-
-        bookmark.refresh_from_db()
-        self.assertEqual("", bookmark.web_archive_snapshot_url)
-
     def test_create_web_archive_snapshot_should_not_save_stale_bookmark_data(self):
         bookmark = self.setup_bookmark()
 
@@ -203,69 +142,6 @@ class BookmarkTasksTestCase(TestCase, BookmarkFactoryMixin):
             bookmark.web_archive_snapshot_url,
         )
 
-    def test_load_web_archive_snapshot_should_update_snapshot_url(self):
-        bookmark = self.setup_bookmark()
-
-        tasks._load_web_archive_snapshot_task(bookmark.id)
-
-        bookmark.refresh_from_db()
-
-        self.assertEqual(self.executed_count(), 1)
-        self.mock_cdx_api.newest.assert_called_once()
-        self.assertEqual(
-            "https://example.com/newest_snapshot", bookmark.web_archive_snapshot_url
-        )
-
-    def test_load_web_archive_snapshot_should_handle_missing_bookmark_id(self):
-        tasks._load_web_archive_snapshot_task(123)
-
-        self.assertEqual(self.executed_count(), 1)
-        self.mock_cdx_api.newest.assert_not_called()
-
-    def test_load_web_archive_snapshot_should_skip_if_snapshot_exists(self):
-        bookmark = self.setup_bookmark(web_archive_snapshot_url="https://example.com")
-
-        tasks._load_web_archive_snapshot_task(bookmark.id)
-
-        self.assertEqual(self.executed_count(), 1)
-        self.mock_cdx_api.newest.assert_not_called()
-
-    def test_load_web_archive_snapshot_should_handle_missing_snapshot(self):
-        bookmark = self.setup_bookmark()
-        self.mock_cdx_api.newest.return_value = None
-
-        tasks._load_web_archive_snapshot_task(bookmark.id)
-
-        self.assertEqual("", bookmark.web_archive_snapshot_url)
-
-    def test_load_web_archive_snapshot_should_handle_wayback_errors(self):
-        bookmark = self.setup_bookmark()
-        self.mock_cdx_api.newest.side_effect = WaybackError
-
-        tasks._load_web_archive_snapshot_task(bookmark.id)
-
-        self.assertEqual("", bookmark.web_archive_snapshot_url)
-
-    def test_load_web_archive_snapshot_should_not_save_stale_bookmark_data(self):
-        bookmark = self.setup_bookmark()
-
-        # update bookmark during API call to check that saving
-        # the snapshot does not overwrite updated bookmark data
-        def mock_newest_impl():
-            bookmark.title = "Updated title"
-            bookmark.save()
-            return mock.DEFAULT
-
-        self.mock_cdx_api.newest.side_effect = mock_newest_impl
-
-        tasks._load_web_archive_snapshot_task(bookmark.id)
-        bookmark.refresh_from_db()
-
-        self.assertEqual("Updated title", bookmark.title)
-        self.assertEqual(
-            "https://example.com/newest_snapshot", bookmark.web_archive_snapshot_url
-        )
-
     @override_settings(LD_DISABLE_BACKGROUND_TASKS=True)
     def test_create_web_archive_snapshot_should_not_run_when_background_tasks_are_disabled(
         self,
@@ -292,59 +168,6 @@ class BookmarkTasksTestCase(TestCase, BookmarkFactoryMixin):
 
         self.assertEqual(self.executed_count(), 0)
 
-    def test_schedule_bookmarks_without_snapshots_should_load_snapshot_for_all_bookmarks_without_snapshot(
-        self,
-    ):
-        user = self.get_or_create_test_user()
-        self.setup_bookmark()
-        self.setup_bookmark()
-        self.setup_bookmark()
-        self.setup_bookmark(web_archive_snapshot_url="https://example.com")
-        self.setup_bookmark(web_archive_snapshot_url="https://example.com")
-        self.setup_bookmark(web_archive_snapshot_url="https://example.com")
-
-        tasks.schedule_bookmarks_without_snapshots(user)
-
-        self.assertEqual(self.executed_count(), 4)
-        self.assertEqual(self.mock_cdx_api.newest.call_count, 3)
-
-    def test_schedule_bookmarks_without_snapshots_should_only_update_user_owned_bookmarks(
-        self,
-    ):
-        user = self.get_or_create_test_user()
-        other_user = User.objects.create_user(
-            "otheruser", "otheruser@example.com", "password123"
-        )
-        self.setup_bookmark()
-        self.setup_bookmark()
-        self.setup_bookmark()
-        self.setup_bookmark(user=other_user)
-        self.setup_bookmark(user=other_user)
-        self.setup_bookmark(user=other_user)
-
-        tasks.schedule_bookmarks_without_snapshots(user)
-
-        self.assertEqual(self.mock_cdx_api.newest.call_count, 3)
-
-    @override_settings(LD_DISABLE_BACKGROUND_TASKS=True)
-    def test_schedule_bookmarks_without_snapshots_should_not_run_when_background_tasks_are_disabled(
-        self,
-    ):
-        tasks.schedule_bookmarks_without_snapshots(self.user)
-
-        self.assertEqual(self.executed_count(), 0)
-
-    def test_schedule_bookmarks_without_snapshots_should_not_run_when_web_archive_integration_is_disabled(
-        self,
-    ):
-        self.user.profile.web_archive_integration = (
-            UserProfile.WEB_ARCHIVE_INTEGRATION_DISABLED
-        )
-        self.user.profile.save()
-        tasks.schedule_bookmarks_without_snapshots(self.user)
-
-        self.assertEqual(self.executed_count(), 0)
-
     def test_load_favicon_should_create_favicon_file(self):
         bookmark = self.setup_bookmark()
 

bookmarks/tests/test_importer.py

@@ -444,17 +444,6 @@ class ImporterTestCase(TestCase, BookmarkFactoryMixin, ImportTestMixin):
         self.assertEqual(Bookmark.objects.all()[0].description, "Example description")
         self.assertEqual(Bookmark.objects.all()[0].notes, "Updated notes")
 
-    def test_schedule_snapshot_creation(self):
-        user = self.get_or_create_test_user()
-        test_html = self.render_html(tags_html="")
-
-        with patch.object(
-            tasks, "schedule_bookmarks_without_snapshots"
-        ) as mock_schedule_bookmarks_without_snapshots:
-            import_netscape_html(test_html, user)
-
-            mock_schedule_bookmarks_without_snapshots.assert_called_once_with(user)
-
     def test_schedule_favicon_loading(self):
         user = self.get_or_create_test_user()
         test_html = self.render_html(tags_html="")

bookmarks/tests/test_signals.py

@@ -1,17 +0,0 @@
-from unittest.mock import patch
-
-from django.test import TestCase
-
-from bookmarks.services import tasks
-from bookmarks.tests.helpers import BookmarkFactoryMixin
-
-
-class SignalsTestCase(TestCase, BookmarkFactoryMixin):
-    def test_login_should_schedule_snapshot_creation(self):
-        user = self.get_or_create_test_user()
-
-        with patch.object(
-            tasks, "schedule_bookmarks_without_snapshots"
-        ) as mock_schedule_bookmarks_without_snapshots:
-            self.client.force_login(user)
-            mock_schedule_bookmarks_without_snapshots.assert_called_once_with(user)

bookmarks/views/partials/contexts.py

@@ -1,12 +1,12 @@
+import re
 import urllib.parse
 from typing import Set, List
-import re
 
+from django.conf import settings
 from django.core.handlers.wsgi import WSGIRequest
 from django.core.paginator import Paginator
 from django.db import models
 from django.urls import reverse
-from django.conf import settings
 
 from bookmarks import queries
 from bookmarks import utils
@@ -18,6 +18,7 @@ from bookmarks.models import (
     UserProfile,
     Tag,
 )
+from bookmarks.services.wayback import generate_fallback_webarchive_url
 
 DEFAULT_PAGE_SIZE = 30
 CJK_RE = re.compile(r"[\u4e00-\u9fff]+")
@@ -144,6 +145,10 @@ class BookmarkItem:
         self.notes = bookmark.notes
         self.tag_names = bookmark.tag_names
         self.web_archive_snapshot_url = bookmark.web_archive_snapshot_url
+        if not self.web_archive_snapshot_url:
+            self.web_archive_snapshot_url = generate_fallback_webarchive_url(
+                bookmark.url, bookmark.date_added
+            )
         self.favicon_file = bookmark.favicon_file
         self.preview_image_file = bookmark.preview_image_file
         self.is_archived = bookmark.is_archived
@@ -412,6 +417,12 @@ class BookmarkDetailsContext:
         # For now hide files section if snapshots are not supported
         self.show_files = settings.LD_ENABLE_SNAPSHOTS
 
+        self.web_archive_snapshot_url = bookmark.web_archive_snapshot_url
+        if not self.web_archive_snapshot_url:
+            self.web_archive_snapshot_url = generate_fallback_webarchive_url(
+                bookmark.url, bookmark.date_added
+            )
+
         self.assets = [
             BookmarkAssetItem(asset) for asset in bookmark.bookmarkasset_set.all()
         ]

docker-compose.yml

@@ -1,5 +1,3 @@
-version: '3'
-
 services:
   linkding:
     container_name: "${LD_CONTAINER_NAME:-linkding}"
@@ -10,4 +8,4 @@ services:
       - "${LD_HOST_DATA_DIR:-./data}:/etc/linkding/data"
     env_file:
       - .env
-    restart: unless-stopped
+    restart: unless-stopped

docker/alpine.Dockerfile

@@ -69,6 +69,7 @@ RUN wget https://www.sqlite.org/${SQLITE_RELEASE_YEAR}/sqlite-amalgamation-${SQL
 
 
 FROM python:3.11.8-alpine3.19 AS linkding
+LABEL org.opencontainers.image.source="https://github.com/sissbruecker/linkding"
 # install runtime dependencies
 RUN apk update && apk add bash curl icu libpq mailcap libssl3
 # create www-data user and group

docker/default.Dockerfile

@@ -71,6 +71,7 @@ RUN wget https://www.sqlite.org/${SQLITE_RELEASE_YEAR}/sqlite-amalgamation-${SQL
 
 
 FROM python:3.11.8-slim-bookworm as linkding
+LABEL org.opencontainers.image.source="https://github.com/sissbruecker/linkding"
 RUN apt-get update && apt-get -y install mime-support libpq-dev libicu-dev libssl3 curl
 WORKDIR /etc/linkding
 # copy prod dependencies

docs/API.md

@@ -49,6 +49,8 @@ Example response:
       "website_title": "Website title",
       "website_description": "Website description",
       "web_archive_snapshot_url": "https://web.archive.org/web/20200926094623/https://example.com",
+      "favicon_url": "http://127.0.0.1:8000/static/https_example_com.png",
+      "preview_image_url": "http://127.0.0.1:8000/static/0ac5c53db923727765216a3a58e70522.jpg",
       "is_archived": false,
       "unread": false,
       "shared": false,

package.json

@@ -1,6 +1,6 @@
 {
   "name": "linkding",
-  "version": "1.31.0",
+  "version": "1.31.1",
   "description": "",
   "main": "index.js",
   "scripts": {

requirements.dev.in

@@ -6,3 +6,4 @@ libsass
 playwright
 pytest
 pytest-django
+pytest-xdist

requirements.dev.txt

@@ -12,7 +12,7 @@ click==8.1.7
     # via black
 coverage==7.4.1
     # via -r requirements.dev.in
-django==5.0.3
+django==5.0.8
     # via
     #   django-appconf
     #   django-debug-toolbar
@@ -22,6 +22,8 @@ django-compressor==4.4
     # via -r requirements.dev.in
 django-debug-toolbar==4.2.0
     # via -r requirements.dev.in
+execnet==2.1.1
+    # via pytest-xdist
 greenlet==3.0.3
     # via playwright
 iniconfig==2.0.0
@@ -48,8 +50,11 @@ pytest==8.0.0
     # via
     #   -r requirements.dev.in
     #   pytest-django
+    #   pytest-xdist
 pytest-django==4.7.0
     # via -r requirements.dev.in
+pytest-xdist==3.6.1
+    # via -r requirements.dev.in
 rcssmin==1.1.1
     # via django-compressor
 rjsmin==1.2.1

requirements.txt

@@ -12,7 +12,7 @@ bleach==6.1.0
     # via -r requirements.in
 bleach-allowlist==1.0.3
     # via -r requirements.in
-certifi==2023.11.17
+certifi==2024.7.4
     # via requests
 cffi==1.16.0
     # via cryptography
@@ -27,7 +27,7 @@ cryptography==42.0.5
     #   josepy
     #   mozilla-django-oidc
     #   pyopenssl
-django==5.0.3
+django==5.0.8
     # via
     #   -r requirements.in
     #   django-registration
@@ -39,7 +39,7 @@ django-sass-processor==1.4
     # via -r requirements.in
 django-widget-tweaks==1.5.0
     # via -r requirements.in
-djangorestframework==3.14.0
+djangorestframework==3.15.2
     # via -r requirements.in
 huey==2.5.0
     # via -r requirements.in
@@ -59,8 +59,6 @@ pyopenssl==24.1.0
     # via josepy
 python-dateutil==2.8.2
     # via -r requirements.in
-pytz==2023.3.post1
-    # via djangorestframework
 requests==2.32.0
     # via
     #   -r requirements.in
@@ -76,7 +74,7 @@ sqlparse==0.5.0
     # via django
 supervisor==4.2.5
     # via -r requirements.in
-urllib3==2.1.0
+urllib3==2.2.2
     # via
     #   requests
     #   waybackpy

siteroot/settings/base.py

@@ -114,7 +114,7 @@ LOGOUT_REDIRECT_URL = "/" + LD_CONTEXT_PATH + "login"
 
 LANGUAGE_CODE = "en-us"
 
-TIME_ZONE = "UTC"
+TIME_ZONE = os.getenv("TZ", "UTC")
 
 USE_I18N = True
 

version.txt

@@ -1 +1 @@
-1.31.0
+1.31.1