Bump version

* feat: trim fetched metadata placeholders

* feat: implement trimming serverside

* Add website loader tests

* Address review comments

Co-authored-by: Sascha Ißbrücker <sascha.issbruecker@gmail.com>

.env.sample

@@ -27,3 +27,21 @@ LD_AUTH_PROXY_LOGOUT_URL=
 # List of trusted origins from which to accept POST requests
 # See docs/Options.md for more details
 LD_CSRF_TRUSTED_ORIGINS=
+
+# Database settings
+# These are currently only required for configuring PostreSQL.
+# By default, linkding uses SQLite for which you don't need to configure anything.
+
+# Database engine, can be sqlite (default) or postgres
+LD_DB_ENGINE=
+# Database name (default: linkding)
+LD_DB_DATABASE=
+# Username to connect to the database server  (default: linkding)
+LD_DB_USER=
+# Password to connect to the database server
+LD_DB_PASSWORD=
+# The hostname where the database is hosted (default: localhost)
+LD_DB_HOST=
+# Port use to connect to the database server
+# Should use the default port if not set
+LD_DB_PORT=

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:current-alpine AS node-build
+FROM node:18.13.0-alpine AS node-build
 WORKDIR /etc/linkding
 # install build dependencies
 COPY package.json package-lock.json ./
@@ -10,7 +10,7 @@ RUN npm run build
 
 
 FROM python:3.10.6-slim-buster AS python-base
-RUN apt-get update && apt-get -y install build-essential
+RUN apt-get update && apt-get -y install build-essential libpq-dev
 WORKDIR /etc/linkding
 
 
@@ -34,7 +34,7 @@ RUN mkdir /opt/venv && \
 
 
 FROM python:3.10.6-slim-buster as final
-RUN apt-get update && apt-get -y install mime-support
+RUN apt-get update && apt-get -y install mime-support libpq-dev
 WORKDIR /etc/linkding
 # copy prod dependencies
 COPY --from=prod-deps /opt/venv /opt/venv

README.md

@@ -53,7 +53,11 @@ The name comes from:
 
 ## Installation
 
-linkding is designed to be run with container solutions like [Docker](https://docs.docker.com/get-started/).  The Docker image is compatible with ARM platforms, so it can be run on a Raspberry Pi.
+linkding is designed to be run with container solutions like [Docker](https://docs.docker.com/get-started/).
+The Docker image is compatible with ARM platforms, so it can be run on a Raspberry Pi.
+
+By default, linkding uses SQLite as a database.
+Alternatively linkding supports PostgreSQL, see the [database options](docs/Options.md#LD_DB_ENGINE) for more information.
 
 ###  Using Docker
 
@@ -104,10 +108,25 @@ When using a reverse proxy, such as Nginx or Apache, you may need to configure y
 <details>
 <summary>Apache</summary>
 
-Not tested yet.
-If you figure out a working setup, feel free to contribute it here.
+Apache2 does not change the headers by default, and should not
+need additional configuration.
 
-In the meanwhile, use the [`LD_CSRF_TRUSTED_ORIGINS` option](docs/Options.md#LD_CSRF_TRUSTED_ORIGINS).
+An example virtual host that proxies to linkding might look like:
+```
+<VirtualHost *:9100>
+    <Proxy *>
+        Order deny,allow
+        Allow from all
+    </Proxy>
+
+    ProxyPass / http://linkding:9090/
+    ProxyPassReverse / http://linkding:9090/
+</VirtualHost>
+```
+
+For a full example, see the docker-compose configuration in [jhauris/apache2-reverse-proxy](https://github.com/jhauris/linkding/tree/apache2-reverse-proxy)
+
+If you still run into CSRF issues, please check out the [`LD_CSRF_TRUSTED_ORIGINS` option](docs/Options.md#LD_CSRF_TRUSTED_ORIGINS).
 
 </details>
 

bookmarks/services/website_loader.py

@@ -1,9 +1,12 @@
+import logging
 from dataclasses import dataclass
 
 import requests
 from bs4 import BeautifulSoup
 from charset_normalizer import from_bytes
 
+logger = logging.getLogger(__name__)
+
 
 @dataclass
 class WebsiteMetadata:
@@ -26,22 +29,44 @@ def load_website_metadata(url: str):
         page_text = load_page(url)
         soup = BeautifulSoup(page_text, 'html.parser')
 
-        title = soup.title.string if soup.title is not None else None
+        title = soup.title.string.strip() if soup.title is not None else None
         description_tag = soup.find('meta', attrs={'name': 'description'})
-        description = description_tag['content'] if description_tag is not None else None
+        description = description = description_tag['content'].strip() if description_tag and description_tag['content'] else None
     finally:
         return WebsiteMetadata(url=url, title=title, description=description)
 
 
+CHUNK_SIZE = 50 * 1024
+MAX_CONTENT_LIMIT = 5000 * 1024
+
+
 def load_page(url: str):
     headers = fake_request_headers()
-    r = requests.get(url, timeout=10, headers=headers)
+    size = 0
+    content = None
+    # Use with to ensure request gets closed even if it's only read partially
+    with requests.get(url, timeout=10, headers=headers, stream=True) as r:
+        for chunk in r.iter_content(chunk_size=CHUNK_SIZE):
+            size += len(chunk)
+            if content is None:
+                content = chunk
+            else:
+                content = content + chunk
+
+            # Stop reading if we have parsed end of head tag
+            if '</head>'.encode('utf-8') in content:
+                logger.debug(f'Found closing head tag after {size} bytes')
+                break
+            # Stop reading if we exceed limit
+            if size > MAX_CONTENT_LIMIT:
+                logger.debug(f'Cancel reading document after {size} bytes')
+                break
 
     # Use charset_normalizer to determine encoding that best matches the response content
     # Several sites seem to specify the response encoding incorrectly, so we ignore it and use custom logic instead
     # This is different from Response.text which does respect the encoding specified in the response first,
     # before trying to determine one
-    results = from_bytes(r.content)
+    results = from_bytes(content or '')
     return str(results.best())
 
 

bookmarks/templates/settings/nav.html

@@ -9,6 +9,7 @@
     <li class="tab-item {% if request.get_full_path == integrations_url %}active{% endif %}">
         <a href="{% url 'bookmarks:settings.integrations' %}">Integrations</a>
     </li>
+    {% if request.user.is_superuser %}
     <li class="tab-item tooltip tooltip-bottom" data-tooltip="The admin panel provides additional features &#010; such as user management and bulk operations.">
         <a href="{% url 'admin:index' %}" target="_blank">
             <span>Admin</span>
@@ -18,5 +19,6 @@
             </svg>
         </a>
     </li>
+    {% endif %}
 </ul>
 <br>

bookmarks/tests/test_settings_general_view.py

@@ -59,13 +59,13 @@ class SettingsGeneralViewTestCase(TestCase, BookmarkFactoryMixin):
         ''', html)
 
     def test_get_version_info_just_displays_latest_when_versions_are_equal(self):
-        latest_version_response_mock = Mock(status_code=201, json=lambda: {'name': f'v{app_version}'})
+        latest_version_response_mock = Mock(status_code=200, json=lambda: {'name': f'v{app_version}'})
         with patch.object(requests, 'get', return_value=latest_version_response_mock):
             version_info = get_version_info(random.random())
             self.assertEqual(version_info, f'{app_version} (latest)')
 
     def test_get_version_info_shows_latest_version_when_versions_are_not_equal(self):
-        latest_version_response_mock = Mock(status_code=201, json=lambda: {'name': f'v123.0.1'})
+        latest_version_response_mock = Mock(status_code=200, json=lambda: {'name': f'v123.0.1'})
         with patch.object(requests, 'get', return_value=latest_version_response_mock):
             version_info = get_version_info(random.random())
             self.assertEqual(version_info, f'{app_version} (latest: 123.0.1)')
@@ -74,3 +74,14 @@ class SettingsGeneralViewTestCase(TestCase, BookmarkFactoryMixin):
         with patch.object(requests, 'get', side_effect=RequestException()):
             version_info = get_version_info(random.random())
             self.assertEqual(version_info, f'{app_version}')
+
+    def test_get_version_info_handles_invalid_response(self):
+        latest_version_response_mock = Mock(status_code=403, json=lambda: {})
+        with patch.object(requests, 'get', return_value=latest_version_response_mock):
+            version_info = get_version_info(random.random())
+            self.assertEqual(version_info, app_version)
+
+        latest_version_response_mock = Mock(status_code=200, json=lambda: {})
+        with patch.object(requests, 'get', return_value=latest_version_response_mock):
+            version_info = get_version_info(random.random())
+            self.assertEqual(version_info, app_version)

bookmarks/tests/test_website_loader.py

@@ -0,0 +1,80 @@
+from unittest import mock
+from bookmarks.services import website_loader
+
+from django.test import TestCase
+
+
+class MockStreamingResponse:
+    def __init__(self, num_chunks, chunk_size, insert_head_after_chunk=None):
+        self.chunks = []
+        for index in range(num_chunks):
+            chunk = ''.zfill(chunk_size)
+            self.chunks.append(chunk.encode('utf-8'))
+
+            if index == insert_head_after_chunk:
+                self.chunks.append('</head>'.encode('utf-8'))
+
+    def iter_content(self, **kwargs):
+        return self.chunks
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        pass
+
+
+class WebsiteLoaderTestCase(TestCase):
+    def render_html_document(self, title, description):
+        return f'''
+        <!DOCTYPE html>
+        <html lang="en">
+        <head>
+            <meta charset="UTF-8">
+            <title>{title}</title>
+            <meta name="description" content="{description}">
+        </head>
+        <body></body>
+        </html>
+        '''
+
+    def test_load_page_returns_content(self):
+        with mock.patch('requests.get') as mock_get:
+            mock_get.return_value = MockStreamingResponse(num_chunks=10, chunk_size=1024)
+            content = website_loader.load_page('https://example.com')
+
+            expected_content_size = 10 * 1024
+            self.assertEqual(expected_content_size, len(content))
+
+    def test_load_page_limits_large_documents(self):
+        with mock.patch('requests.get') as mock_get:
+            mock_get.return_value = MockStreamingResponse(num_chunks=10, chunk_size=1024 * 1000)
+            content = website_loader.load_page('https://example.com')
+
+            # Should have read six chunks, after which content exceeds the max of 5MB
+            expected_content_size = 6 * 1024 * 1000
+            self.assertEqual(expected_content_size, len(content))
+
+    def test_load_page_stops_reading_at_closing_head_tag(self):
+        with mock.patch('requests.get') as mock_get:
+            mock_get.return_value = MockStreamingResponse(num_chunks=10, chunk_size=1024 * 1000,
+                                                          insert_head_after_chunk=0)
+            content = website_loader.load_page('https://example.com')
+
+            # Should have read first chunk, and second chunk containing closing head tag
+            expected_content_size = 1 * 1024 * 1000 + len('</head>')
+            self.assertEqual(expected_content_size, len(content))
+
+    def test_load_website_metadata(self):
+        with mock.patch('bookmarks.services.website_loader.load_page') as mock_load_page:
+            mock_load_page.return_value = self.render_html_document('test title', 'test description')
+            metadata = website_loader.load_website_metadata('https://example.com')
+            self.assertEqual('test title', metadata.title)
+            self.assertEqual('test description', metadata.description)
+
+    def test_load_website_metadata_trims_title_and_description(self):
+        with mock.patch('bookmarks.services.website_loader.load_page') as mock_load_page:
+            mock_load_page.return_value = self.render_html_document('  test title  ', '  test description  ')
+            metadata = website_loader.load_website_metadata('https://example.com')
+            self.assertEqual('test title', metadata.title)
+            self.assertEqual('test description', metadata.description)

bookmarks/views/settings.py

@@ -54,7 +54,8 @@ def get_version_info(ttl_hash=None):
         latest_version_url = 'https://api.github.com/repos/sissbruecker/linkding/releases/latest'
         response = requests.get(latest_version_url, timeout=5)
         json = response.json()
-        latest_version = json['name'][1:]
+        if response.status_code == 200 and 'name' in json:
+            latest_version = json['name'][1:]
     except requests.exceptions.RequestException:
         pass
 

bootstrap.sh

@@ -22,4 +22,4 @@ if [ "$LD_DISABLE_BACKGROUND_TASKS" != "True" ]; then
 fi
 
 # Start uwsgi server
-uwsgi --http :$LD_SERVER_PORT uwsgi.ini
+exec uwsgi --http :$LD_SERVER_PORT uwsgi.ini

docs/Options.md

@@ -83,7 +83,7 @@ Enables support for authentication proxies such as Authelia.
 This effectively disables credentials-based authentication and instead authenticates users if a specific request header contains a known username.
 You must make sure that your proxy (nginx, Traefik, Caddy, ...) forwards this header from your auth proxy to linkding. Check the documentation of your auth proxy and your reverse proxy on how to correctly set this up.
 
-Note that this does not automatically create new users, you still need to create users as described in the README, and users need to have the same username as in the auth proxy.
+Note that this automatically creates new users in the database if they do not already exist.
 
 Enabling this setting also requires configuring the following options:
 - `LD_AUTH_PROXY_USERNAME_HEADER` - The name of the request header that the auth proxy passes to the proxied application (linkding in this case), so that the application can identify the user.
@@ -108,3 +108,44 @@ Note that the setting **must** include the correct protocol (`https` or `http`),
 Multiple origins can be specified by separating them with a comma (`,`).
 
 This setting is adopted from the Django framework used by linkding, more information on the setting is available in the [Django documentation](https://docs.djangoproject.com/en/4.0/ref/settings/#std-setting-CSRF_TRUSTED_ORIGINS).
+
+### `LD_DB_ENGINE`
+
+Values: `postgres` or `sqlite` | Default = `sqlite`
+
+Database engine used by linkding to store data.
+Currently, linkding supports SQLite and PostgreSQL.
+By default, linkding uses SQLite, for which you don't need to configure anything.
+All the other database variables below are only required for configured PostgresSQL.
+
+### `LD_DB_DATABASE`
+
+Values: `String` | Default =  `linkding`
+
+The name of the database. 
+
+### `LD_DB_USER`
+
+Values: `String` | Default =  `linkding`
+
+The name of the user to connect to the database server.
+
+### `LD_DB_PASSWORD`
+
+Values: `String` | Default =  None
+
+The password of the user to connect to the database server.
+The password must be configured when using a database other than SQLite, there is no default value.
+
+### `LD_DB_HOST`
+
+Values: `String` | Default =  `localhost`
+
+The hostname or IP of the database server.
+
+### `LD_DB_PORT`
+
+Values: `Integer` | Default =  None
+
+The port of the database server.
+Should use the default port if left empty, for example `5432` for PostgresSQL.

docs/how-to.md

@@ -26,13 +26,13 @@ For more info see here: https://paul.kinlan.me/use-bookmarklets-on-chrome-on-and
 
 - Install HTTP Shortcuts from [Play Store](https://play.google.com/store/apps/details?id=ch.rmy.android.http_shortcuts) or [F-Droid](https://f-droid.org/en/packages/ch.rmy.android.http_shortcuts/).
 
-- Copy the raw URL of [linkding_shortcut.json](/docs/linkding_shortcut.json) in this repository.
+- Copy the URL of [linkding_shortcut.json](https://raw.githubusercontent.com/sissbruecker/linkding/master/docs/linkding_shortcut.json).
 
 - Open HTTP Shortcuts, tap the 3-dot-button at the top-right corner, tap `Import/Export`, then tap `Import from URL`.
 
 - Paste the URL you copied earlier, tap OK, go back, tap the 3-dot-button again, then tap `Variables`.
 
-- Edit the `values` of `linkding_instance` and `linkding_api_token`.
+- Edit the `values` of `linkding_instance` and `linkding_api_key`.
 
 Try using share button on an app, a new item `Send to...` should appear on the share sheet. You can also manually share by tapping the shortcut inside the HTTP Shortcuts app itself.
 

docs/linkding_shortcut.json

@@ -1,60 +1,95 @@
 {
   "categories": [
     {
-      "id": "8f4299d4-4c30-4a8e-a3f9-c90694011713",
+      "id": "e260b423-db01-4743-a671-2cd38594c63c",
+      "layoutType": "wide_grid",
       "name": "Shortcuts",
       "shortcuts": [
         {
-          "bodyContent": "{ \"url\": \"{{b2953f61-b302-4c79-b90d-39858a06d9a6}}\", \"tag_names\": [ \"{{7871474b-e325-4ca0-a142-a5ef5d3f7ed8}}\" ] }",
+          "bodyContent": "{{7b26d228-4ad6-4b1c-8b7b-076dc03385cc}}",
+          "codeOnPrepare": "const sharedValue \u003d getVariable(\u0027text_and_url\u0027)\nconst matches \u003d sharedValue.match(/\\bhttps?:\\/\\/\\S+/gi);\nconst url \u003d matches[0];\nsetVariable(\u0027cleaned_url\u0027, url);",
           "contentType": "application/json",
-          "description": "Bookmark to linkding",
+          "description": "bookmark link",
           "headers": [
             {
-              "id": "fd6306d7-e09d-4c14-a538-3fc258460028",
+              "id": "b66dd9b9-13e8-4802-b527-6e32f3980f4b",
               "key": "Authorization",
-              "value": "Token {{6a739a16-d16d-4a06-93a5-3457da3c3d20}}"
+              "value": "Token {{908e3a30-ae82-400d-93c8-561c36d11d6d}}"
             }
           ],
-          "iconName": "flat_grey_ribbon",
-          "id": "1e047d02-a4a3-4cad-b4cc-123cc16c8398",
-          "launcherShortcut": true,
+          "iconName": "flat_grey_pin",
+          "id": "871c3219-9e9f-46bb-8a7f-78f1496f78fc",
           "method": "POST",
           "name": "Linkding",
-          "quickSettingsTileShortcut": true,
           "responseHandling": {
             "failureOutput": "simple",
             "uiType": "toast"
           },
-          "url": "{{ea2db14b-b9ca-45d8-8555-403271a38f5a}}/api/bookmarks/"
+          "url": "{{26253fe2-d202-4ce8-acd1-55c1ad3ae7d1}}/api/bookmarks/"
         }
       ]
     }
   ],
   "variables": [
     {
-      "id": "ea2db14b-b9ca-45d8-8555-403271a38f5a",
+      "id": "26253fe2-d202-4ce8-acd1-55c1ad3ae7d1",
       "key": "linkding_instance",
-      "value": "https://your.instance.tld.without.slashed.end"
+      "value": "https://your.linkding.host.no.slashed.end"
+    },
+    {
+      "id": "a3c8efa2-3e3a-4bb4-8919-3e831f95fe6a",
+      "jsonEncode": true,
+      "key": "linkding_tag",
+      "message": "Comma separated",
+      "title": "One or more tags",
+      "type": "text"
+    },
+    {
+      "id": "908e3a30-ae82-400d-93c8-561c36d11d6d",
+      "key": "linkding_api_key",
+      "value": "your_api_key_here"
+    },
+    {
+      "id": "d76696e7-1ee1-4d98-b6f9-b570ec69ef40",
+      "key": "cleaned_url"
     },
     {
       "flags": 1,
-      "id": "b2953f61-b302-4c79-b90d-39858a06d9a6",
-      "key": "linkding_add_url",
-      "title": "Enter URL",
+      "id": "da66cdad-8118-4a87-9581-4db33852b610",
+      "key": "text_and_url",
+      "message": "Any text that contains one URL",
+      "title": "URL",
       "type": "text"
     },
     {
-      "id": "6a739a16-d16d-4a06-93a5-3457da3c3d20",
-      "key": "linkding_api_token",
-      "value": "your_token_from_integrations_tab"
+      "data": "{\"select\":{\"multi_select\":\"false\",\"separator\":\",\"}}",
+      "id": "7b26d228-4ad6-4b1c-8b7b-076dc03385cc",
+      "key": "tag_yes_no_default",
+      "options": [
+        {
+          "id": "9365e43e-0572-4621-ac06-caec1ccff09d",
+          "label": "Tagged",
+          "value": "{{5be61e61-d8f5-475b-b1b1-88ddaebf8fd5}}"
+        },
+        {
+          "id": "9f1caeaf-af57-42b4-8b10-4391354ad0f0",
+          "label": "Untagged and unread",
+          "value": "{{71ac9c4d-c03e-4b6f-ad75-9c112a591c50}}"
+        }
+      ],
+      "title": "Tagged or unread?",
+      "type": "select"
     },
     {
-      "id": "7871474b-e325-4ca0-a142-a5ef5d3f7ed8",
-      "key": "linkding_custom_tag",
-      "message": "Enter one or more comma separated tags",
-      "title": "Tag",
-      "type": "text"
+      "id": "5be61e61-d8f5-475b-b1b1-88ddaebf8fd5",
+      "key": "request_body_tagged",
+      "value": "{ \"url\": \"{{d76696e7-1ee1-4d98-b6f9-b570ec69ef40}}\", \"tag_names\": [ \"{{a3c8efa2-3e3a-4bb4-8919-3e831f95fe6a}}\" ] }"
+    },
+    {
+      "id": "71ac9c4d-c03e-4b6f-ad75-9c112a591c50",
+      "key": "request_body_untagged",
+      "value": "{ \"url\": \"{{d76696e7-1ee1-4d98-b6f9-b570ec69ef40}}\", \"unread\": true }"
     }
   ],
-  "version": 53
-}
+  "version": 56
+}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "linkding",
-  "version": "1.15.1",
+  "version": "1.16.0",
   "description": "",
   "main": "index.js",
   "scripts": {

requirements.prod.txt

@@ -12,6 +12,7 @@ django-widget-tweaks==1.4.12
 django4-background-tasks==1.2.7
 djangorestframework==3.13.1
 idna==3.3
+psycopg2==2.9.5
 python-dateutil==2.8.2
 pytz==2022.2.1
 requests==2.28.1

requirements.txt

@@ -17,6 +17,7 @@ django4-background-tasks==1.2.7
 djangorestframework==3.13.1
 idna==3.3
 libsass==0.21.0
+psycopg2-binary==2.9.5
 python-dateutil==2.8.2
 pytz==2022.2.1
 rcssmin==1.1.0

siteroot/settings/base.py

@@ -79,16 +79,6 @@ DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
 
 WSGI_APPLICATION = 'siteroot.wsgi.application'
 
-# Database
-# https://docs.djangoproject.com/en/2.2/ref/settings/#databases
-
-DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': os.path.join(BASE_DIR, 'data', 'db.sqlite3'),
-    }
-}
-
 # Password validation
 # https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators
 
@@ -204,3 +194,31 @@ trusted_origins = os.getenv('LD_CSRF_TRUSTED_ORIGINS', '')
 if trusted_origins:
     CSRF_TRUSTED_ORIGINS = trusted_origins.split(',')
 
+# Database
+# https://docs.djangoproject.com/en/2.2/ref/settings/#databases
+
+LD_DB_ENGINE = os.getenv('LD_DB_ENGINE', 'sqlite')
+LD_DB_HOST = os.getenv('LD_DB_HOST', 'localhost')
+LD_DB_DATABASE = os.getenv('LD_DB_DATABASE', 'linkding')
+LD_DB_USER = os.getenv('LD_DB_USER', 'linkding')
+LD_DB_PASSWORD = os.getenv('LD_DB_PASSWORD', None)
+LD_DB_PORT = os.getenv('LD_DB_PORT', None)
+
+if LD_DB_ENGINE == 'postgres':
+    default_database = {
+        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'NAME': LD_DB_DATABASE,
+        'USER': LD_DB_USER,
+        'PASSWORD': LD_DB_PASSWORD,
+        'HOST': LD_DB_HOST,
+        'PORT': LD_DB_PORT,
+    }
+else:
+    default_database = {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': os.path.join(BASE_DIR, 'data', 'db.sqlite3'),
+    }
+
+DATABASES = {
+    'default': default_database
+}

uwsgi.ini

@@ -11,6 +11,7 @@ stats = 127.0.0.1:9191
 uid = www-data
 gid = www-data
 buffer-size = 8192
+die-on-term = true
 
 if-env = LD_CONTEXT_PATH
 static-map = /%(_)static=static

version.txt

@@ -1 +1 @@
-1.15.1
+1.16.0