From 779de41b65f4c78225f0a06c031a8ef4aa421455 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sascha=20I=C3=9Fbr=C3=BCcker?= Date: Thu, 31 Dec 2020 07:02:28 +0100 Subject: [PATCH] Implement custom netscape file parser (#51) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Implement custom Netscape file parser (#50) * Add environment variable to configure request timeouts (#50) Co-authored-by: Sascha Ißbrücker --- README.md | 15 +++++++ bookmarks/services/importer.py | 64 +++++++---------------------- bookmarks/services/parser.py | 73 ++++++++++++++++++++++++++++++++++ bookmarks/views/settings.py | 2 +- requirements.prod.txt | 1 + requirements.txt | 1 + uwsgi.ini | 6 +++ 7 files changed, 111 insertions(+), 51 deletions(-) create mode 100644 bookmarks/services/parser.py diff --git a/README.md b/README.md index 891043f..0a6e43f 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,21 @@ For automatic backups you want to backup the applications database. As described The application provides a REST API that can be used by 3rd party applications to manage bookmarks. Check the [API docs](API.md) for further information. +## Troubleshooting + +**Import fails with `502 Bad Gateway`** + +The default timeout for requests is 60 seconds, after which the application server will cancel the request and return the above error. +Depending on the system that the application runs on, and the number of bookmarks that need to be imported, the import may take longer than the default 60 seconds. + +To increase the timeout you can provide a custom timeout to the Docker container using the `LD_REQUEST_TIMEOUT` environment variable: + +``` +docker run --name linkding -p 9090:9090 -e LD_REQUEST_TIMEOUT=180 -d sissbruecker/linkding:latest +``` + +Note that any proxy servers that you are running in front of linkding may have their own timeout settings, which are not affected by the variable. + ## Development The application is open source, so you are free to modify or contribute. The application is built using the Django web framework. You can get started by checking out the excellent Django docs: https://docs.djangoproject.com/en/3.0/. The `bookmarks` folder contains the actual bookmark application, `siteroot` is the Django root application. Other than that the code should be self-explanatory / standard Django stuff 🙂. diff --git a/bookmarks/services/importer.py b/bookmarks/services/importer.py index 6919b23..6dfe949 100644 --- a/bookmarks/services/importer.py +++ b/bookmarks/services/importer.py @@ -2,11 +2,10 @@ import logging from dataclasses import dataclass from datetime import datetime -import bs4 -from bs4 import BeautifulSoup from django.contrib.auth.models import User from bookmarks.models import Bookmark, parse_tag_string +from bookmarks.services.parser import parse, NetscapeBookmark from bookmarks.services.tags import get_or_create_tags logger = logging.getLogger(__name__) @@ -23,52 +22,41 @@ def import_netscape_html(html: str, user: User): result = ImportResult() try: - soup = BeautifulSoup(html, 'html.parser') + netscape_bookmarks = parse(html) except: logging.exception('Could not read bookmarks file.') raise - bookmark_tags = soup.find_all('dt') - - for bookmark_tag in bookmark_tags: + for netscape_bookmark in netscape_bookmarks: result.total = result.total + 1 try: - _import_bookmark_tag(bookmark_tag, user) + _import_bookmark_tag(netscape_bookmark, user) result.success = result.success + 1 except: - shortened_bookmark_tag_str = str(bookmark_tag)[:100] + '...' + shortened_bookmark_tag_str = str(netscape_bookmark)[:100] + '...' logging.exception('Error importing bookmark: ' + shortened_bookmark_tag_str) result.failed = result.failed + 1 return result -def _import_bookmark_tag(bookmark_tag: bs4.Tag, user: User): - link_tag = bookmark_tag.a - - if link_tag is None: - return - +def _import_bookmark_tag(netscape_bookmark: NetscapeBookmark, user: User): # Either modify existing bookmark for the URL or create new one - url = link_tag['href'] - description = _extract_description(bookmark_tag) - bookmark = _get_or_create_bookmark(url, user) + bookmark = _get_or_create_bookmark(netscape_bookmark.href, user) - bookmark.url = url - add_date = link_tag.get('add_date', datetime.now().timestamp()) - bookmark.date_added = datetime.utcfromtimestamp(int(add_date)).astimezone() + bookmark.url = netscape_bookmark.href + bookmark.date_added = datetime.utcfromtimestamp(int(netscape_bookmark.date_added)).astimezone() bookmark.date_modified = bookmark.date_added - bookmark.unread = link_tag.get('toread', '0') == '1' - bookmark.title = link_tag.string - if description: - bookmark.description = description + bookmark.unread = False + bookmark.title = netscape_bookmark.title + if netscape_bookmark.description: + bookmark.description = netscape_bookmark.description bookmark.owner = user bookmark.save() # Set tags - tag_string = link_tag.get('tags', '') - tag_names = parse_tag_string(tag_string) + tag_names = parse_tag_string(netscape_bookmark.tag_string) tags = get_or_create_tags(tag_names, user) bookmark.tags.set(tags) @@ -80,27 +68,3 @@ def _get_or_create_bookmark(url: str, user: User): return Bookmark.objects.get(url=url, owner=user) except Bookmark.DoesNotExist: return Bookmark() - - -def _extract_description(bookmark_tag: bs4.Tag): - """ - Since the Netscape HTML format has no closing tags, all following bookmark tags are part of the description tag - so to extract the description text we have to get creative. For now we combine the text of all text nodes until we - detect a
tag which indicates a new bookmark - :param bookmark_tag: - :return: - """ - description_tag = bookmark_tag.find('dd', recursive=False) - - if description_tag is None: - return None - - description = '' - - for content in description_tag.contents: - if type(content) is bs4.element.Tag and content.name == 'dt': - break - if type(content) is bs4.element.NavigableString: - description += content - - return description.strip() diff --git a/bookmarks/services/parser.py b/bookmarks/services/parser.py new file mode 100644 index 0000000..39079a4 --- /dev/null +++ b/bookmarks/services/parser.py @@ -0,0 +1,73 @@ +from dataclasses import dataclass +from datetime import datetime + +import pyparsing as pp + + +@dataclass +class NetscapeBookmark: + href: str + title: str + description: str + date_added: int + tag_string: str + + +def extract_bookmark_link(tag): + href = tag[0].href + title = tag[0].text + tag_string = tag[0].tags + date_added_string = tag[0].add_date if tag[0].add_date else datetime.now().timestamp() + date_added = int(date_added_string) + + return { + 'href': href, + 'title': title, + 'tag_string': tag_string, + 'date_added': date_added + } + + +def extract_bookmark(tag): + link = tag[0].link + description = tag[0].description + description = description[0] if description else '' + + return { + 'link': link, + 'description': description, + } + + +def extract_description(tag): + return tag[0].strip() + + +# define grammar +dt_start, _ = pp.makeHTMLTags("DT") +dd_start, _ = pp.makeHTMLTags("DD") +a_start, a_end = pp.makeHTMLTags("A") +bookmark_link_tag = pp.Group(a_start + a_start.tag_body("text") + a_end.suppress()) +bookmark_link_tag.addParseAction(extract_bookmark_link) +bookmark_description_tag = dd_start.suppress() + pp.SkipTo(pp.anyOpenTag | pp.anyCloseTag)("description") +bookmark_description_tag.addParseAction(extract_description) +bookmark_tag = pp.Group(dt_start + bookmark_link_tag("link") + pp.ZeroOrMore(bookmark_description_tag)("description")) +bookmark_tag.addParseAction(extract_bookmark) + + +def parse(html: str) -> [NetscapeBookmark]: + matches = bookmark_tag.searchString(html) + bookmarks = [] + + for match in matches: + bookmark_match = match[0] + bookmark = NetscapeBookmark( + href=bookmark_match['link']['href'], + title=bookmark_match['link']['title'], + description=bookmark_match['description'], + tag_string=bookmark_match['link']['tag_string'], + date_added=bookmark_match['link']['date_added'], + ) + bookmarks.append(bookmark) + + return bookmarks diff --git a/bookmarks/views/settings.py b/bookmarks/views/settings.py index a8066ef..b1cb62d 100644 --- a/bookmarks/views/settings.py +++ b/bookmarks/views/settings.py @@ -35,7 +35,7 @@ def bookmark_import(request): return HttpResponseRedirect(reverse('bookmarks:settings.index')) try: - content = import_file.read() + content = import_file.read().decode() result = import_netscape_html(content, request.user) success_msg = str(result.success) + ' bookmarks were successfully imported.' messages.success(request, success_msg, 'bookmark_import_success') diff --git a/requirements.prod.txt b/requirements.prod.txt index 8821e4d..9c9952b 100644 --- a/requirements.prod.txt +++ b/requirements.prod.txt @@ -11,6 +11,7 @@ django-sass-processor==0.7.3 django-widget-tweaks==1.4.5 djangorestframework==3.11.1 idna==2.8 +pyparsing==2.4.7 pytz==2019.1 rcssmin==1.0.6 requests==2.22.0 diff --git a/requirements.txt b/requirements.txt index d3e5d16..95ae925 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,7 @@ django-widget-tweaks==1.4.5 djangorestframework==3.11.1 idna==2.8 libsass==0.19.2 +pyparsing==2.4.7 pytz==2019.1 rcssmin==1.0.6 requests==2.22.0 diff --git a/uwsgi.ini b/uwsgi.ini index 571eee8..3fae04e 100644 --- a/uwsgi.ini +++ b/uwsgi.ini @@ -11,3 +11,9 @@ vacuum=True stats = 127.0.0.1:9191 uid = www-data gid = www-data + +if-env = LD_REQUEST_TIMEOUT +http-timeout = %(_) +socket-timeout = %(_) +harakiri = %(_) +endif = \ No newline at end of file