BOARD.txt

@@ -1,26 +0,0 @@
-TODO                                    DONE
---------------------------------------------------------------------------------
-                                        browsing
-                                        links
-                                        redirections
-                                        web links
-                                        history (back/forward)
-                                        simple caching
-                                        simple text files
-                                        encodings
-                                        bookmarks
-                                        view/edit sources
-non shit command-line
-home page
-downloads
-media files
-view history
-identity management
-configuration
---------------------------------------------------------------------------------
-BACKLOG
-margins / centering
-pre blocks folding
-buffers (tabs)
-handle soft-hyphens on wrapping
-bug: combining chars reduce lengths

README.md

@@ -27,16 +27,16 @@ Features
 
 ### What works
 
-Common basic browsing features work: go to URL, scrolling, follow links,
+- Basic browsing: scrolling, follow links, redirections, Web links.
-redirections, page encoding.
 
-Bebop also provide these neat features:
+### What is planned
 
-- History
+- Handle more content types.
-- Caching
+- Great config options.
-- Bookmarks: it's just a text file with bindings.
+- Identity management with temporary and managed certificates.
-
+- Buffers (or tabs if you prefer).
-Check out [this board](BOARD.txt) for what's done and coming next.
+- Home page.
+- Bookmarks.
 
 ### What is not planned for now
 

bebop/__main__.py

@@ -1,13 +1,13 @@
 import argparse
 
-from bebop.browser.browser import Browser
+from bebop.browser import Browser
 from bebop.fs import get_user_data_path
 from bebop.tofu import load_cert_stash, save_cert_stash
 
 
 def main():
     argparser = argparse.ArgumentParser()
-    argparser.add_argument("url", nargs="?", default=None)
+    argparser.add_argument("url", default=None)
     args = argparser.parse_args()
 
     if args.url:
@@ -20,7 +20,7 @@ def main():
         user_data_path.mkdir()
 
     cert_stash_path = user_data_path / "known_hosts.txt"
-    cert_stash = load_cert_stash(cert_stash_path) or {}
+    cert_stash = load_cert_stash(cert_stash_path)
     try:
         Browser(cert_stash).run(start_url=start_url)
     finally:

bebop/bookmarks.py

@@ -1,3 +1,4 @@
+import io
 from pathlib import Path
 
 from bebop.fs import get_user_data_path
@@ -8,7 +9,7 @@ TEMPLATE = """\
 
 Welcome to your bookmark page! This file has been created in "{original_path}" \
 and you can edit it as you wish. New bookmarks will be added on a new \
-line at the end, so always keep an empty line there!
+line at the end. Always keep an empty line at the end!
 """
 
 

bebop/browser.py

@@ -4,27 +4,24 @@ import curses
 import curses.ascii
 import curses.textpad
 import os
-import subprocess
+import webbrowser
-import tempfile
 from math import inf
 
-from bebop.bookmarks import (
+from bebop.bookmarks import get_bookmarks_document, save_bookmark
-    get_bookmarks_path, get_bookmarks_document, save_bookmark
-)
 from bebop.colors import ColorPair, init_colors
 from bebop.command_line import CommandLine
 from bebop.history import History
 from bebop.links import Links
 from bebop.mouse import ButtonState
-from bebop.navigation import (
+from bebop.navigation import *
-    get_parent_url, get_root_url, join_url, parse_url, sanitize_url)
 from bebop.page import Page
 from bebop.page_pad import PagePad
+from bebop.protocol import Request, Response
 
 
 class Browser:
     """Manage the events, inputs and rendering."""
-
+    
     def __init__(self, cert_stash):
         self.stash = cert_stash or {}
         self.screen = None
@@ -135,8 +132,6 @@ class Browser:
             self.open_bookmarks()
         elif char == ord("B"):
             self.add_bookmark()
-        elif char == ord("e"):
-            self.edit_page()
         elif curses.ascii.isdigit(char):
             self.handle_digit_input(char)
         elif char == curses.KEY_MOUSE:
@@ -256,38 +251,133 @@ class Browser:
         if redirects > 5:
             self.set_status_error(f"Too many redirections ({url}).")
             return
-
         if assume_absolute or not self.current_url:
             parts = parse_url(url, absolute=True)
             join = False
         else:
             parts = parse_url(url)
             join = True
-
         if parts.scheme == "gemini":
-            from bebop.browser.gemini import open_gemini_url
             # If there is no netloc, this is a relative URL.
             if join or base_url:
                 url = join_url(base_url or self.current_url, url)
-            open_gemini_url(
+            self.open_gemini_url(sanitize_url(url), redirects=redirects,
-                self,
+                                 history=history, use_cache=use_cache)
-                sanitize_url(url),
-                redirects=redirects,
-                history=history,
-                use_cache=use_cache
-            )
         elif parts.scheme.startswith("http"):
-            from bebop.browser.web import open_web_url
+            self.open_web_url(url)
-            open_web_url(self, url)
         elif parts.scheme == "file":
-            from bebop.browser.file import open_file
+            self.open_file(parts.path, history=history)
-            open_file(self, parts.path, history=history)
         elif parts.scheme == "bebop":
             if parts.netloc == "bookmarks":
                 self.open_bookmarks()
         else:
             self.set_status_error(f"Protocol {parts.scheme} not supported.")
 
+    def open_gemini_url(self, url, redirects=0, history=True, use_cache=True):
+        """Open a Gemini URL and set the formatted response as content.
+
+        After initiating the connection, TODO
+        """
+        self.set_status(f"Loading {url}")
+
+        if use_cache and url in self.cache:
+            self.load_page(self.cache[url])
+            if self.current_url and history:
+                self.history.push(self.current_url)
+            self.current_url = url
+            self.set_status(url)
+            return
+
+        req = Request(url, self.stash)
+        connected = req.connect()
+        if not connected:
+            if req.state == Request.STATE_ERROR_CERT:
+                error = f"Certificate was missing or corrupt ({url})."
+            elif req.state == Request.STATE_UNTRUSTED_CERT:
+                error = f"Certificate has been changed ({url})."
+                # TODO propose the user ways to handle this.
+            elif req.state == Request.STATE_CONNECTION_FAILED:
+                error_details = f": {req.error}" if req.error else "."
+                error = f"Connection failed ({url})" + error_details
+            else:
+                error = f"Connection failed ({url})."
+            self.set_status_error(error)
+            return
+
+        if req.state == Request.STATE_INVALID_CERT:
+            # TODO propose abort / temp trust
+            pass
+        elif req.state == Request.STATE_UNKNOWN_CERT:
+            # TODO propose abort / temp trust / perm trust
+            pass
+        else:
+            pass # TODO
+
+        data = req.proceed()
+        if not data:
+            self.set_status_error(f"Server did not respond in time ({url}).")
+            return
+        response = Response.parse(data)
+        if not response:
+            self.set_status_error(f"Server response parsing failed ({url}).")
+            return
+
+        if response.code == 20:
+            handle_code = self.handle_response_content(response)
+            if handle_code == 0:
+                if self.current_url and history:
+                    self.history.push(self.current_url)
+                self.current_url = url
+                self.cache[url] = self.page_pad.current_page
+                self.set_status(url)
+            elif handle_code == 1:
+                self.set_status(f"Downloaded {url}.")
+        elif response.generic_code == 30 and response.meta:
+            self.open_url(response.meta, base_url=url, redirects=redirects + 1)
+        elif response.generic_code in (40, 50):
+            error = f"Server error: {response.meta or Response.code.name}"
+            self.set_status_error(error)
+        elif response.generic_code == 10:
+            self.handle_input_request(url, response)
+        else:
+            error = f"Unhandled response code {response.code}"
+            self.set_status_error(error)
+
+    def handle_response_content(self, response: Response) -> int:
+        """Handle a response's content from a Gemini server.
+
+        According to the MIME type received or inferred, render or download the
+        response's content.
+
+        Currently only text/gemini content is rendered.
+
+        Arguments:
+        - response: a successful Response.
+
+        Returns:
+        An error code: 0 means a page has been loaded, so any book-keeping such
+        as history management can be applied; 1 means a content has been
+        successfully retrieved but has not been displayed (e.g. non-text
+        content) nor saved as a page; 2 means that the content could not be
+        handled, either due to bogus MIME type or MIME parameters.
+        """
+        mime_type = response.get_mime_type()
+        if mime_type.main_type == "text":
+            if mime_type.sub_type == "gemini":
+                encoding = mime_type.charset
+                try:
+                    text = response.content.decode(encoding, errors="replace")
+                except LookupError:
+                    self.set_status_error("Unknown encoding {encoding}.")
+                    return 2
+                self.load_page(Page.from_gemtext(text))
+                return 0
+            else:
+                pass  # TODO
+        else:
+            pass  # TODO
+        return 1
+
     def load_page(self, page: Page):
         """Load Gemtext data as the current page."""
         old_pad_height = self.page_pad.dim[0]
@@ -301,11 +391,9 @@ class Browser:
 
     def handle_digit_input(self, init_char: int):
         """Focus command-line to select the link ID to follow."""
-        if self.page_pad.current_page is None:
+        if not self.page_pad or self.page_pad.current_page.links is None:
             return
         links = self.page_pad.current_page.links
-        if links is None:
-            return
         err, val = self.command_line.focus_for_link_navigation(init_char, links)
         if err == 0:
             self.open_link(links, val)  # type: ignore
@@ -319,6 +407,17 @@ class Browser:
             return
         self.open_url(links[link_id])
 
+    def handle_input_request(self, from_url: str, response: Response):
+        """Focus command-line to pass input to the server."""
+        if response.meta:
+            self.set_status(f"Input needed: {response.meta}")
+        else:
+            self.set_status("Input needed:")
+        user_input = self.command_line.focus("?")
+        if user_input:
+            url = set_parameter(from_url, user_input)
+            self.open_gemini_url(url)
+
     def handle_mouse(self, mouse_id: int, x: int, y: int, z: int, bstate: int):
         """Handle mouse events.
 
@@ -389,7 +488,11 @@ class Browser:
     def reload_page(self):
         """Reload the page, if one has been previously loaded."""
         if self.current_url:
-            self.open_url(self.current_url, history=False, use_cache=False)
+            self.open_url(
+                self.current_url,
+                history=False,
+                use_cache=False
+            )
 
     def go_back(self):
         """Go back in history if possible."""
@@ -399,12 +502,37 @@ class Browser:
     def go_to_parent_page(self):
         """Go to the parent URL if possible."""
         if self.current_url:
-            self.open_url(get_parent_url(self.current_url))
+            self.open_gemini_url(get_parent_url(self.current_url))
 
     def go_to_root_page(self):
         """Go to the root URL if possible."""
         if self.current_url:
-            self.open_url(get_root_url(self.current_url))
+            self.open_gemini_url(get_root_url(self.current_url))
+
+    def open_web_url(self, url):
+        """Open a Web URL. Currently relies in Python's webbrowser module."""
+        self.set_status(f"Opening {url}")
+        webbrowser.open_new_tab(url)
+
+    def open_file(self, filepath, encoding="utf-8", history=True):
+        """Open a file and render it.
+
+        This should be used only on Gemtext files or at least text files.
+        Anything else will produce garbage and may crash the program. In the
+        future this should be able to use a different parser according to a MIME
+        type or something.
+        """
+        try:
+            with open(filepath, "rt", encoding=encoding) as f:
+                text = f.read()
+        except (OSError, ValueError) as exc:
+            self.set_status_error(f"Failed to open file: {exc}")
+            return
+        self.load_page(Page.from_gemtext(text))
+        file_url = "file://" + filepath
+        if history:
+            self.history.push(file_url)
+        self.current_url = file_url
 
     def open_bookmarks(self):
         """Open bookmarks."""
@@ -419,7 +547,7 @@ class Browser:
         """Add the current URL as bookmark."""
         if not self.current_url:
             return
-        self.set_status("Bookmark title?")
+        self.set_status("Title?")
         current_title = self.page_pad.current_page.title or ""
         title = self.command_line.focus(">", prefix=current_title)
         if title:
@@ -427,42 +555,3 @@ class Browser:
             if title:
                 save_bookmark(self.current_url, title)
         self.reset_status()
-
-    def open_external_program(self, command):
-        """Pauses the curses modes to open an external program."""
-        curses.nocbreak()
-        curses.echo()
-        subprocess.run(command)
-        curses.noecho()
-        curses.cbreak()
-        self.refresh_windows()
-
-    def edit_page(self):
-        """Open a text editor to edit the page source.
-
-        For external pages, the source is written in a temporary file, opened in
-        its editor of choice and so it's up to the user to save it where she
-        needs it, if needed. Internal pages, e.g. the bookmarks page, are loaded
-        directly from their location on disk.
-        """
-        command = ["vi"]
-        delete_source_after = False
-
-        special_pages = {
-            "bebop://bookmarks": str(get_bookmarks_path())
-        }
-        if self.current_url in special_pages:
-            source_filename = special_pages[self.current_url]
-        else:
-            if not self.page_pad.current_page:
-                return
-            source = self.page_pad.current_page.source
-            with tempfile.NamedTemporaryFile("wt", delete=False) as source_file:
-                source_file.write(source)
-                source_filename = source_file.name
-            delete_source_after = True
-
-        command.append(source_filename)
-        self.open_external_program(command)
-        if delete_source_after:
-            os.unlink(source_filename)

bebop/browser/file.py

@@ -1,25 +0,0 @@
-"""Local files browser."""
-
-from bebop.browser.browser import Browser
-from bebop.page import Page
-
-
-def open_file(browser: Browser, filepath: str, encoding="utf-8", history=True):
-    """Open a file and render it.
-
-    This should be used only on Gemtext files or at least text files.
-    Anything else will produce garbage and may crash the program. In the
-    future this should be able to use a different parser according to a MIME
-    type or something.
-    """
-    try:
-        with open(filepath, "rt", encoding=encoding) as f:
-            text = f.read()
-    except (OSError, ValueError) as exc:
-        browser.set_status_error(f"Failed to open file: {exc}")
-        return
-    browser.load_page(Page.from_text(text))
-    file_url = "file://" + filepath
-    if history:
-        browser.history.push(file_url)
-    browser.current_url = file_url

bebop/browser/gemini.py

@@ -1,130 +0,0 @@
-"""Gemini-related features of the browser."""
-
-from bebop.browser.browser import Browser
-from bebop.navigation import set_parameter
-from bebop.page import Page
-from bebop.protocol import Request, Response
-
-
-def open_gemini_url(browser: Browser, url, redirects=0, history=True,
-                    use_cache=True):
-    """Open a Gemini URL and set the formatted response as content.
-
-    After initiating the connection, TODO
-    """
-    browser.set_status(f"Loading {url}")
-
-    if use_cache and url in browser.cache:
-        browser.load_page(browser.cache[url])
-        if browser.current_url and history:
-            browser.history.push(browser.current_url)
-        browser.current_url = url
-        browser.set_status(url)
-        return
-
-    req = Request(url, browser.stash)
-    connected = req.connect()
-    if not connected:
-        if req.state == Request.STATE_ERROR_CERT:
-            error = f"Certificate was missing or corrupt ({url})."
-        elif req.state == Request.STATE_UNTRUSTED_CERT:
-            error = f"Certificate has been changed ({url})."
-            # TODO propose the user ways to handle this.
-        elif req.state == Request.STATE_CONNECTION_FAILED:
-            error_details = f": {req.error}" if req.error else "."
-            error = f"Connection failed ({url})" + error_details
-        else:
-            error = f"Connection failed ({url})."
-        browser.set_status_error(error)
-        return
-
-    if req.state == Request.STATE_INVALID_CERT:
-        # TODO propose abort / temp trust
-        pass
-    elif req.state == Request.STATE_UNKNOWN_CERT:
-        # TODO propose abort / temp trust / perm trust
-        pass
-    else:
-        pass # TODO
-
-    data = req.proceed()
-    if not data:
-        browser.set_status_error(f"Server did not respond in time ({url}).")
-        return
-    response = Response.parse(data)
-    if not response:
-        browser.set_status_error(f"Server response parsing failed ({url}).")
-        return
-
-    if response.code == 20:
-        handle_code = handle_response_content(browser, response)
-        if handle_code == 0:
-            if browser.current_url and history:
-                browser.history.push(browser.current_url)
-            browser.current_url = url
-            browser.cache[url] = browser.page_pad.current_page
-            browser.set_status(url)
-        elif handle_code == 1:
-            browser.set_status(f"Downloaded {url}.")
-    elif response.generic_code == 30 and response.meta:
-        browser.open_url(response.meta, base_url=url, redirects=redirects + 1)
-    elif response.generic_code in (40, 50):
-        error = f"Server error: {response.meta or Response.code.name}"
-        browser.set_status_error(error)
-    elif response.generic_code == 10:
-        handle_input_request(browser, url, response.meta)
-    else:
-        error = f"Unhandled response code {response.code}"
-        browser.set_status_error(error)
-
-
-def handle_response_content(browser: Browser, response: Response) -> int:
-    """Handle a response's content from a Gemini server.
-
-    According to the MIME type received or inferred, render or download the
-    response's content.
-
-    Currently only text content is rendered. For Gemini, the encoding specified
-    in the response is used, if available on the Python distribution. For other
-    text formats, only UTF-8 is attempted.
-
-    Arguments:
-    - response: a successful Response.
-
-    Returns:
-    An error code: 0 means a page has been loaded, so any book-keeping such
-    as history management can be applied; 1 means a content has been
-    successfully retrieved but has not been displayed (e.g. non-text
-    content) nor saved as a page; 2 means that the content could not be
-    handled, either due to bogus MIME type or MIME parameters.
-    """
-    mime_type = response.get_mime_type()
-    if mime_type.main_type == "text":
-        if mime_type.sub_type == "gemini":
-            encoding = mime_type.charset
-            try:
-                text = response.content.decode(encoding, errors="replace")
-            except LookupError:
-                browser.set_status_error("Unknown encoding {encoding}.")
-                return 2
-            browser.load_page(Page.from_gemtext(text))
-            return 0
-        else:
-            text = response.content.decode("utf-8", errors="replace")
-            browser.load_page(Page.from_text(text))
-            return 0
-    else:
-        pass  # TODO
-    return 1
-
-
-def handle_input_request(browser: Browser, from_url: str, message: str =None):
-    """Focus command-line to pass input to the server."""
-    if message:
-        browser.set_status(f"Input needed: {message}")
-    else:
-        browser.set_status("Input needed:")
-    user_input = browser.command_line.focus("?")
-    if user_input:
-        url = set_parameter(from_url, user_input)
-        open_gemini_url(browser, url)

bebop/browser/web.py

@@ -1,11 +0,0 @@
-"""Ha! You thought there would be a Web browser in there?"""
-
-import webbrowser
-
-from bebop.browser.browser import Browser
-
-
-def open_web_url(browser: Browser, url):
-    """Open a Web URL. Currently relies in Python's webbrowser module."""
-    browser.set_status(f"Opening {url}")
-    webbrowser.open_new_tab(url)

bebop/gemtext.py

@@ -6,12 +6,9 @@ module. A renderer can then completely abstract the original document.
 """
 
 import re
-from collections import namedtuple
 from dataclasses import dataclass
 from typing import List
 
-from bebop.links import Links
-
 
 @dataclass
 class Paragraph:
@@ -29,7 +26,6 @@ class Title:
 class Link:
     url: str
     text: str
-    ident: int = 0
     RE = re.compile(r"=>\s*(?P<url>\S+)(\s+(?P<text>.+))?")
 
 
@@ -51,15 +47,9 @@ class ListItem:
     RE = re.compile(r"\*\s(.*)")
 
 
-ParsedGemtext = namedtuple("ParsedGemtext", ("elements", "links", "title"))
+def parse_gemtext(text: str):
-
-
-def parse_gemtext(text: str) -> ParsedGemtext:
     """Parse a string of Gemtext into a list of elements."""
     elements = []
-    links = Links()
-    last_link_id = 0
-    title = ""
     preformatted = None
     for line in text.splitlines():
         line = line.rstrip()
@@ -69,19 +59,14 @@ def parse_gemtext(text: str) -> ParsedGemtext:
         match = Title.RE.match(line)
         if match:
             hashtags, text = match.groups()
-            level = hashtags.count("#")
+            elements.append(Title(hashtags.count("#"), text))
-            elements.append(Title(level, text))
-            if not title and level == 1:
-                title = text
             continue
 
         match = Link.RE.match(line)
         if match:
             match_dict = match.groupdict()
             url, text = match_dict["url"], match_dict.get("text", "")
-            last_link_id += 1
+            elements.append(Link(url, text))
-            links[last_link_id] = url
-            elements.append(Link(url, text, last_link_id))
             continue
 
         if line.startswith(Preformatted.FENCE):
@@ -114,4 +99,4 @@ def parse_gemtext(text: str) -> ParsedGemtext:
     if preformatted:
         elements.append(preformatted)
 
-    return ParsedGemtext(elements, links, title)
+    return elements

bebop/history.py

@@ -1,6 +1,5 @@
 """History management."""
 
-
 class History:
     """Basic browsing history manager."""
 

bebop/links.py

@@ -13,3 +13,11 @@ class Links(dict):
             link_id for link_id, url in self.items()
             if str(link_id).startswith(digits)
         ]
+
+    @staticmethod
+    def from_metalines(metalines: List):
+        links = Links()
+        for meta, _ in metalines:
+            if "link_id" in meta and "url" in meta:
+                links[meta["link_id"]] = meta["url"]
+        return links

bebop/metalines.py

@@ -1,222 +0,0 @@
-"""Metalines generation.
-
-In Bebop we use a list of elements as produced by our parser. These elements are
-converted into so-called "metalines", which are the text lines as they will be
-displayed, along with associated meta-data such as its type or a link's URL.
-"""
-
-import string
-from enum import IntEnum
-from typing import List
-
-from bebop.gemtext import (
-    Blockquote, Link, ListItem, Paragraph, Preformatted, Title)
-
-
-SPLIT_CHARS = " \t-"
-JOIN_CHAR = "-"
-LIST_ITEM_MARK = "• "
-
-
-class LineType(IntEnum):
-    """Type of line.
-
-    Keep lines type along with the content for later rendering.
-    Title type values match the title level to avoid looking it up.
-    """
-    NONE = 0
-    TITLE_1 = 1
-    TITLE_2 = 2
-    TITLE_3 = 3
-    PARAGRAPH = 4
-    LINK = 5
-    PREFORMATTED = 6
-    BLOCKQUOTE = 7
-    LIST_ITEM = 8
-
-
-def generate_metalines(elements, width):
-    """Format elements into a list of lines with metadata.
-
-    The returned list ("metalines") are tuples (meta, line), meta being a
-    dict of metadata and line a text line to display. Currently the only
-    metadata keys used are:
-    - type: one of the Renderer.TYPE constants.
-    - url: only for links, the URL the link on this line refers to. Note
-      that this key is present only for the first line of the link, i.e.
-      long link descriptions wrapped on multiple lines will not have a this
-      key except for the first line.
-    - link_id: only alongside "url" key, ID generated for this link.
-    """
-    metalines = []
-    context = {"width": width}
-    separator = ({"type": LineType.NONE}, "")
-    has_margins = False
-    thin_type = None
-    for index, element in enumerate(elements):
-        previous_had_margins = has_margins
-        last_thin_type = thin_type
-        has_margins = False
-        thin_type = None
-        if isinstance(element, Title):
-            element_metalines = format_title(element, context)
-            has_margins = True
-        elif isinstance(element, Paragraph):
-            element_metalines = format_paragraph(element, context)
-            has_margins = True
-        elif isinstance(element, Link):
-            element_metalines = format_link(element, context)
-            thin_type = LineType.LINK
-        elif isinstance(element, Preformatted):
-            element_metalines = format_preformatted(element, context)
-            has_margins = True
-        elif isinstance(element, Blockquote):
-            element_metalines = format_blockquote(element, context)
-            has_margins = True
-        elif isinstance(element, ListItem):
-            element_metalines = format_list_item(element, context)
-            thin_type = LineType.LIST_ITEM
-        else:
-            continue
-        # If current element requires margins and is not the first elements,
-        # separate from previous element. Also do it if the current element does
-        # not require margins but follows an element that required it (e.g. link
-        # after a paragraph). Also do it if both the current and previous
-        # elements do not require margins but differ in type.
-        if (
-            (has_margins and index > 0)
-            or (not has_margins and previous_had_margins)
-            or (not has_margins and thin_type != last_thin_type)
-        ):
-            metalines.append(separator)
-        # Append the element metalines now.
-        metalines += element_metalines
-    return metalines
-
-
-def generate_dumb_metalines(lines):
-    """Generate dumb metalines: all lines are given the PARAGRAPH line type."""
-    return [({"type": LineType.PARAGRAPH}, line) for line in lines]
-
-
-def format_title(title: Title, context: dict):
-    """Return metalines for this title."""
-    width = context["width"]
-    if title.level == 1:
-        wrapped = wrap_words(title.text, width)
-        line_template = f"{{:^{width}}}"
-        lines = (line_template.format(line) for line in wrapped)
-    else:
-        if title.level == 2:
-            lines = wrap_words(title.text, width, indent=2)
-        else:
-            lines = wrap_words(title.text, width)
-    # Title levels match the type constants of titles.
-    return [({"type": LineType(title.level)}, line) for line in lines]
-
-
-def format_paragraph(paragraph: Paragraph, context: dict):
-    """Return metalines for this paragraph."""
-    lines = wrap_words(paragraph.text, context["width"])
-    return [({"type": LineType.PARAGRAPH}, line) for line in lines]
-
-
-def format_link(link: Link, context: dict):
-    """Return metalines for this link."""
-    # Get a new link and build the "[id]" anchor.
-    link_anchor = f"[{link.ident}] "
-    link_text = link.text or link.url
-    # Wrap lines, indented by the link anchor length.
-    lines = wrap_words(link_text, context["width"], indent=len(link_anchor))
-    first_line_meta = {
-        "type": LineType.LINK,
-        "url": link.url,
-        "link_id": link.ident
-    }
-    # Replace first line indentation with the anchor.
-    first_line_text = link_anchor + lines[0][len(link_anchor):]
-    first_line = [(first_line_meta, first_line_text)]
-    other_lines = [({"type": LineType.LINK}, line) for line in lines[1:]]
-    return first_line + other_lines
-
-
-def format_preformatted(preformatted: Preformatted, context: dict):
-    """Return metalines for this preformatted block."""
-    return [
-        ({"type": LineType.PREFORMATTED}, line)
-        for line in preformatted.lines
-    ]
-
-
-def format_blockquote(blockquote: Blockquote, context: dict):
-    """Return metalines for this blockquote."""
-    lines = wrap_words(blockquote.text, context["width"], indent=2)
-    return [({"type": LineType.BLOCKQUOTE}, line) for line in lines]
-
-
-def format_list_item(item: ListItem, context: dict):
-    """Return metalines for this list item."""
-    indent = len(LIST_ITEM_MARK)
-    lines = wrap_words(item.text, context["width"], indent=indent)
-    first_line = LIST_ITEM_MARK + lines[0][indent:]
-    lines[0] = first_line
-    return [({"type": LineType.LIST_ITEM}, line) for line in lines]
-
-
-def wrap_words(text: str, width: int, indent: int =0) -> List[str]:
-    """Wrap a text in several lines according to the renderer's width."""
-    lines = []
-    line = " " * indent
-    words = _explode_words(text)
-    for word in words:
-        line_len, word_len = len(line), len(word)
-        # If adding the new word would overflow the line, use a new line.
-        if line_len + word_len > width:
-            # Push only non-empty lines.
-            if line_len > 0:
-                lines.append(line)
-                line = " " * indent
-            # Force split words that are longer than the width.
-            while word_len > width:
-                split_offset = width - 1 - indent
-                word_line = " " * indent + word[:split_offset] + JOIN_CHAR
-                lines.append(word_line)
-                word = word[split_offset:]
-                word_len = len(word)
-            word = word.lstrip()
-        line += word
-    if line:
-        lines.append(line)
-    return lines
-
-
-def _explode_words(text: str) -> List[str]:
-    """Split a string into a list of words."""
-    words = []
-    pos = 0
-    while True:
-        sep, sep_index = _find_next_sep(text[pos:])
-        if not sep:
-            words.append(text[pos:])
-            return words
-        word = text[pos : pos + sep_index]
-        # If the separator is not a space char, append it to the word.
-        if sep in string.whitespace:
-            words.append(word)
-            words.append(sep)
-        else:
-            words.append(word + sep)
-        pos += sep_index + 1
-
-
-def _find_next_sep(text: str):
-    """Find the next separator index and return both the separator and index."""
-    indices = []
-    for sep in SPLIT_CHARS:
-        try:
-            indices.append((sep, text.index(sep)))
-        except ValueError:
-            pass
-    if not indices:
-        return ("", 0)
-    return min(indices, key=lambda e: e[1])

bebop/page.py

@@ -1,24 +1,13 @@
 from dataclasses import dataclass, field
 
 from bebop.gemtext import parse_gemtext, Title
-from bebop.metalines import generate_dumb_metalines, generate_metalines
+from bebop.rendering import generate_metalines
 from bebop.links import Links
 
 
 @dataclass
 class Page:
-    """Page-related data.
+    """Page-related data."""
-
-    Attributes:
-    - source: str used to create the page.
-    - metalines: lines ready to be rendered.
-    - links: Links instance, mapping IDs to links on the page; this data is
-      redundant as the links' URLs/IDs are already available in the
-      corresponding metalines, it is meant to be used as a quick map for link ID
-      lookup and disambiguation.
-    - title: optional page title.
-    """
-    source: str
     metalines: list = field(default_factory=list)
     links: Links = field(default_factory=Links)
     title: str = ""
@@ -26,12 +15,13 @@ class Page:
     @staticmethod
     def from_gemtext(gemtext: str):
         """Produce a Page from a Gemtext file or string."""
-        elements, links, title = parse_gemtext(gemtext)
+        elements = parse_gemtext(gemtext)
         metalines = generate_metalines(elements, 80)
-        return Page(gemtext, metalines, links, title)
+        links = Links.from_metalines(metalines)
-
+        # TODO this is horrible; merge parsing with page generation directly
-    @staticmethod
+        title = ""
-    def from_text(text: str):
+        for element in elements:
-        """Produce a Page for a text string."""
+            if isinstance(element, Title) and element.level == 1:
-        metalines = generate_dumb_metalines(text.splitlines())
+                title = element.text
-        return Page(text, metalines)
+                break
+        return Page(metalines, links, title)

bebop/rendering.py

@@ -1,9 +1,223 @@
-"""Rendering Gemtext in curses."""
+"""Rendering Gemtext in curses.
+
+In Bebop we use a list of elements as produced by our parser. These elements are
+rendered into so-called "metalines", which are the text lines as they will be
+displayed, along with associated meta-data such as its type or a link's URL.
+"""
 
 import curses
+import string
+from enum import IntEnum
+from typing import List
 
 from bebop.colors import ColorPair
-from bebop.metalines import LineType
+from bebop.gemtext import (Blockquote, Link, ListItem, Paragraph, Preformatted,
+    Title)
+
+
+SPLIT_CHARS = " \t-"
+JOIN_CHAR = "-"
+LIST_ITEM_MARK = "• "
+
+
+class LineType(IntEnum):
+    """Type of line.
+
+    Keep lines type along with the content for later rendering.
+    Title type values match the title level to avoid looking it up.
+    """
+    NONE = 0
+    TITLE_1 = 1
+    TITLE_2 = 2
+    TITLE_3 = 3
+    PARAGRAPH = 4
+    LINK = 5
+    PREFORMATTED = 6
+    BLOCKQUOTE = 7
+    LIST_ITEM = 8
+
+
+def generate_metalines(elements, width):
+    """Format elements into a list of lines with metadata.
+
+    The returned list ("metalines") are tuples (meta, line), meta being a
+    dict of metadata and line a text line to display. Currently the only
+    metadata keys used are:
+    - type: one of the Renderer.TYPE constants.
+    - url: only for links, the URL the link on this line refers to. Note
+      that this key is present only for the first line of the link, i.e.
+      long link descriptions wrapped on multiple lines will not have a this
+      key except for the first line.
+    - link_id: only alongside "url" key, ID generated for this link.
+    """
+    metalines = []
+    context = {"last_link_id": 0, "width": width}
+    separator = ({"type": LineType.NONE}, "")
+    has_margins = False
+    thin_type = None
+    for index, element in enumerate(elements):
+        previous_had_margins = has_margins
+        last_thin_type = thin_type
+        has_margins = False
+        thin_type = None
+        if isinstance(element, Title):
+            element_metalines = format_title(element, context)
+            has_margins = True
+        elif isinstance(element, Paragraph):
+            element_metalines = format_paragraph(element, context)
+            has_margins = True
+        elif isinstance(element, Link):
+            element_metalines = format_link(element, context)
+            thin_type = LineType.LINK
+        elif isinstance(element, Preformatted):
+            element_metalines = format_preformatted(element, context)
+            has_margins = True
+        elif isinstance(element, Blockquote):
+            element_metalines = format_blockquote(element, context)
+            has_margins = True
+        elif isinstance(element, ListItem):
+            element_metalines = format_list_item(element, context)
+            thin_type = LineType.LIST_ITEM
+        else:
+            continue
+        # If current element requires margins and is not the first elements,
+        # separate from previous element. Also do it if the current element does
+        # not require margins but follows an element that required it (e.g. link
+        # after a paragraph). Also do it if both the current and previous
+        # elements do not require margins but differ in type.
+        if (
+            (has_margins and index > 0)
+            or (not has_margins and previous_had_margins)
+            or (not has_margins and thin_type != last_thin_type)
+        ):
+            metalines.append(separator)
+        # Append the element metalines now.
+        metalines += element_metalines
+    return metalines
+
+
+def format_title(title: Title, context: dict):
+    """Return metalines for this title."""
+    if title.level == 1:
+        wrapped = wrap_words(title.text, context["width"])
+        line_template = f"{{:^{context['width']}}}"
+        lines = (line_template.format(line) for line in wrapped)
+    else:
+        if title.level == 2:
+            lines = wrap_words(title.text, context["width"], indent=2)
+        else:
+            lines = wrap_words(title.text, context["width"])
+    # Title levels match the type constants of titles.
+    return [({"type": LineType(title.level)}, line) for line in lines]
+
+
+def format_paragraph(paragraph: Paragraph, context: dict):
+    """Return metalines for this paragraph."""
+    lines = wrap_words(paragraph.text, context["width"])
+    return [({"type": LineType.PARAGRAPH}, line) for line in lines]
+
+
+def format_link(link: Link, context: dict):
+    """Return metalines for this link."""
+    # Get a new link and build the "[id]" anchor.
+    link_id = context["last_link_id"] + 1
+    context["last_link_id"] = link_id
+    link_text = link.text or link.url
+    link_anchor = f"[{link_id}] "
+    # Wrap lines, indented by the link anchor length.
+    lines = wrap_words(link_text, context["width"], indent=len(link_anchor))
+    first_line_meta = {
+        "type": LineType.LINK,
+        "url": link.url,
+        "link_id": link_id
+    }
+    # Replace first line indentation with the anchor.
+    first_line_text = link_anchor + lines[0][len(link_anchor):]
+    first_line = [(first_line_meta, first_line_text)]
+    other_lines = [({"type": LineType.LINK}, line) for line in lines[1:]]
+    return first_line + other_lines
+
+
+def format_preformatted(preformatted: Preformatted, context: dict):
+    """Return metalines for this preformatted block."""
+    return [
+        ({"type": LineType.PREFORMATTED}, line)
+        for line in preformatted.lines
+    ]
+
+
+def format_blockquote(blockquote: Blockquote, context: dict):
+    """Return metalines for this blockquote."""
+    lines = wrap_words(blockquote.text, context["width"])
+    return [({"type": LineType.BLOCKQUOTE}, line) for line in lines]
+
+
+def format_list_item(item: ListItem, context: dict):
+    """Return metalines for this list item."""
+    indent = len(LIST_ITEM_MARK)
+    lines = wrap_words(item.text, context["width"], indent=indent)
+    first_line = LIST_ITEM_MARK + lines[0][indent:]
+    lines[0] = first_line
+    return [({"type": LineType.LIST_ITEM}, line) for line in lines]
+
+
+def wrap_words(text: str, width: int, indent: int =0) -> List[str]:
+    """Wrap a text in several lines according to the renderer's width."""
+    lines = []
+    line = " " * indent
+    words = _explode_words(text)
+    for word in words:
+        line_len, word_len = len(line), len(word)
+        # If adding the new word would overflow the line, use a new line.
+        if line_len + word_len > width:
+            # Push only non-empty lines.
+            if line_len > 0:
+                lines.append(line)
+                line = " " * indent
+            # Force split words that are longer than the width.
+            while word_len > width:
+                split_offset = width - 1 - indent
+                word_line = " " * indent + word[:split_offset] + JOIN_CHAR
+                lines.append(word_line)
+                word = word[split_offset:]
+                word_len = len(word)
+            word = word.lstrip()
+        line += word
+    if line:
+        lines.append(line)
+    return lines
+
+
+def _explode_words(text: str) -> List[str]:
+    """Split a string into a list of words."""
+    words = []
+    pos = 0
+    while True:
+        sep, sep_index = _find_next_sep(text[pos:])
+        if not sep:
+            words.append(text[pos:])
+            return words
+        word = text[pos : pos + sep_index]
+        # If the separator is not a space char, append it to the word.
+        if sep in string.whitespace:
+            words.append(word)
+            words.append(sep)
+        else:
+            words.append(word + sep)
+        pos += sep_index + 1
+
+
+def _find_next_sep(text: str):
+    """Find the next separator index and return both the separator and index."""
+    indices = []
+    for sep in SPLIT_CHARS:
+        try:
+            indices.append((sep, text.index(sep)))
+        except ValueError:
+            pass
+    if not indices:
+        return ("", 0)
+    return min(indices, key=lambda e: e[1])
 
 
 def render_lines(metalines, window, max_width):