Merge pull request #413 from kdmukai/livepreview_speedup

[Enhancement] Live preview speedup

src/seedsigner/gui/renderer.py

@@ -34,7 +34,12 @@ class Renderer(ConfigurableSingleton):
         renderer.draw = ImageDraw.Draw(renderer.canvas)
 
 
-    def show_image(self, image=None, alpha_overlay=None):
+    def show_image(self, image=None, alpha_overlay=None, show_direct=False):
+        if show_direct:
+            # Use the incoming image as the canvas and immediately render
+            self.disp.ShowImage(image, 0, 0)
+            return
+
         if alpha_overlay:
             if image == None:
                 image = self.canvas

src/seedsigner/gui/screens/scan_screens.py

@@ -1,7 +1,7 @@
 import time
 
 from dataclasses import dataclass
-from typing import List, Tuple
+from PIL import Image, ImageDraw
 
 from seedsigner.gui import renderer
 from seedsigner.hardware.buttons import HardwareButtonsConstants
@@ -9,19 +9,43 @@ from seedsigner.hardware.camera import Camera
 from seedsigner.models import DecodeQR, DecodeQRStatus
 from seedsigner.models.threads import BaseThread
 
-from .screen import BaseScreen, BaseTopNavScreen, ButtonListScreen
-from ..components import BaseComponent, Button, GUIConstants, Fonts, IconButton, TextArea, calc_text_centering
+from .screen import BaseScreen, ButtonListScreen
+from ..components import GUIConstants, Fonts, TextArea
 
 
 
 
 @dataclass
 class ScanScreen(BaseScreen):
+    """
+    Live preview has to balance three competing threads:
+    * Camera capturing frames and making them available to read.
+    * Decoder analyzing frames for QR codes.
+    * Live preview display writing frames to the screen.
+
+    All of this would ideally be rewritten as in C/C++/Rust with python bindings for
+    vastly improved performance.
+
+    Until then, we have to balance the resources the Pi Zero has to work with. Thus, we
+    set a modest fps target for the camera: 5fps. At this pace, the decoder and the live
+    display can more or less keep up with the flow of frames without much wasted effort
+    in any of the threads.
+
+    Note: performance tuning was targeted for the Pi Zero.
+
+    The resolution (480x480) has not been tweaked in order to guarantee that our
+    decoding abilities remain as-is. It's possible that more optimizations could be made
+    here (e.g. higher res w/no performance impact? Lower res w/same decoding but faster
+    performance? etc).
+
+    Note: This is quite a lot of important tasks for a Screen to be managing; much of
+    this should probably be refactored into the Controller.
+    """
     decoder: DecodeQR = None
     instructions_text: str = "< back  |  Scan a QR code"
-    resolution: Tuple[int,int] = (480, 480)
-    framerate: int = 12
-    render_rect: Tuple[int,int,int,int] = None
+    resolution: tuple[int,int] = (480, 480)
+    framerate: int = 5  # TODO: alternate optimization for Pi Zero 2W?
+    render_rect: tuple[int,int,int,int] = None
 
 
     def __post_init__(self):
@@ -42,7 +66,7 @@ class ScanScreen(BaseScreen):
 
 
     class LivePreviewThread(BaseThread):
-        def __init__(self, camera: Camera, decoder: DecodeQR, renderer: renderer.Renderer, instructions_text: str, render_rect: Tuple[int,int,int,int]):
+        def __init__(self, camera: Camera, decoder: DecodeQR, renderer: renderer.Renderer, instructions_text: str, render_rect: tuple[int,int,int,int]):
             self.camera = camera
             self.decoder = decoder
             self.renderer = renderer
@@ -53,9 +77,7 @@ class ScanScreen(BaseScreen):
                 self.render_rect = (0, 0, self.renderer.canvas_width, self.renderer.canvas_height)
             self.render_width = self.render_rect[2] - self.render_rect[0]
             self.render_height = self.render_rect[3] - self.render_rect[1]
-
-            print(f"render_width: {self.render_width}")
-            print(f"render_height: {self.render_height}")
+            self.decoder_fps = "0.0"
 
             super().__init__()
 
@@ -64,44 +86,63 @@ class ScanScreen(BaseScreen):
             from timeit import default_timer as timer
 
             instructions_font = Fonts.get_font(GUIConstants.BODY_FONT_NAME, GUIConstants.BUTTON_FONT_SIZE)
+
+            start_time = time.time()
+            num_frames = 0
+            show_framerate = False  # enable for debugging / testing
             while self.keep_running:
-                start = timer()
                 frame = self.camera.read_video_stream(as_image=True)
                 if frame is not None:
-                    scan_text = self.instructions_text
+                    num_frames += 1
+                    cur_time = time.time()
+                    cur_fps = num_frames / (cur_time - start_time)
                     if self.decoder and self.decoder.get_percent_complete() > 0 and self.decoder.is_psbt:
                         scan_text = str(self.decoder.get_percent_complete()) + "% Complete"
+                        if show_framerate:
+                            scan_text += f" {cur_fps:0.2f} | {self.decoder_fps}"
+                    else:
+                        if show_framerate:
+                            scan_text = f"{cur_fps:0.2f} | {self.decoder_fps}"
+                        else:
+                            scan_text = self.instructions_text
 
                     with self.renderer.lock:
                         if frame.width > self.render_width or frame.height > self.render_height:
                             frame = frame.resize(
-                                (self.render_width, self.render_height)
+                                (self.render_width, self.render_height),
+                                resample=Image.NEAREST  # Use nearest neighbor for max speed
                             )
-                        self.renderer.canvas.paste(
-                            frame,
-                            (self.render_rect[0], self.render_rect[1])
-                        )
+
+                        draw = ImageDraw.Draw(frame)
 
                         if scan_text:
-                            self.renderer.draw.text(
-                                xy=(
-                                    int(self.renderer.canvas_width/2),
-                                    self.renderer.canvas_height - GUIConstants.EDGE_PADDING
-                                ),
-                                text=scan_text,
-                                fill=GUIConstants.BODY_FONT_COLOR,
-                                font=instructions_font,
-                                stroke_width=4,
-                                stroke_fill=GUIConstants.BACKGROUND_COLOR,
-                                anchor="ms"
-                            )
+                            # Note: shadowed text (adding a 'stroke' outline) can
+                            # significantly slow down the rendering.
+                            # Temp solution: render a slight 1px shadow behind the text
+                            # TODO: Replace the instructions_text with a disappearing
+                            # toast/popup (see: QR Brightness UI)?
+                            draw.text(xy=(
+                                        int(self.renderer.canvas_width/2 + 2),
+                                        self.renderer.canvas_height - GUIConstants.EDGE_PADDING + 2
+                                     ),
+                                     text=scan_text,
+                                     fill="black",
+                                     font=instructions_font,
+                                     anchor="ms")
 
-                        self.renderer.show_image()
+                            # Render the onscreen instructions
+                            draw.text(xy=(
+                                        int(self.renderer.canvas_width/2),
+                                        self.renderer.canvas_height - GUIConstants.EDGE_PADDING
+                                     ),
+                                     text=scan_text,
+                                     fill=GUIConstants.BODY_FONT_COLOR,
+                                     font=instructions_font,
+                                     anchor="ms")
 
-                        end = timer()
-                        # print(f"{1.0/(end - start)} fps") # Time in seconds, e.g. 5.38091952400282
+                        self.renderer.show_image(frame, show_direct=True)
+                        # print(f" {cur_fps:0.2f} | {self.decoder_fps}")
 
-                time.sleep(0.05) # turn this up or down to tune performance while decoding psbt
                 if self.camera._video_stream is None:
                     break
 
@@ -112,16 +153,21 @@ class ScanScreen(BaseScreen):
             Screen. Once interaction starts, the display updates have to be managed in
             _run(). The live preview is an extra-complex case.
         """
+        num_frames = 0
+        start_time = time.time()
         while True:
             frame = self.camera.read_video_stream()
             if frame is not None:
                 status = self.decoder.add_image(frame)
 
+                num_frames += 1
+                decoder_fps = f"{num_frames / (time.time() - start_time):0.2f}"
+                self.threads[0].decoder_fps = decoder_fps
+
                 if status in (DecodeQRStatus.COMPLETE, DecodeQRStatus.INVALID):
                     self.camera.stop_video_stream_mode()
                     break
                 
-                # TODO: KEY_UP gives control to NavBar; use its back arrow to cancel
                 if self.hw_inputs.check_for_low(HardwareButtonsConstants.KEY_RIGHT) or self.hw_inputs.check_for_low(HardwareButtonsConstants.KEY_LEFT):
                     self.camera.stop_video_stream_mode()
                     break

src/seedsigner/hardware/camera.py

@@ -1,5 +1,4 @@
 import io
-import numpy
 
 from picamera import PiCamera
 from PIL import Image