refactor: use PyQT5 for window display (#49)

* wip: use PyQT5 for window display

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* wip: first slide is shown

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* wip: pushing non-working code

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* wip: some logging

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* feat: new configuration wizard working

* fix: prevent key error

* wip: making action work

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* wip: soon done! info + video

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix: bugs in sleep and exiting

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* try: offscreen

* fix: pop default value if not present

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* feat: add aspect ratio option

* chore: typing wip

* fix: now() function returns seconds, not milliseconds anymore

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
Jérome Eertmans
2022-10-19 11:08:41 +02:00
committed by GitHub
parent bc3d55fce2
commit d717bc651d
10 changed files with 464 additions and 255 deletions

View File

@ -1,35 +1,38 @@
import math
import os
import platform
import sys
import time
from enum import IntEnum, auto, unique
from typing import List, Tuple
from typing import List, Optional, Tuple
import click
import cv2
import numpy as np
from pydantic import ValidationError
from PyQt5 import QtGui
from PyQt5.QtCore import Qt, QThread, pyqtSignal, pyqtSlot
from PyQt5.QtGui import QPixmap
from PyQt5.QtWidgets import QApplication, QGridLayout, QLabel, QWidget
from tqdm import tqdm
from .commons import config_path_option, verbosity_option
from .config import Config, PresentationConfig, SlideConfig, SlideType
from .defaults import FOLDER_PATH, FONT_ARGS
from .defaults import FOLDER_PATH
from .manim import logger
INTERPOLATION_FLAGS = {
"nearest": cv2.INTER_NEAREST,
"linear": cv2.INTER_LINEAR,
"cubic": cv2.INTER_CUBIC,
"area": cv2.INTER_AREA,
"lanczos4": cv2.INTER_LANCZOS4,
"linear-exact": cv2.INTER_LINEAR_EXACT,
"nearest-exact": cv2.INTER_NEAREST_EXACT,
}
os.environ.pop(
"QT_QPA_PLATFORM_PLUGIN_PATH", None
) # See why here: https://stackoverflow.com/a/67863156
WINDOW_NAME = "Manim Slides"
WINDOW_INFO_NAME = f"{WINDOW_NAME}: Info"
WINDOWS = platform.system() == "Windows"
ASPECT_RATIO_MODES = {
"ignore": Qt.IgnoreAspectRatio,
"keep": Qt.KeepAspectRatio,
}
@unique
class State(IntEnum):
@ -44,14 +47,9 @@ class State(IntEnum):
return self.name.capitalize()
def now() -> int:
"""Returns time.time() in milliseconds."""
return round(time.time() * 1000)
def fix_time(t: float) -> float:
"""Clips time t such that it is always positive."""
return t if t > 0 else 1
def now() -> float:
"""Returns time.time() in seconds."""
return time.time()
class Presentation:
@ -61,17 +59,17 @@ class Presentation:
self.slides: List[SlideConfig] = config.slides
self.files: List[str] = config.files
self.current_slide_index = 0
self.current_animation = self.current_slide.start_animation
self.current_file = None
self.current_slide_index: int = 0
self.current_animation: int = self.current_slide.start_animation
self.current_file: Optional[str] = None
self.loaded_animation_cap = -1
self.loaded_animation_cap: int = -1
self.cap = None # cap = cv2.VideoCapture
self.reverse = False
self.reversed_animation = -1
self.reverse: bool = False
self.reversed_animation: int = -1
self.lastframe = None
self.lastframe: Optional[np.ndarray] = None
self.reset()
self.add_last_slide()
@ -105,9 +103,11 @@ class Presentation:
self.reverse and self.reversed_animation != animation
): # cap already loaded
logger.debug(f"Loading new cap for animation #{animation}")
self.release_cap()
file = self.files[animation]
file: str = self.files[animation]
if self.reverse:
file = "{}_reversed{}".format(*os.path.splitext(file))
@ -126,16 +126,19 @@ class Presentation:
def rewind_current_slide(self) -> None:
"""Rewinds current slide to first frame."""
logger.debug("Rewinding curring slide")
if self.reverse:
self.current_animation = self.current_slide.end_animation - 1
else:
self.current_animation = self.current_slide.start_animation
self.current_cap.set(cv2.CAP_PROP_POS_FRAMES, 0)
cap = self.current_cap
cap.set(cv2.CAP_PROP_POS_FRAMES, 0)
def cancel_reverse(self) -> None:
"""Cancels any effet produced by a reversed slide."""
if self.reverse:
logger.debug("Cancelling effects from previous 'reverse' action'")
self.reverse = False
self.reversed_animation = -1
self.release_cap()
@ -147,6 +150,7 @@ class Presentation:
def load_next_slide(self) -> None:
"""Loads next slide."""
logger.debug("Loading next slide")
if self.reverse:
self.cancel_reverse()
self.rewind_current_slide()
@ -160,6 +164,7 @@ class Presentation:
def load_previous_slide(self) -> None:
"""Loads previous slide."""
logger.debug("Loading previous slide")
self.cancel_reverse()
self.current_slide_index = max(0, self.current_slide_index - 1)
self.rewind_current_slide()
@ -167,7 +172,12 @@ class Presentation:
@property
def fps(self) -> int:
"""Returns the number of frames per second of the current video."""
return self.current_cap.get(cv2.CAP_PROP_FPS)
fps = self.current_cap.get(cv2.CAP_PROP_FPS)
if fps == 0:
logger.warn(
f"Something is wrong with video file {self.current_file}, as the fps returned by frame {self.current_frame_number} is 0"
)
return max(fps, 1) # TODO: understand why we sometimes get 0 fps
def add_last_slide(self) -> None:
"""Add a 'last' slide to the end of slides."""
@ -264,56 +274,37 @@ class Presentation:
return self.lastframe, state
class Display:
class Display(QThread):
"""Displays one or more presentations one after each other."""
change_video_signal = pyqtSignal(np.ndarray)
change_info_signal = pyqtSignal(dict)
finished = pyqtSignal()
def __init__(
self,
presentations,
config,
start_paused=False,
fullscreen=False,
skip_all=False,
resolution=(1980, 1080),
interpolation_flag=cv2.INTER_LINEAR,
record_to=None,
exit_after_last_slide=False,
) -> None:
super().__init__()
self.presentations = presentations
self.start_paused = start_paused
self.config = config
self.skip_all = skip_all
self.fullscreen = fullscreen
self.resolution = resolution
self.interpolation_flag = interpolation_flag
self.record_to = record_to
self.recordings = []
self.window_flags = (
cv2.WINDOW_GUI_NORMAL | cv2.WINDOW_FREERATIO | cv2.WINDOW_NORMAL
)
self.recordings: List[Tuple[str, int, int]] = []
self.state = State.PLAYING
self.lastframe = None
self.lastframe: Optional[np.ndarray] = None
self.current_presentation_index = 0
self.exit = False
self.run_flag = True
self.lag = 0
self.last_time = now()
cv2.namedWindow(
WINDOW_INFO_NAME,
cv2.WINDOW_GUI_NORMAL | cv2.WINDOW_FREERATIO | cv2.WINDOW_AUTOSIZE,
)
if self.fullscreen:
cv2.namedWindow(
WINDOW_NAME, cv2.WINDOW_GUI_NORMAL | cv2.WND_PROP_FULLSCREEN
)
cv2.setWindowProperty(
WINDOW_NAME, cv2.WND_PROP_FULLSCREEN, cv2.WINDOW_FULLSCREEN
)
else:
cv2.namedWindow(WINDOW_NAME, self.window_flags)
cv2.resizeWindow(WINDOW_NAME, *self.resolution)
self.key = -1
self.exit_after_last_slide = exit_after_last_slide
@property
def current_presentation(self) -> Presentation:
@ -322,7 +313,8 @@ class Display:
def run(self) -> None:
"""Runs a series of presentations until end or exit."""
while not self.exit:
while self.run_flag:
last_time = now()
self.lastframe, self.state = self.current_presentation.update_state(
self.state
)
@ -332,88 +324,98 @@ class Display:
self.start_paused = False
if self.state == State.END:
if self.current_presentation_index == len(self.presentations) - 1:
self.quit()
continue
if self.exit_after_last_slide:
self.run_flag = False
continue
else:
self.current_presentation_index += 1
self.state = State.PLAYING
self.handle_key()
if self.exit:
continue
self.show_video()
self.show_info()
lag = now() - last_time
sleep_time = 1 / self.current_presentation.fps
sleep_time = max(sleep_time - lag, 0)
time.sleep(sleep_time)
last_time = now()
self.current_presentation.release_cap()
if self.record_to is not None:
self.record_movie()
logger.debug("Closing video thread gracully and exiting")
self.finished.emit()
def record_movie(self) -> None:
logger.debug(
f"A total of {len(self.recordings)} frames will be saved to {self.record_to}"
)
file, frame_number, fps = self.recordings[0]
cap = cv2.VideoCapture(file)
cap.set(cv2.CAP_PROP_POS_FRAMES, frame_number - 1)
_, frame = cap.read()
w, h = frame.shape[:2]
fourcc = cv2.VideoWriter_fourcc(*"XVID")
out = cv2.VideoWriter(self.record_to, fourcc, fps, (h, w))
out.write(frame)
for _file, frame_number, _ in tqdm(
self.recordings[1:], desc="Creating recording file", leave=False
):
if file != _file:
cap.release()
file = _file
cap = cv2.VideoCapture(_file)
cap.set(cv2.CAP_PROP_POS_FRAMES, frame_number - 1)
_, frame = cap.read()
out.write(frame)
cap.release()
out.release()
def show_video(self) -> None:
"""Shows updated video."""
self.lag = now() - self.last_time
self.last_time = now()
if self.record_to is not None:
pres = self.current_presentation
self.recordings.append(
(pres.current_file, pres.current_frame_number, pres.fps)
)
frame = self.lastframe
# If Window was manually closed (impossible in fullscreen),
# we reopen it
if cv2.getWindowProperty(WINDOW_NAME, cv2.WND_PROP_VISIBLE) < 1:
cv2.namedWindow(WINDOW_NAME, self.window_flags)
cv2.resizeWindow(WINDOW_NAME, *self.resolution)
if WINDOWS: # Only resize on Windows
_, _, w, h = cv2.getWindowImageRect(WINDOW_NAME)
if (h, w) != frame.shape[:2]: # Only if shape is different
frame = cv2.resize(frame, (w, h), self.interpolation_flag)
cv2.imshow(WINDOW_NAME, frame)
frame: np.ndarray = self.lastframe
self.change_video_signal.emit(frame)
def show_info(self) -> None:
"""Shows updated information about presentations."""
info = np.zeros((130, 420), np.uint8)
font_args = (FONT_ARGS[0], 0.7, *FONT_ARGS[2:])
grid_x = [30, 230]
grid_y = [30, 70, 110]
cv2.putText(
info,
f"Animation: {self.current_presentation.current_animation}",
(grid_x[0], grid_y[0]),
*font_args,
)
cv2.putText(info, f"State: {self.state}", (grid_x[1], grid_y[0]), *font_args)
cv2.putText(
info,
f"Slide {self.current_presentation.current_slide.number}/{len(self.current_presentation.slides)}",
(grid_x[0], grid_y[1]),
*font_args,
)
cv2.putText(
info,
f"Slide Type: {self.current_presentation.current_slide.type}",
(grid_x[1], grid_y[1]),
*font_args,
self.change_info_signal.emit(
{
"animation": self.current_presentation.current_animation,
"state": self.state,
"slide_index": self.current_presentation.current_slide.number,
"n_slides": len(self.current_presentation.slides),
"type": self.current_presentation.current_slide.type,
"scene_index": self.current_presentation_index + 1,
"n_scenes": len(self.presentations),
}
)
cv2.putText(
info,
f"Scene {self.current_presentation_index + 1}/{len(self.presentations)}",
((grid_x[0] + grid_x[1]) // 2, grid_y[2]),
*font_args,
)
cv2.imshow(WINDOW_INFO_NAME, info)
@pyqtSlot(int)
def set_key(self, key: int) -> None:
"""Sets the next key to be handled."""
self.key = key
def handle_key(self) -> None:
"""Handles key strokes."""
sleep_time = math.ceil(1000 / self.current_presentation.fps)
key = cv2.waitKeyEx(fix_time(sleep_time - self.lag))
key = self.key
if self.config.QUIT.match(key):
self.quit()
self.run_flag = False
elif self.state == State.PLAYING and self.config.PLAY_PAUSE.match(key):
self.state = State.PAUSED
elif self.state == State.PAUSED and self.config.PLAY_PAUSE.match(key):
@ -446,42 +448,165 @@ class Display:
self.current_presentation.rewind_current_slide()
self.state = State.PLAYING
def quit(self) -> None:
"""Destroys all windows created by presentations and exits gracefully."""
cv2.destroyAllWindows()
self.key = -1 # No more key to be handled
if self.record_to is not None and len(self.recordings) > 0:
logger.debug(
f"A total of {len(self.recordings)} frames will be saved to {self.record_to}"
def stop(self):
"""Stops current thread, without doing anything after."""
self.run_flag = False
self.wait()
class Info(QWidget):
def __init__(self):
super().__init__()
self.setWindowTitle(WINDOW_INFO_NAME)
self.layout = QGridLayout()
self.setLayout(self.layout)
self.animationLabel = QLabel()
self.stateLabel = QLabel()
self.slideLabel = QLabel()
self.typeLabel = QLabel()
self.sceneLabel = QLabel()
self.layout.addWidget(self.animationLabel, 0, 0, 1, 2)
self.layout.addWidget(self.stateLabel, 1, 0)
self.layout.addWidget(self.slideLabel, 1, 1)
self.layout.addWidget(self.typeLabel, 2, 0)
self.layout.addWidget(self.sceneLabel, 2, 1)
self.update_info({})
@pyqtSlot(dict)
def update_info(self, info: dict):
self.animationLabel.setText("Animation: {}".format(info.get("animation", "na")))
self.stateLabel.setText("State: {}".format(info.get("state", "unknown")))
self.slideLabel.setText(
"Slide: {}/{}".format(
info.get("slide_index", "na"), info.get("n_slides", "na")
)
file, frame_number, fps = self.recordings[0]
)
self.typeLabel.setText("Slide Type: {}".format(info.get("type", "unknown")))
self.sceneLabel.setText(
"Scene: {}/{}".format(
info.get("scene_index", "na"), info.get("n_scenes", "na")
)
)
cap = cv2.VideoCapture(file)
cap.set(cv2.CAP_PROP_POS_FRAMES, frame_number - 1)
_, frame = cap.read()
w, h = frame.shape[:2]
fourcc = cv2.VideoWriter_fourcc(*"XVID")
out = cv2.VideoWriter(self.record_to, fourcc, fps, (h, w))
class InfoThread(QThread):
def __init__(self):
super().__init__()
self.dialog = Info()
self.run_flag = True
out.write(frame)
def start(self):
super().start()
for _file, frame_number, _ in tqdm(
self.recordings[1:], desc="Creating recording file", leave=False
):
if file != _file:
cap.release()
file = _file
cap = cv2.VideoCapture(_file)
self.dialog.show()
cap.set(cv2.CAP_PROP_POS_FRAMES, frame_number - 1)
_, frame = cap.read()
out.write(frame)
def stop(self):
self.dialog.deleteLater()
cap.release()
out.release()
self.exit = True
class App(QWidget):
send_key_signal = pyqtSignal(int)
def __init__(
self,
*args,
fullscreen: bool = False,
resolution: Tuple[int, int] = (1980, 1080),
hide_mouse: bool = False,
aspect_ratio: Qt.AspectRatioMode = Qt.IgnoreAspectRatio,
**kwargs,
):
super().__init__()
self.setWindowTitle(WINDOW_NAME)
self.display_width, self.display_height = resolution
self.aspect_ratio = aspect_ratio
if hide_mouse:
self.setCursor(Qt.BlankCursor)
self.label = QLabel(self)
self.label.setAlignment(Qt.AlignCenter)
self.label.resize(self.display_width, self.display_height)
self.pixmap = QPixmap(self.width(), self.height())
self.label.setPixmap(self.pixmap)
self.label.setMinimumSize(1, 1)
# create the video capture thread
self.thread = Display(*args, **kwargs)
# create the info dialog
self.info = Info()
self.info.show()
# info widget will also listen to key presses
self.info.keyPressEvent = self.keyPressEvent
if fullscreen:
self.showFullScreen()
# connect signals
self.thread.change_video_signal.connect(self.update_image)
self.thread.change_info_signal.connect(self.info.update_info)
self.thread.finished.connect(self.closeAll)
self.send_key_signal.connect(self.thread.set_key)
# start the thread
self.thread.start()
def keyPressEvent(self, event):
key = event.key()
# We send key to be handled by video display
self.send_key_signal.emit(key)
event.accept()
def closeAll(self):
logger.debug("Closing all QT windows")
self.thread.stop()
self.info.deleteLater()
self.deleteLater()
def resizeEvent(self, event):
self.pixmap = self.pixmap.scaled(self.width(), self.height(), self.aspect_ratio)
self.label.setPixmap(self.pixmap)
self.label.resize(self.width(), self.height())
def closeEvent(self, event):
self.closeAll()
event.accept()
@pyqtSlot(np.ndarray)
def update_image(self, cv_img: dict):
"""Updates the image_label with a new opencv image"""
self.pixmap = self.convert_cv_qt(cv_img)
self.label.setPixmap(self.pixmap)
@pyqtSlot(dict)
def update_info(self, info: dict):
"""Updates the image_label with a new opencv image"""
pass
def convert_cv_qt(self, cv_img):
"""Convert from an opencv image to QPixmap"""
rgb_image = cv2.cvtColor(cv_img, cv2.COLOR_BGR2RGB)
h, w, ch = rgb_image.shape
bytes_per_line = ch * w
convert_to_Qt_format = QtGui.QImage(
rgb_image.data, w, h, bytes_per_line, QtGui.QImage.Format_RGB888
)
p = convert_to_Qt_format.scaled(
self.width(),
self.height(),
self.aspect_ratio,
)
return QPixmap.fromImage(p)
@click.command()
@ -517,7 +642,7 @@ def _list_scenes(folder) -> List[str]:
)
pass
logger.info(f"Found {len(scenes)} valid scene configuration files in `{folder}`.")
logger.debug(f"Found {len(scenes)} valid scene configuration files in `{folder}`.")
return scenes
@ -535,9 +660,10 @@ def _list_scenes(folder) -> List[str]:
@click.option("--start-paused", is_flag=True, help="Start paused.")
@click.option("--fullscreen", is_flag=True, help="Fullscreen mode.")
@click.option(
"-s",
"--skip-all",
is_flag=True,
help="Skip all slides, useful the test if slides are working.",
help="Skip all slides, useful the test if slides are working. Automatically sets `--skip-after-last-slide` to True.",
)
@click.option(
"-r",
@ -548,19 +674,30 @@ def _list_scenes(folder) -> List[str]:
show_default=True,
)
@click.option(
"-i",
"--interpolation-flag",
type=click.Choice(INTERPOLATION_FLAGS.keys(), case_sensitive=False),
default="linear",
help="Set the interpolation flag to be used when resizing image. See OpenCV cv::InterpolationFlags.",
show_default=True,
)
@click.option(
"--to",
"--record-to",
"record_to",
type=click.Path(dir_okay=False),
default=None,
help="If set, the presentation will be recorded into a AVI video file with given name.",
)
@click.option(
"--exit-after-last-slide",
is_flag=True,
help="At the end of last slide, the application will be exited.",
)
@click.option(
"--hide-mouse",
is_flag=True,
help="Hide mouse cursor.",
)
@click.option(
"--aspect-ratio",
type=click.Choice(ASPECT_RATIO_MODES.keys(), case_sensitive=False),
default="ignore",
help="Set the aspect ratio mode to be used when rescaling video.",
show_default=True,
)
@click.help_option("-h", "--help")
@verbosity_option
def present(
@ -571,8 +708,10 @@ def present(
fullscreen,
skip_all,
resolution,
interpolation_flag,
record_to,
exit_after_last_slide,
hide_mouse,
aspect_ratio,
) -> None:
"""
Present SCENE(s), one at a time, in order.
@ -584,6 +723,9 @@ def present(
Use `manim-slide list-scenes` to list all available scenes in a given folder.
"""
if skip_all:
exit_after_last_slide = True
if len(scenes) == 0:
scene_choices = _list_scenes(folder)
@ -638,7 +780,7 @@ def present(
except ValidationError as e:
raise click.UsageError(str(e))
else:
logger.info("No configuration file found, using default configuration.")
logger.debug("No configuration file found, using default configuration.")
config = Config()
if record_to is not None:
@ -648,14 +790,19 @@ def present(
"Recording only support '.avi' extension. For other video formats, please convert the resulting '.avi' file afterwards."
)
display = Display(
app = QApplication(sys.argv)
a = App(
presentations,
config=config,
start_paused=start_paused,
fullscreen=fullscreen,
skip_all=skip_all,
resolution=resolution,
interpolation_flag=INTERPOLATION_FLAGS[interpolation_flag],
record_to=record_to,
exit_after_last_slide=exit_after_last_slide,
hide_mouse=hide_mouse,
aspect_ratio=ASPECT_RATIO_MODES[aspect_ratio],
)
display.run()
a.show()
sys.exit(app.exec_())
logger.debug("After")