Methods and GUI entries for landmark based ROI

.gitignore

@@ -12,8 +12,8 @@ BabbleApp/dist
 /testing
 /old_models
 /Lib
-/scripts
 /share
+/BabbleApp/Models/dev
 
 scripts/example_build_app_and_installer.bat
 scripts/example_build_app_and_installer.bat

BabbleApp/algo_settings_widget.py

@@ -43,7 +43,7 @@ def __init__(self, widget_id: Tab, main_config: BabbleSettingsConfig, osc_queue:
                 tooltip = "How many threads to use for processing the model.",
             ),
              ],
-            [sg.Text("Runtime:", background_color='#424042'),   # Replace with Dropdown once I have internet to view docs. 
+            [sg.Text("Runtime:", background_color='#424042'), 
                 sg.OptionMenu(
                     self.runtime_list,
                     self.config.gui_runtime,

BabbleApp/babble_processor.py

@@ -96,8 +96,8 @@ def __init__(
             self.input_name = self.sess.get_inputs()[0].name
             self.output_name = self.sess.get_outputs()[0].name
         try:
-            min_cutoff = float(self.settings.gui_min_cutoff)  # 15.5004
-            beta = float(self.settings.gui_speed_coefficient)  # 0.62
+            min_cutoff = float(self.settings.gui_min_cutoff)
+            beta = float(self.settings.gui_speed_coefficient)
         except:
             print('\033[93m[WARN] OneEuroFilter values must be a legal number.\033[0m')
             min_cutoff = 0.9

BabbleApp/babbleapp.py

@@ -46,7 +46,7 @@
 ALGO_SETTINGS_RADIO_NAME = "-ALGOSETTINGSRADIO-"
 
 page_url = "https://github.com/SummerSigh/ProjectBabble/releases/latest"
-appversion = "Babble v2.0.5"
+appversion = "Babble v2.0.6 Alpha"
 
 
 def main():

BabbleApp/camera_widget.py

@@ -4,6 +4,7 @@
 from collections import deque
 from threading import Event, Thread
 from babble_processor import BabbleProcessor, CamInfoOrigin
+from landmark_processor import LandmarkProcessor
 from enum import Enum
 from queue import Queue, Empty
 from camera import Camera, CameraState
@@ -22,6 +23,7 @@ def __init__(self, widget_id: Tab, main_config: BabbleConfig, osc_queue: Queue):
         self.gui_roi_layout = f"-ROILAYOUT{widget_id}-"
         self.gui_roi_selection = f"-GRAPH{widget_id}-"
         self.gui_tracking_button = f"-TRACKINGMODE{widget_id}-"
+        self.gui_autoroi = f"-AUTOROI{widget_id}-"
         self.gui_save_tracking_button = f"-SAVETRACKINGBUTTON{widget_id}-"
         self.gui_tracking_layout = f"-TRACKINGLAYOUT{widget_id}-"
         self.gui_tracking_image = f"-IMAGE{widget_id}-"
@@ -55,7 +57,18 @@ def __init__(self, widget_id: Tab, main_config: BabbleConfig, osc_queue: Queue):
         self.roi_queue = Queue(maxsize=2)
         self.image_queue = Queue(maxsize=500)
 
-        self.ransac = BabbleProcessor(
+        self.babble_cnn = BabbleProcessor(
+            self.config,
+            self.settings_config,
+            self.main_config,
+            self.cancellation_event,
+            self.capture_event,
+            self.capture_queue,
+            self.image_queue,
+            self.cam_id,
+        )
+
+        self.babble_landmark = LandmarkProcessor(
             self.config,
             self.settings_config,
             self.main_config,
@@ -79,14 +92,17 @@ def __init__(self, widget_id: Tab, main_config: BabbleConfig, osc_queue: Queue):
 
         self.roi_layout = [
             [
-                sg.Graph( 
-                    (640, 480),
-                    (0, 480),
-                    (640, 0),
-                    key=self.gui_roi_selection,
-                    drag_submits=True,
-                    enable_events=True,
-                    background_color='#424042',
+            sg.Button("Auto ROI", key=self.gui_autoroi, button_color='#539e8a', tooltip = "Automatically set ROI",),
+            ],
+            [
+            sg.Graph( 
+                (640, 480),
+                (0, 480),
+                (640, 0),
+                key=self.gui_roi_selection,
+                drag_submits=True,
+                enable_events=True,
+                background_color='#424042',
                 )
             ]
         ]
@@ -110,7 +126,7 @@ def __init__(self, widget_id: Tab, main_config: BabbleConfig, osc_queue: Queue):
             ],
             [
                 sg.Checkbox(
-                    "Use Neurtal Calibration:",
+                    "Use Neutral Calibration:",
                     default=self.config.use_n_calibration,
                     key=self.use_n_calibration,
                     background_color='#424042',
@@ -196,8 +212,10 @@ def start(self):
         if not self.cancellation_event.is_set():
             return
         self.cancellation_event.clear()
-        self.ransac_thread = Thread(target=self.ransac.run)
-        self.ransac_thread.start()
+        self.babble_cnn_thread = Thread(target=self.babble_cnn.run)
+        self.babble_cnn_thread.start()
+        self.babble_landmark_thread = Thread(target=self.babble_landmark.run)
+        self.babble_landmark_thread.start()
         self.camera_thread = Thread(target=self.camera.run)
         self.camera_thread.start()
 
@@ -206,7 +224,8 @@ def stop(self):
         if self.cancellation_event.is_set():
             return
         self.cancellation_event.set()
-        self.ransac_thread.join()
+        self.babble_cnn_thread.join()
+        self.babble_landmark_thread.join()
         self.camera_thread.join()
 
     def render(self, window, event, values):
@@ -288,7 +307,27 @@ def render(self, window, event, values):
                 self.config.roi_window_w = abs(self.x0 - self.x1)
                 self.config.roi_window_h = abs(self.y0 - self.y1)
                 self.main_config.save()
-
+
+        if event == self.gui_autoroi:
+            print("Auto ROI")
+            #image = self.image_queue.get()
+            #image = self.babble_landmark.get_frame()    # Get image for pfld 
+            #print(image)
+            #print(len(image))
+            #print(image)
+            #cv2.imwrite("yeah.png", image)
+            self.babble_landmark.infer_frame()
+            output = self.babble_landmark.output
+            print(f"Output: {output}")
+            self.x1 = output[2]
+            self.y1 = output[3]
+            self.x0 = output[0]
+            self.y0 = output[1]
+            self.config.roi_window_x = min([self.x0, self.x1])
+            self.config.roi_window_y = min([self.y0, self.y1])
+            self.config.roi_window_w = abs(self.x0 - self.x1)
+            self.config.roi_window_h = abs(self.y0 - self.y1)
+            self.main_config.save()
 
         if event == self.gui_roi_selection:
             # Event for mouse button down or mouse drag in ROI mode
@@ -298,11 +337,11 @@ def render(self, window, event, values):
             self.x1, self.y1 = values[self.gui_roi_selection]
 
         if event == self.gui_restart_calibration:
-            self.ransac.calibration_frame_counter = 1500
+            self.babble_cnn.calibration_frame_counter = 1500
             PlaySound('Audio/start.wav', SND_FILENAME | SND_ASYNC)
 
         if event == self.gui_stop_calibration:
-            self.ransac.calibration_frame_counter = 0
+            self.babble_cnn.calibration_frame_counter = 0
 
         needs_roi_set = self.config.roi_window_h <= 0 or self.config.roi_window_w <= 0
 
@@ -319,15 +358,15 @@ def render(self, window, event, values):
             window[self.gui_mode_readout].update("Camera Reconnecting...")
         elif needs_roi_set:
             window[self.gui_mode_readout].update("Awaiting Mouth Crop")
-        elif self.ransac.calibration_frame_counter != None:
+        elif self.babble_cnn.calibration_frame_counter != None:
             window[self.gui_mode_readout].update("Calibration")
         else:
             window[self.gui_mode_readout].update("Tracking")
             window[self.gui_tracking_fps].update(self._movavg_fps(self.camera.fps))
             window[self.gui_tracking_bps].update(self._movavg_bps(self.camera.bps))
 
         if self.in_roi_mode:
-            try:
+            try:    
                 if self.roi_queue.empty():
                     self.capture_event.set()
                 maybe_image = self.roi_queue.get(block=False)

BabbleApp/landmark_model_loader.py

@@ -0,0 +1,35 @@
+import os
+import json
+os.environ["OMP_NUM_THREADS"] = "1"
+import onnxruntime as ort
+import time
+import PySimpleGUI as sg
+import cv2
+import numpy as np
+from pythonosc import udp_client
+import utils.image_transforms as transforms
+import PIL.Image as Image
+from threading import Thread
+from one_euro_filter import OneEuroFilter
+
+def run_model(self): # Replace transforms n shit for the pfld model
+    if self.runtime == "ONNX" or self.runtime == "Default (ONNX)":
+        frame = cv2.resize(self.current_image_gray, (256, 256))
+        frame = transforms.to_tensor(frame)
+        frame = transforms.unsqueeze(frame,0)
+        out = self.sess.run([self.output_name], {self.input_name: frame})
+        output = out[0][0]
+
+        output = self.one_euro_filter(output)
+
+        for i in range(len(output)):  # Clip values between 0 - 1
+            output[i] = max(min(output[i], 1), 0)
+
+        self.output = output
+
+
+def write_image(self): # Debug function for development, remove once pfld is implemented. 
+    frame = cv2.resize(self.current_image_gray, (256, 256))
+    cv2.imwrite("yeah.png", frame)
+    print("Image Wrote")
+    self.output = (0, 0, 100, 100)