improvements for 1.5.2

This commit is contained in:
2026-07-29 17:20:23 -07:00
parent 8b017005c5
commit f15a5d9433
8 changed files with 750 additions and 218 deletions
+391 -47
View File
@@ -22,6 +22,7 @@ from datetime import datetime
from multiprocessing import Process, current_process, freeze_support, Queue
# External library imports
import pandas as pd
import psutil
from mne.io import read_raw_snirf
@@ -29,8 +30,8 @@ from mne.preprocessing.nirs import source_detector_distances
from mne_nirs.channels import get_short_channels # type: ignore
from PySide6.QtWidgets import (
QApplication, QWidget, QMessageBox, QVBoxLayout, QHBoxLayout, QTextEdit, QScrollArea, QComboBox, QGridLayout, QSplitter,
QPushButton, QMainWindow, QFileDialog, QLabel, QLineEdit, QFrame, QSizePolicy, QGroupBox, QDialog, QMenu, QSpinBox
QApplication, QWidget, QMessageBox, QVBoxLayout, QHBoxLayout, QTextEdit, QScrollArea, QComboBox, QGridLayout, QSplitter, QDialogButtonBox, QHeaderView,
QPushButton, QMainWindow, QFileDialog, QLabel, QLineEdit, QFrame, QSizePolicy, QGroupBox, QDialog, QMenu, QSpinBox, QTableWidget, QTableWidgetItem
)
from PySide6.QtCore import QThread, Signal, Qt, QTimer, QPoint
from PySide6.QtGui import QAction, QKeySequence, QIcon
@@ -105,7 +106,7 @@ SECTIONS = [
"title": "Short/Long Channels",
"params": [
{"name": "SHORT_CHANNELS", "default": True, "type": bool, "help": "This should be set to True if the data has a short channel present in the data. For more information about short channels, please visit the Wiki."},
{"name": "SHORT_CHANNELS_THRESHOLD", "default": 0.015, "type": float, "depends_on": "SHORT_CHANNEL", "help": "The maximum distance the short channel can be in metres before it is no longer considered a short channel."},
{"name": "SHORT_CHANNELS_THRESHOLD", "default": 0.015, "type": float, "depends_on": "SHORT_CHANNELS", "help": "The maximum distance the short channel can be in metres before it is no longer considered a short channel."},
{"name": "LONG_CHANNELS_THRESHOLD", "default": 0.045, "type": float, "help": "The maximum distance channels can be in metres. Any channel longer than this distance will be discarded."},
]
},
@@ -259,7 +260,7 @@ SECTIONS = [
{"name": "OVERSAMPLING", "default": 50, "type": int, "help": "Oversampling factor used in temporal convolutions."},
# TODO: Re-implement this without causing a memory leak
# {"name": "REMOVE_EVENTS", "default": "None", "type": list, "help": "Remove events matching the names provided before generating the Design Matrix"},
{"name": "SHORT_CHANNEL_REGRESSION", "default": True, "type": bool, "depends_on": "SHORT_CHANNEL", "help": "Should short channel regression be used to create the design matrix? This will use the 'signal' from the short channel and regress it out of all other channels."},
{"name": "SHORT_CHANNEL_REGRESSION", "default": True, "type": bool, "depends_on": "SHORT_CHANNELS", "help": "Should short channel regression be used to create the design matrix? This will use the 'signal' from the short channel and regress it out of all other channels."},
]
},
{
@@ -349,18 +350,109 @@ class SavingOverlay(QDialog):
class GroupAssignmentDialog(QDialog):
"""Dialog allowing users to create groups and assign unique metadata values to them."""
def __init__(self, parent=None, unique_values=None, field_name="AGE"):
super().__init__(parent)
self.setWindowTitle(f"Assign Groups by {field_name}")
self.resize(520, 420)
self.unique_values = unique_values or []
self.field_name = field_name
# Initial default groups
self.groups = ["Group A", "Group B"]
self.combos = []
self._init_ui()
def _init_ui(self):
layout = QVBoxLayout(self)
# Header info
info_label = QLabel(
f"Found <b>{len(self.unique_values)}</b> unique <i>{self.field_name}</i> value(s) "
f"across loaded files.<br>Create custom group names and assign each value below:"
)
info_label.setWordWrap(True)
layout.addWidget(info_label)
# Group Creation Bar
group_box_layout = QHBoxLayout()
self.group_input = QLineEdit()
self.group_input.setPlaceholderText(
"Enter new group name (e.g. Infants, Control)..."
)
self.group_input.returnPressed.connect(self._add_group)
add_btn = QPushButton("Add Group")
add_btn.clicked.connect(self._add_group)
group_box_layout.addWidget(self.group_input)
group_box_layout.addWidget(add_btn)
layout.addLayout(group_box_layout)
# Mapping Table (Values -> Group Dropdown)
self.table = QTableWidget(len(self.unique_values), 2)
self.table.setHorizontalHeaderLabels(
[f"{self.field_name} Value", "Assigned Group"]
)
self.table.horizontalHeader().setSectionResizeMode(
QHeaderView.ResizeMode.Stretch
)
for row, val in enumerate(self.unique_values):
# Metadata Value Column (Read-Only)
val_item = QTableWidgetItem(str(val))
val_item.setFlags(val_item.flags() ^ Qt.ItemFlag.ItemIsEditable)
self.table.setItem(row, 0, val_item)
# Assigned Group Column (ComboBox)
combo = QComboBox()
self.combos.append(combo)
self.table.setCellWidget(row, 1, combo)
self._refresh_combos()
layout.addWidget(self.table)
# Dialog Buttons
buttons = QDialogButtonBox(
QDialogButtonBox.StandardButton.Ok
| QDialogButtonBox.StandardButton.Cancel
)
buttons.accepted.connect(self.accept)
buttons.rejected.connect(self.reject)
layout.addWidget(buttons)
def _add_group(self):
"""Adds a new group to the available options."""
name = self.group_input.text().strip()
if name and name not in self.groups:
self.groups.append(name)
self.group_input.clear()
self._refresh_combos()
def _refresh_combos(self):
"""Refreshes all dropdown choices while preserving active selections."""
for combo in self.combos:
current_selection = combo.currentText()
combo.clear()
combo.addItem("-- Select Group --")
combo.addItems(self.groups)
# Preserve previous selection if it still exists
if current_selection in self.groups:
combo.setCurrentText(current_selection)
def get_mappings(self) -> dict:
"""Returns a mapping dictionary: { metadata_value: assigned_group_name }"""
mappings = {}
for row, val in enumerate(self.unique_values):
assigned = self.combos[row].currentText()
if assigned and assigned != "-- Select Group --":
mappings[str(val)] = assigned
return mappings
@@ -389,6 +481,8 @@ class ProgressBubble(QWidget):
self.spinner_idx = 0
self.is_loading = False
self.base_text = display_name
self.status_icon = ""
self.suffix_text = ""
self.label.setAlignment(Qt.AlignmentFlag.AlignCenter)
self.label.setStyleSheet("""
QLabel {
@@ -426,6 +520,17 @@ class ProgressBubble(QWidget):
# Resize policy to make bubbles responsive
self.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Minimum)
def _update_label_text(self):
"""Combines base text, green checkmark/spinner, and metadata into one display label."""
text = self.base_text
if self.status_icon:
text += f" {self.status_icon}"
if self.suffix_text:
text += f" ({self.suffix_text})"
self.label.setText(text)
def set_loading_state(self, loading=True):
self.is_loading = loading
if loading:
@@ -433,7 +538,15 @@ class ProgressBubble(QWidget):
else:
self.loading_timer.stop()
# Transition to a green checkmark
self.setSuffixText(" <span style='color: green;'>✔</span>")
self.status_icon = "<span style='color: green;'>✔</span>"
self._update_label_text()
def setSuffixText(self, suffix):
"""Updates the metadata text portion without destroying the checkmark."""
self.suffix_text = suffix if suffix else ""
self._update_label_text()
def update_progress(self, step_index, active=True):
self.current_step = step_index
@@ -445,11 +558,17 @@ class ProgressBubble(QWidget):
rect.setStyleSheet(f"background-color: {color}; border: 1px solid gray;")
else:
rect.setStyleSheet("background-color: white; border: 1px solid gray;")
def mark_cancelled(self):
if 0 <= self.current_step < len(self.rects):
rect = self.rects[self.current_step]
rect.setStyleSheet("background-color: red; border: 1px solid gray;")
for i, rect in enumerate(self.rects):
if i < self.current_step:
rect.setStyleSheet("background-color: green; border: 1px solid gray;")
elif i == self.current_step:
rect.setStyleSheet("background-color: red; border: 1px solid gray;")
else:
rect.setStyleSheet("background-color: white; border: 1px solid gray;")
def mousePressEvent(self, event):
if event.button() == Qt.MouseButton.LeftButton:
@@ -458,17 +577,13 @@ class ProgressBubble(QWidget):
self.rightClicked.emit(self, event.globalPosition().toPoint())
super().mousePressEvent(event)
def setSuffixText(self, suffix):
if suffix:
self.label.setText(f"{self.base_text} {suffix}")
else:
self.label.setText(self.base_text)
def _rotate_spinner(self):
frame = self.spinner_frames[self.spinner_idx % len(self.spinner_frames)]
# Using HTML in setText allows us to style the spinner specifically
self.setSuffixText(f" <span style='color: #555;'>{frame}</span>")
self.status_icon = f"<span style='color: #555;'>{frame}</span>"
self.spinner_idx += 1
self._update_label_text()
@@ -1280,29 +1395,42 @@ class MainApplication(QMainWindow):
if not hasattr(self, 'pending_files_count'): self.pending_files_count = 0
self.pending_files_count += len(new_files)
for path in new_files:
self.selected_paths.append(path)
self.add_to_recent_files(path)
# Create the UI Bubble (Disconnected by default)
display_name = os.path.basename(path)
bubble = ProgressBubble(display_name, path)
bubble.setCursor(Qt.CursorShape.WaitCursor)
bubble.set_loading_state(True)
self.bubble_widgets[path] = bubble
self.bubble_layout.addWidget(bubble)
# 4. Queue the background work
future = self.file_executor.submit(_extract_metadata_worker, path)
# Use lambda with defaults to freeze the path and session at this moment
future.add_done_callback(
lambda f, p=path, s=current_session: self._on_metadata_ready(f, p, s)
)
self.button1.setVisible(True)
self.statusBar().showMessage(f"Loading {len(new_files)} new file(s)...")
# Queue chunked widget creation
CHUNK_SIZE = 10
def process_chunk(file_queue):
# Extract the next batch of files to build
chunk = file_queue[:CHUNK_SIZE]
remaining = file_queue[CHUNK_SIZE:]
for path in chunk:
self.selected_paths.append(path)
self.add_to_recent_files(path)
display_name = os.path.basename(path)
bubble = ProgressBubble(display_name, path)
bubble.setCursor(Qt.CursorShape.WaitCursor)
bubble.set_loading_state(True)
self.bubble_widgets[path] = bubble
self.bubble_layout.addWidget(bubble)
# Submit background task as each bubble is constructed
future = self.file_executor.submit(_extract_metadata_worker, path)
future.add_done_callback(
lambda f, p=path, s=current_session: self._on_metadata_ready(f, p, s)
)
# If more files remain, schedule the next batch without locking the UI
if remaining:
QTimer.singleShot(0, lambda: process_chunk(remaining))
# Trigger the first batch
process_chunk(new_files)
# TODO: Is this needed?
# def open_multiple_folders_dialog(self):
@@ -1779,12 +1907,25 @@ class MainApplication(QMainWindow):
self.statusBar().showMessage(msg)
def get_suffix_from_meta_fields(self):
def get_suffix_from_meta_fields(self, data_dict=None):
"""
Returns formatted suffix string.
If data_dict is passed, uses stored dictionary values.
Otherwise, reads directly from live UI QLineEdits.
"""
parts = []
for key, line_edit in self.meta_fields.items():
val = line_edit.text().strip()
if val:
parts.append(f"{key}: {val}")
if data_dict is not None:
for key in self.meta_fields.keys():
val = str(data_dict.get(key, '')).strip()
if val:
parts.append(f"{key}: {val}")
else:
for key, line_edit in self.meta_fields.items():
val = line_edit.text().strip()
if val:
parts.append(f"{key}: {val}")
return ", ".join(parts)
def on_bubble_clicked(self, bubble):
@@ -2116,8 +2257,7 @@ class MainApplication(QMainWindow):
all_params = {}
for section_widget in self.param_sections:
section_params = section_widget.get_param_values()
all_params.update(section_params)
all_params.update(section_params)
if self.folding_bypass:
all_params['FOLDING_BYP'] = True
@@ -2150,7 +2290,13 @@ class MainApplication(QMainWindow):
self.statusbar.showMessage("Task started in separate process.")
def _format_elapsed(self, seconds: float) -> str:
seconds = int(seconds)
h, rem = divmod(seconds, 3600)
m, s = divmod(rem, 60)
if h:
return f"{h:d}:{m:02d}:{s:02d}"
return f"{m:02d}:{s:02d}"
def check_for_pipeline_results(self):
try:
@@ -2176,14 +2322,25 @@ class MainApplication(QMainWindow):
for item, value in zip(DATA_SCHEMA, results):
getattr(self, item["key"])[file_path] = value
self.statusbar.showMessage(f"Processed: {os.path.basename(file_path)}")
elapsed_str = self._format_elapsed(getattr(self, "_last_elapsed", 0))
self.statusbar.showMessage(
f"Processed: {os.path.basename(file_path)} | Elapsed: {elapsed_str}"
)
else:
self.files_failed.add(file_path)
error_msg = msg.get("error", "Unknown worker error")
print(f"[DEBUG] File Failed: {os.path.basename(file_path)} - {error_msg}")
self.mark_file_failed(file_path)
self.show_error_popup(f"Error: {file_path}", error_msg, msg.get("traceback", ""))
self.statusbar.showMessage(f"Failed: {os.path.basename(file_path)}")
elapsed_str = self._format_elapsed(getattr(self, "_last_elapsed", 0))
self.statusbar.showMessage(
f"Failed: {os.path.basename(file_path)} | Elapsed: {elapsed_str}"
)
elif isinstance(msg, dict) and msg.get("type") == "elapsed":
# Live tick, once a second, independent of file completions
self._last_elapsed = msg["seconds"]
self.statusbar.showMessage(f"Processing... Elapsed: {self._format_elapsed(msg['seconds'])}")
elif isinstance(msg, dict) and msg.get("type") == "FINISHED_SUCCESSFULLY":
# The child has finished its work AND its own cleanup.
@@ -2196,7 +2353,12 @@ class MainApplication(QMainWindow):
success_count = len(self.files_results)
fail_count = self.files_total - success_count
self.statusbar.showMessage(f"Complete: {success_count} succeeded, {fail_count} failed.")
elapsed_str = self._format_elapsed(msg.get("elapsed", getattr(self, "_last_elapsed", 0)))
speedup = msg.get("speedup")
speedup_str = f" | Speedup: {speedup:.1f}x" if speedup else ""
self.statusbar.showMessage(
f"Complete: {success_count} succeeded, {fail_count} failed. | Total time: {elapsed_str}{speedup_str}"
)
if success_count > 0:
self.button3.setVisible(True)
@@ -2214,6 +2376,9 @@ class MainApplication(QMainWindow):
elif isinstance(msg, dict) and (msg.get("success") is False or msg.get("type") == "error"):
file_path = msg.get("file", "Process")
error_msg = msg.get("error", "Unknown error")
if file_path:
self.mark_file_failed(file_path)
self.files_done.add(file_path)
self.show_error_popup(f"Error: {file_path}", error_msg, msg.get("traceback", ""))
self.files_done.add(file_path)
if msg.get("success") is False: # Fatal crash
@@ -2232,6 +2397,14 @@ class MainApplication(QMainWindow):
self.statusbar.showMessage("Background process died.")
self.result_timer.stop()
def mark_file_failed(self, file_path):
if not file_path:
return
key = os.path.normpath(file_path)
bubble = self.bubble_widgets.get(key)
if bubble:
bubble.mark_cancelled()
def show_error_popup(self, title, error_message, traceback_str=""):
msgbox = QMessageBox(self)
@@ -2305,6 +2478,23 @@ class MainApplication(QMainWindow):
# Gracefully shut down multiprocessing children
print("Window is closing. Cleaning up...")
if hasattr(self, 'loading_session_id'):
self.loading_session_id += 1
if hasattr(self, 'file_executor') and self.file_executor is not None:
try:
# cancel_futures=True drops pending tasks (Python 3.9+)
self.file_executor.shutdown(wait=False, cancel_futures=True)
except TypeError:
# Fallback for older Python versions
self.file_executor.shutdown(wait=False)
self.file_executor = None
if hasattr(self, 'result_process') and self.result_process is not None:
if self.result_process.is_alive():
self.result_process.terminate()
self.result_process.join(timeout=0.2)
if hasattr(self, 'manager'):
self.manager.shutdown()
@@ -2321,6 +2511,9 @@ class MainApplication(QMainWindow):
if session_id != self.loading_session_id:
return
if future.cancelled():
return
try:
result = future.result()
@@ -2331,6 +2524,14 @@ class MainApplication(QMainWindow):
elif 'status' not in result:
# Wrap the raw extraction dictionary into our unified UI format
extracted_age = result.get('age', '')
# Save AGE directly into file_metadata tracking
if file_path not in self.file_metadata:
self.file_metadata[file_path] = {}
if extracted_age:
self.file_metadata[file_path]['AGE'] = extracted_age
result = {'status': 'success', 'data': result}
except Exception as e:
@@ -2355,7 +2556,34 @@ class MainApplication(QMainWindow):
return
# Success path
self.metadata_cache[file_path] = result.get('data', result)
data = result.get('data', result)
norm_path = os.path.normpath(file_path)
self.metadata_cache[norm_path] = data
# 3. Store extracted BIDS age into file_metadata store
if not hasattr(self, 'file_metadata'):
self.file_metadata = {}
if norm_path not in self.file_metadata:
self.file_metadata[norm_path] = {}
# 1. Store extracted BIDS metadata into file_metadata map
extracted_age = data.get('age', '')
if extracted_age:
self.file_metadata[norm_path]['AGE'] = str(extracted_age)
# 4. Update ONLY the text suffix on the bubble (spinner stays active!)
bubble = self.bubble_widgets.get(norm_path) or self.bubble_widgets.get(file_path)
if bubble:
suffix = self.get_suffix_from_meta_fields(self.file_metadata[norm_path])
bubble.setSuffixText(suffix)
# DO NOT call set_loading_state(False) here.
# The spinner keeps running while the rest of the pipeline executes.
# 5. Sync active form if this file is currently selected in UI
current_active = getattr(self, 'current_file', None)
if current_active and os.path.normpath(current_active) == norm_path:
self.populate_metadata_fields(file_path)
self.metadata_processed.emit(file_path, session_id)
@@ -2398,6 +2626,70 @@ class MainApplication(QMainWindow):
if self.pending_files_count <= 0:
self._cleanup_executor()
self.statusbar.showMessage("All files loaded sucessfully.")
has_metadata = any(
bool(meta_dict)
for meta_dict in getattr(self, "file_metadata", {}).values()
)
if has_metadata:
reply = QMessageBox.question(
self,
"Metadata Detected",
"Extracted metadata was found in the loaded files. Would you like to assign groups based on the metadata?",
QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No,
QMessageBox.StandardButton.Yes,
)
if reply == QMessageBox.StandardButton.Yes:
# A. Collect all unique non-empty AGE values across loaded files
unique_ages = sorted(
list(
{
str(meta.get("AGE", "")).strip()
for meta in getattr(
self, "file_metadata", {}
).values()
if str(meta.get("AGE", "")).strip()
}
)
)
if not unique_ages:
QMessageBox.information(
self,
"No Groupable Metadata",
"No 'AGE' values were found in the metadata to group.",
)
return
# B. Open Group Assignment Dialog
dialog = GroupAssignmentDialog(
self, unique_values=unique_ages, field_name="AGE"
)
if dialog.exec() == QDialog.DialogCode.Accepted:
mappings = dialog.get_mappings()
# C. Update 'GROUP' in self.file_metadata for matching files
for path_key, meta in self.file_metadata.items():
file_age = str(meta.get("AGE", "")).strip()
if file_age in mappings:
meta["GROUP"] = mappings[file_age]
# D. Update the text on ALL bubbles immediately
for path_key, b_widget in self.bubble_widgets.items():
normalized_k = os.path.normpath(path_key)
meta_dict = self.file_metadata.get(
normalized_k, self.file_metadata.get(path_key, {})
)
suffix = self.get_suffix_from_meta_fields(meta_dict)
b_widget.setSuffixText(suffix)
# E. Refresh form fields if a file is currently active in UI
current_active = getattr(self, "current_file", None)
if current_active:
self.populate_metadata_fields(current_active)
def _cleanup_executor(self):
"""Safely shuts down the executor and clears the reference."""
@@ -2407,6 +2699,51 @@ class MainApplication(QMainWindow):
print("[System] Background worker dismissed. RAM reclaimed.")
def _get_bids_age(snirf_path: str) -> str:
"""Traverses the path of a SNIRF file to extract age from BIDS TSV files."""
path = Path(snirf_path)
# Extract sub-XX and ses-YY labels from the path
sub_id = next((part for part in path.parts if part.startswith("sub-")), None)
ses_id = next((part for part in path.parts if part.startswith("ses-")), None)
if not sub_id:
return ""
# 1. Look for sub-<id>/sub-<id>_sessions.tsv
sub_dir = next((p for p in path.parents if p.name == sub_id), None)
if sub_dir and ses_id:
sessions_tsv = sub_dir / f"{sub_id}_sessions.tsv"
if sessions_tsv.exists():
try:
df = pd.read_csv(sessions_tsv, sep="\t")
# Handle session_id matching whether it includes 'ses-' or not
matching = df[df["session_id"].astype(str).str.replace("ses-", "") == ses_id.replace("ses-", "")]
if not matching.empty and "age" in matching.columns:
val = matching.iloc[0]["age"]
if pd.notna(val):
return str(val)
except Exception:
pass
# 2. Fallback: Check dataset root participants.tsv
bids_root = sub_dir.parent if sub_dir else None
if bids_root:
participants_tsv = bids_root / "participants.tsv"
if participants_tsv.exists():
try:
df = pd.read_csv(participants_tsv, sep="\t")
matching = df[df["participant_id"].astype(str).str.replace("sub-", "") == sub_id.replace("sub-", "")]
if not matching.empty and "age" in matching.columns:
val = matching.iloc[0]["age"]
if pd.notna(val):
return str(val)
except Exception:
pass
return ""
def _extract_metadata_worker(file_name):
"""Runs in the separate worker process. Returns a clean dict."""
@@ -2454,6 +2791,7 @@ def _extract_metadata_worker(file_name):
else:
snirf_info['Annotations'] = "No annotations found"
snirf_info["age"] = _get_bids_age(file_name)
return snirf_info
except Exception as e:
@@ -2473,6 +2811,12 @@ def run_gui_entry_wrapper(config, gui_queue, progress_queue, ack_queue):
"""
Where the processing happens
"""
# TODO: Are these needed?
os.environ["OMP_NUM_THREADS"] = "1"
os.environ["OPENBLAS_NUM_THREADS"] = "1"
os.environ["MKL_NUM_THREADS"] = "1"
os.environ["NUMEXPR_NUM_THREADS"] = "1"
os.environ["VECLIB_MAXIMUM_THREADS"] = "1"
try:
import flares as flares