Add native installers and independent arena annotation workflow

.github/workflows/publish.yml

@@ -1,112 +1,87 @@
-name: Build and Publish
+name: Native installers and releases
 
 on:
   push:
-    branches: [ master ]
-    tags: [ 'v*' ]
+    branches: [main, master]
+    tags: ['v*']
   pull_request:
-    branches: [ master ]
+    branches: [main, master]
+  workflow_dispatch:
 
-jobs:
-  build-windows:
-    runs-on: windows-latest
-    steps:
-    - uses: actions/checkout@v3
-
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.10'
-        cache: 'pip'
-
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-        pip install pyinstaller
+permissions:
+  contents: read
 
-    - name: Build with PyInstaller
-      run: |
-        pyinstaller ethoscore.spec
-
-    - name: Upload Artifact
-      uses: actions/upload-artifact@v4
-      with:
-        name: windows-exe
-        path: dist/ethoscore.exe
-
-  build-linux:
-    runs-on: ubuntu-22.04
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - runner: windows-latest
+            platform: Windows-x86_64
+          - runner: ubuntu-22.04
+            platform: Linux-x86_64
+          - runner: macos-15
+            platform: macOS-arm64
+          - runner: macos-15-intel
+            platform: macOS-x86_64
+    runs-on: ${{ matrix.runner }}
     steps:
-    - uses: actions/checkout@v3
-
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.10'
-        cache: 'pip'
-
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-        pip install pyinstaller
-
-    - name: Build with PyInstaller
-      run: |
-        pyinstaller ethoscore.spec
-
-    - name: Create AppImage
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y libfuse2
-        wget https://github.com/linuxdeploy/linuxdeploy/releases/download/continuous/linuxdeploy-x86_64.AppImage
-        chmod +x linuxdeploy-x86_64.AppImage
-
-        # Prepare AppDir structure
-        mkdir -p AppDir/usr/bin
-        cp dist/ethoscore AppDir/usr/bin/
-        chmod +x AppDir/usr/bin/ethoscore
-        cp ethoscore.desktop AppDir/
-        cp assets/controller-logo.svg AppDir/ethoscore.svg
-
-        # Run linuxdeploy
-        export ARCH=x86_64
-        ./linuxdeploy-x86_64.AppImage --appimage-extract-and-run --appdir AppDir -e AppDir/usr/bin/ethoscore -d ethoscore.desktop -i AppDir/ethoscore.svg --output appimage
-
-        # Rename output
-        mv Ethoscore-x86_64.AppImage dist/ethoscore.AppImage
-
-    - name: Upload Artifact
-      uses: actions/upload-artifact@v4
-      with:
-        name: linux-appimage
-        path: dist/ethoscore.AppImage
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: pip
+      - name: Install native Linux Qt runtime libraries
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libegl1 libopengl0 libxkbcommon-x11-0 libxcb-cursor0 libxcb-icccm4 libxcb-keysyms1 libxcb-shape0
+      - name: Install application and build dependencies
+        run: python -m pip install -r requirements.txt 'pyinstaller>=6.11'
+      - name: Validate installer transactions and malicious-archive rejection
+        run: python -m unittest installer.test_backend -v
+      - name: Validate arena detection, export, and annotation workflows
+        env:
+          QT_QPA_PLATFORM: offscreen
+          SDL_AUDIODRIVER: dummy
+        run: python -m unittest discover -s tests -v
+      - name: Build native application and graphical setup
+        env:
+          ETHOSCORE_BUILD_VERSION: ${{ github.ref_name }}-${{ github.sha }}
+        run: python -m installer.build --version "$ETHOSCORE_BUILD_VERSION"
+        shell: bash
+      - name: Smoke-test the actual native setup and installed application
+        env:
+          QT_QPA_PLATFORM: offscreen
+          SDL_AUDIODRIVER: dummy
+        run: python -m installer.native_smoke
+      - name: Upload native installers and update payloads
+        uses: actions/upload-artifact@v4
+        with:
+          name: Ethoscore-${{ matrix.platform }}
+          path: dist/releases/*
+          if-no-files-found: error
 
   publish:
-    needs: [build-windows, build-linux]
-    if: github.event_name == 'push' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/'))
+    needs: build
+    if: github.event_name == 'push'
+    permissions:
+      contents: write
     runs-on: ubuntu-latest
     steps:
-    - name: Download Windows Artifact
-      uses: actions/download-artifact@v4
-      with:
-        name: windows-exe
-        path: ./publish
-
-    - name: Download Linux Artifact
-      uses: actions/download-artifact@v4
-      with:
-        name: linux-appimage
-        path: ./publish
-
-    - name: Create Release
-      uses: ncipollo/release-action@v1
-      with:
-        artifacts: "./publish/*"
-        token: ${{ secrets.GITHUB_TOKEN }}
-        generateReleaseNotes: true
-        allowUpdates: true
-        tag: ${{ startsWith(github.ref, 'refs/tags/') && github.ref_name || 'latest' }}
-        name: ${{ startsWith(github.ref, 'refs/tags/') && github.ref_name || 'Latest Build' }}
-        prerelease: ${{ !startsWith(github.ref, 'refs/tags/') }}
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: Ethoscore-*
+          path: publish
+          merge-multiple: true
+      - name: Publish native installer release
+        uses: ncipollo/release-action@v1
+        with:
+          artifacts: publish/*
+          generateReleaseNotes: true
+          allowUpdates: true
+          replacesArtifacts: true
+          tag: ${{ startsWith(github.ref, 'refs/tags/') && github.ref_name || 'latest' }}
+          name: ${{ startsWith(github.ref, 'refs/tags/') && github.ref_name || 'Latest build' }}
+          prerelease: ${{ !startsWith(github.ref, 'refs/tags/') }}

.gitignore

@@ -1,4 +1,10 @@
 roadmap.md
 __pycache__/
 *.pyc
-behaviors.csv
+behaviors.csv
+SESSION_MEMORY.md
+build/
+dist/
+.test-artifacts/
+tests/artifacts/
+installer-build.log

README.md

@@ -18,7 +18,18 @@ Ethoscore is a video annotation tool for behavioral analysis, built with PySide6
 
 ## Installation
 
-**Prerequisites**: Python 3.8+, pip
+Download the **Ethoscore Setup** package for Windows, macOS, or Linux from the
+[releases page](https://github.com/BelloneLab/ethoscore/releases). Setup includes
+the application and dependencies, lets you choose its installation folder and
+desktop / Start menu / Applications shortcuts, and can update an existing install.
+Reopen Setup and choose **Download the latest official build**, then
+**Update / Reinstall**. Videos, annotations, preferences, and earlier installed
+versions are retained. Relaunch Ethoscore to use the new version.
+
+Native packaging and developer build instructions are in
+[installer/INSTALLATION.md](installer/INSTALLATION.md).
+
+**Run from source**: Python 3.12 and pip are recommended. Use a dedicated environment.
 
 **Install dependencies**:
 ```bash
@@ -30,6 +41,98 @@ pip install -r requirements.txt
 python ethoscore.py
 ```
 
+## Multiple arenas in one recording
+
+1. Open a video and select **Set up arenas** in the right sidebar or **Arenas** menu.
+2. Choose **Auto-detect arenas** to propose crops from stable dividers. Review the
+   proposals. If the layout is irregular or detection finds no reliable dividers,
+   drag rectangles over the source preview or create a grid with your chosen row
+   and column counts. Edit names and pixel coordinates in the table.
+3. Select **Export arenas**. A background FFmpeg job decodes the source once and
+   writes all crops. The original recording is never edited. Export can be
+   cancelled; partial crops are not published.
+4. Select an arena using the dropdown or previous / next buttons. **Only that
+   arena's subvideo is displayed and annotated.** Switching saves the current
+   arena and restores the other arena's frame position. Annotations, undo history,
+   and held labeling controls cannot cross into another arena.
+5. Track progress with **Unlabeled** (gray), **Ongoing** (amber), and **Completed**
+   (green). Completion is an explicit decision, including arenas with no events.
+   Editing a completed arena returns it to Ongoing.
+
+The source's arena project reopens automatically when you reopen the recording.
+You can rename exported arenas using **Manage arena names**. Crop geometry is
+fixed after export to protect existing annotations. Original CSVs belonging to
+the full video stay separate from arena CSVs.
+
+Outputs sit next to the source in a dedicated folder:
+
+```text
+recording_arenas/
+  arenas.json                # Source identity, crop geometry, names, progress
+  arena_1/
+    video.mkv                # Only this arena, every original frame in order
+    video.csv                # Independent annotations, Frames starts at 1
+    arena.json               # Crop and verified encoding metadata
+  arena_2/
+    video.mkv
+    video.csv
+    arena.json
+```
+
+The CSV appears when an arena is first saved or switched away from. Custom arena
+names do not change the stable folder identifiers or break saved annotations.
+Existing exported videos are never overwritten. A source fingerprint detects
+accidental source replacement before a project is reopened.
+
+Cropping uses FFmpeg's `crop` filter and frame passthrough, with no frame-rate
+conversion. Every source and exported crop is decoded to verify exact frame
+counts before publication; fractional nominal frame rates and odd crop dimensions
+are checked too. Output contains video only. The default H.264 ultrafast encoding
+is fast and visually high quality. **Lossless crops** uses lossless YUV encoding;
+conversion between RGB and YUV can still change RGB values slightly. Variable-rate
+movies retain frame count and order; the existing time-based annotation statistics
+use the video's nominal frame rate.
+
+Automatic detection is a proposal tool for straight, dark arena dividers. It does
+not identify animals or guarantee arena boundaries. Manual rectangles and explicit
+grids cover layouts without clear divider evidence. Pixel coordinates can be
+adjusted before export. See the official [FFmpeg crop documentation](https://ffmpeg.org/ffmpeg-filters.html#crop)
+and [frame passthrough options](https://ffmpeg.org/ffmpeg.html).
+
+## Validation and performance
+
+Run the integrity and GUI regression checks with:
+
+```console
+python -m unittest discover -s tests -v
+python -m unittest installer.test_backend -v
+```
+
+The GUI tests run offscreen with isolated settings and generated videos. They
+cover per-arena CSV isolation, frame restoration, statuses, manual drawing,
+failed saves, malformed annotations, unreadable videos, and cached/direct frame
+identity. Installer tests cover fresh installs, repeat-run updates, checksums,
+shortcut targets, failed updates, and preservation of user files. After building,
+`python -m installer.native_smoke` tests the actual packaged setup, repeated
+installation, and the installed application's two-arena annotation workflow.
+
+Reproduce the crop benchmark and visual checks:
+
+```console
+python -m tests.test_arena_backend --benchmark
+python tools/plot_export_benchmark.py
+python tools/preview_arena_workflow.py
+```
+
+Benchmark CSVs, PNG/PDF/SVG figures, and UI previews are written under ignored
+`tests/artifacts/`. Plotting requires Matplotlib in the environment used for that
+optional report. On one Windows workstation, three paired synthetic trials
+(180 frames, 640 x 480, 30000/1001 fps, four crops, including exact validation)
+gave median times of **2.17 seconds shared decoding** versus **4.66 seconds for
+four sequential exports**, a **2.15x** ratio. This describes that workload, not a
+universal performance guarantee. Long movies, storage speed, codec, and image
+dimensions affect the result.
+
 ## Usage
 
 1. Launch the application

annotator_libs/annotation_logic.py

@@ -1,6 +1,7 @@
 import os
 import pandas as pd
 import numpy as np
+import tempfile
 from PySide6.QtWidgets import QMessageBox, QStatusBar
 
 
@@ -131,8 +132,13 @@ def sync_video_csv_with_template(df, template_behaviors, csv_path, parent=None):
     return synced_df
 
 
-def save_annotations_to_csv(video_path, annotations, behaviors, status_bar=None):
-    """Save annotations to CSV file using optimized NumPy operations"""
+def save_annotations_to_csv(video_path, annotations, behaviors, status_bar=None, total_frames=None):
+    """Atomically save one CSV row per frame, using a verified count if supplied.
+
+    Arena exports supply a decoded frame count because container headers can be
+    inaccurate for variable-frame-rate movies. A failed save leaves the prior
+    CSV intact and raises so the arena switch can be cancelled.
+    """
     if not video_path:
         if status_bar:
             status_bar.showMessage("Error: No video loaded. Cannot save annotations.", 3000)
@@ -141,9 +147,9 @@ def save_annotations_to_csv(video_path, annotations, behaviors, status_bar=None)
         return
 
     csv_path = os.path.splitext(video_path)[0] + '.csv'
-    total_frames = get_total_frames_from_video(video_path)
-    if total_frames == 0:
-        return
+    total_frames = get_total_frames_from_video(video_path) if total_frames is None else int(total_frames)
+    if total_frames <= 0:
+        raise ValueError("Cannot save annotations without a valid video frame count.")
 
     # Use NumPy for fast data preparation
     num_behaviors = len(behaviors)
@@ -154,7 +160,7 @@ def save_annotations_to_csv(video_path, annotations, behaviors, status_bar=None)
 
     # Fill array from annotations dict
     for frame, active_list in annotations.items():
-        if frame < total_frames:
+        if 0 <= frame < total_frames:
             if isinstance(active_list, list):
                 for b in active_list:
                     if b in behavior_to_idx:
@@ -168,12 +174,23 @@ def save_annotations_to_csv(video_path, annotations, behaviors, status_bar=None)
         data[b] = arr[:, i]
 
     df = pd.DataFrame(data)
-    df.to_csv(csv_path, index=False)
+    descriptor, temporary = tempfile.mkstemp(prefix='.annotations-', suffix='.tmp',
+                                             dir=os.path.dirname(os.path.abspath(csv_path)))
+    try:
+        with os.fdopen(descriptor, 'w', encoding='utf-8', newline='') as handle:
+            df.to_csv(handle, index=False)
+            handle.flush()
+            os.fsync(handle.fileno())
+        os.replace(temporary, csv_path)
+    finally:
+        if os.path.exists(temporary):
+            os.unlink(temporary)
 
     if status_bar:
         status_bar.showMessage(f"Annotations saved to {csv_path}", 2000)
     else:
         print(f"Annotations saved to {csv_path}")
+    return csv_path
 
 
 def get_total_frames_from_video(video_path):