feat: modernize simulation pipeline with centralized scripts and high-frequency UI telemetry

dashboard/app.py

@@ -113,15 +113,72 @@ def run_simulation():
                 creationflags=subprocess.CREATE_NEW_PROCESS_GROUP,
             )
 
+            import re
+            ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
+            
+            full_log = []
+            log_file_path = None
+
             yield f"data: [INFO] Starting command: {' '.join(cmd)}\n\n"
 
-            for line in iter(process.stdout.readline, ""):
-                if line:
-                    yield f"data: {line.rstrip()}\n\n"
+            def write_to_log(text):
+                if log_file_path:
+                    try:
+                        with open(log_file_path, "a", encoding="utf-8") as f:
+                            f.write(text + "\n")
+                    except Exception:
+                        pass
+
+            frame_show_count = 0
+            for raw_line in iter(process.stdout.readline, ""):
+                if raw_line:
+                    # Strip ANSI and clean up trailing newlines
+                    clean_line = ansi_escape.sub('', raw_line).rstrip('\r\n')
+                    
+                    if not clean_line.strip():
+                        continue
+
+                    # ------------------------------------------------------
+                    # SSE Filtering: Sample high-frequency logs for the UI
+                    # ------------------------------------------------------
+                    should_yield = True
+                    if "[FRAME " in clean_line:
+                        frame_show_count += 1
+                        if frame_show_count % 10 != 1: # Show 1st, 11th, 21st, etc.
+                            should_yield = False
+                    
+                    if should_yield:
+                        yield f"data: {clean_line}\n\n"
+                    
+                    # ------------------------------------------------------
+                    # Log Mirroring: Always write every cleaned line to disk
+                    # ------------------------------------------------------
+                    if not log_file_path:
+                        full_log.append(clean_line)
+                        if "Saving data to: " in clean_line:
+                            try:
+                                # Extract path: "Saving data to: data\showcase_XXXX ---"
+                                # Note: the log says "--- Recorder initialized. Saving data to: data\showcase_2026... ---"
+                                path_part = clean_line.split("Saving data to: ")[1]
+                                rel_path = path_part.split("---")[0].strip()
+                                
+                                log_file_path = os.path.join(PROJECT_ROOT, rel_path, "console.log")
+                                os.makedirs(os.path.dirname(log_file_path), exist_ok=True)
+                                
+                                # Flush backlog
+                                for prev in full_log:
+                                    write_to_log(prev)
+                            except Exception:
+                                pass
+                    else:
+                        # Once file path is known, append all new lines
+                        write_to_log(clean_line)
 
             process.stdout.close()
             return_code = process.wait()
-            yield f"data: [PROCESS_COMPLETED] Exit Code: {return_code}\n\n"
+            end_msg = f"[PROCESS_COMPLETED] Exit Code: {return_code}"
+            yield f"data: {end_msg}\n\n"
+            write_to_log(end_msg)
 
         except Exception as e:
             yield f"data: [ERROR] Failed to start process: {str(e)}\n\n"

dashboard/static/app.js

@@ -108,10 +108,12 @@ document.addEventListener('DOMContentLoaded', () => {
         const line = document.createElement('div');
         line.className = `log-line ${type}`;
         
-        // Simple heuristic for colored logs
+        // Advanced heuristic for colored logs
         if (text.includes('[INFO]')) line.classList.add('info');
         if (text.includes('[WARN]')) line.classList.add('warn');
         if (text.includes('[ERROR]')) line.classList.add('error');
+        if (text.includes('[SUCCESS]')) line.classList.add('success');
+        if (text.includes('[FRAME ')) line.classList.add('frame-log');
         if (text.includes('Traceback') || text.includes('Exception')) line.classList.add('error');
 
         line.textContent = text;
@@ -150,13 +152,23 @@ document.addEventListener('DOMContentLoaded', () => {
                         if (data.startsWith('[PROCESS_COMPLETED]')) {
                             appendLog(`> Simulation Finished. ${data}`, 'info');
                             setLoadingState(false);
+                            document.getElementById('sim-progress-container').style.display = 'none';
                         } else {
-                            // Only append if it's not a carriage return override (like tqdm progress bars)
-                            // We do a simple replacement to simulate carriage returns in HTML
+                            // Intercept progress bar logs
+                            if (data.includes('SIMULATING:')) {
+                                const match = data.match(/(\d+)%\|.*\|\s*(\d+)\/(\d+)/);
+                                if (match) {
+                                    document.getElementById('sim-progress-container').style.display = 'block';
+                                    document.getElementById('sim-progress-fill').style.width = match[1] + '%';
+                                    document.getElementById('sim-progress-text').textContent = `${match[1]}% (${match[2]} / ${match[3]} frames)`;
+                                    continue; // Skip appending this line to the terminal box
+                                }
+                            }
+                            
+                            // Prevent carriage return visual artifacts
                             if (data.includes('\r')) {
                                 const parts = data.split('\r');
                                 const lastPart = parts[parts.length - 1];
-                                // Replace the content of the very last line if it exists
                                 const lastLine = terminalOutput.lastElementChild;
                                 if (lastLine && !lastLine.textContent.includes('\n')) {
                                     lastLine.textContent = lastPart;

dashboard/static/style.css

@@ -445,6 +445,24 @@ input:checked + .slider:before {
 .log-line.info { color: var(--accent-color); }
 .log-line.warn { color: var(--warning); }
 .log-line.error { color: var(--danger); font-weight: bold; }
+.log-line.success { color: var(--success); font-weight: 600; }
+.log-line.frame-log { color: #818cf8; font-family: 'JetBrains Mono', monospace; font-size: 0.8rem; opacity: 0.9; }
+
+/* Progress Bar Styling */
+#sim-progress-container {
+    border-bottom: 1px solid var(--panel-border) !important;
+    animation: slideDown 0.3s cubic-bezier(0.16, 1, 0.3, 1);
+}
+
+#sim-progress-fill {
+    transition: width 0.3s cubic-bezier(0.4, 0, 0.2, 1) !important;
+    background: linear-gradient(90deg, #3b82f6, #2dd4bf) !important;
+}
+
+@keyframes slideDown {
+    from { opacity: 0; transform: translateY(-10px); }
+    to { opacity: 1; transform: translateY(0); }
+}
 
 /* Loader Spinner */
 .loader-spinner {

dashboard/templates/index.html

@@ -103,6 +103,16 @@
                         </button>
                     </div>
                 </div>
+                <!-- Interactive Progress Bar -->
+                <div id="sim-progress-container" style="display: none; padding: 12px 15px; border-bottom: 1px solid rgba(255,255,255,0.05); background: rgba(0,0,0,0.15);">
+                    <div style="display: flex; justify-content: space-between; margin-bottom: 6px; font-family: 'JetBrains Mono', monospace; font-size: 0.8rem; color: #a0aec0; letter-spacing: 0.5px;">
+                        <span>SIMULATION STATUS</span>
+                        <span id="sim-progress-text">0% (0 / 0 frames)</span>
+                    </div>
+                    <div style="width: 100%; background: #1a202c; border-radius: 4px; overflow: hidden; height: 10px; box-shadow: inset 0 1px 3px rgba(0,0,0,0.5);">
+                        <div id="sim-progress-fill" style="width: 0%; height: 100%; background: linear-gradient(90deg, #3b82f6, #60a5fa); box-shadow: 0 0 10px rgba(96,165,250,0.5); transition: width 0.15s ease-out;"></div>
+                    </div>
+                </div>
                 <div class="terminal-body" id="terminal-output">
                     <div class="log-line system">> Application initialized. Fetching presets...</div>
                 </div>

intel/context.md

@@ -27,8 +27,10 @@ Fox/
 ├── dashboard.bat            ← One-click launcher for the GUI orchestrator (Flask)
 ├── run.bat                  ← One-click launcher (activates carla312 conda env)
 ├── config.py                ← All tuneable constants (FPS, sensor params, scenario defaults)
-├── data_to_mcap.py          ← Converts recorded dataset folders → .mcap files
-├── data_inspector.py        ← Utility to inspect / debug recorded datasets
+│
+├── scripts/
+│   ├── data_to_mcap.py      ← Converts recorded dataset folders → .mcap files
+│   └── data_inspector.py    ← Utility to inspect / debug recorded datasets
 │
 ├── dashboard/               ← Flask backend and web frontend (GUI)
 │   ├── app.py               ← Web server bridging API requests to run.bat
@@ -233,7 +235,7 @@ All scenarios now encapsulate their own defaults and support CLI injection via `
 
 ---
 
-### `data_to_mcap.py` — MCAP Converter
+### `scripts/data_to_mcap.py` — MCAP Converter
 Scans `data/` for subfolders containing `frames.jsonl` and converts each to a `.mcap` file.
 Output is written as `data/<session>/<session>.mcap` (skips if already exists).
 
@@ -251,7 +253,7 @@ Output is written as `data/<session>/<session>.mcap` (skips if already exists).
 
 **Run:**
 ```
-python data_to_mcap.py
+python scripts/data_to_mcap.py
 ```
 Processes all unprocessed session folders in `data/` automatically.
 
@@ -264,7 +266,7 @@ Processes all unprocessed session folders in `data/` automatically.
 2. run.bat braking          → data/braking_<ts>/ (PNG + NPY + JSONL)
 3. run.bat cutin            → data/cutin_<ts>/
 4. run.bat obstacle         → data/obstacle_<ts>/
-5. python data_to_mcap.py   → data/*/<session>.mcap
+5. python scripts/data_to_mcap.py   → data/*/<session>.mcap
 6. Open .mcap in Foxglove Studio
 ```
 

intel/showcase.md

@@ -68,7 +68,7 @@ One of the biggest challenges in CARLA is consistent timing. Rolling friction, g
 We turned a 2-step manual process into a single-step autonomous pipeline:
 1. `main.py` runs the simulation.
 2. `recorder.py` captures frames (Async) + generates `.mp4` previews.
-3. `main.py` cleanup callback triggers `data_to_mcap.py` automatically.
+3. `main.py` cleanup callback triggers `scripts/data_to_mcap.py` automatically.
 
 **Run Command**:
 ```powershell

scenarios/braking.py

@@ -14,6 +14,7 @@ import os
 
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
+import math
 import carla
 from scenarios.base import ScenarioBase
 
@@ -110,6 +111,20 @@ class BrakingScenario(ScenarioBase):
                 self._braked = True
                 print(f"[{self.name}] EMERGENCY BRAKE applied at frame {frame_count}.")
 
+        # Verbose Logging (Full frequency for 1:1 log mirroring)
+        e_vel = ego_vehicle.get_velocity()
+        e_speed = 3.6 * math.sqrt(e_vel.x**2 + e_vel.y**2 + e_vel.z**2)
+        
+        l_dist = -1.0
+        if self._lead_vehicle and self._lead_vehicle.is_alive:
+            l_dist = ego_vehicle.get_location().distance(self._lead_vehicle.get_location())
+            
+        msg = f"[FRAME {frame_count:03d}] EGO (spd={e_speed:.1f}kph) | LEAD DIST: {l_dist:.1f}m {'(BRAKING)' if self._braked else ''}"
+        if pbar:
+            pbar.write(msg)
+        else:
+            print(msg)
+
     def cleanup(self) -> None:
         self._destroy_actors()
         print(f"[{self.name}] Cleanup complete.")

scenarios/cutin.py

@@ -13,6 +13,7 @@ import os
 
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
+import math
 from scenarios.base import ScenarioBase
 
 
@@ -102,6 +103,20 @@ class CutInScenario(ScenarioBase):
                 self._cut_triggered = True
                 print(f"[{self.name}] Lane change triggered at frame {frame_count}.")
 
+        # Verbose Logging (Full frequency for 1:1 log mirroring)
+        e_vel = ego_vehicle.get_velocity()
+        e_speed = 3.6 * math.sqrt(e_vel.x**2 + e_vel.y**2 + e_vel.z**2)
+        
+        n_dist = -1.0
+        if self._npc and self._npc.is_alive:
+            n_dist = ego_vehicle.get_location().distance(self._npc.get_location())
+            
+        msg = f"[FRAME {frame_count:03d}] EGO (spd={e_speed:.1f}kph) | NPC DIST: {n_dist:.1f}m {'(CUT-IN)' if self._cut_triggered else ''}"
+        if pbar:
+            pbar.write(msg)
+        else:
+            print(msg)
+
     def cleanup(self) -> None:
         self._destroy_actors()
         print(f"[{self.name}] Cleanup complete.")

scenarios/showcase.py

@@ -122,15 +122,14 @@ class ShowcaseScenario(ScenarioBase):
             self._npc.set_target_velocity(fwd * 9.72)
             self._npc.apply_control(carla.VehicleControl(throttle=1.0, steer=n_steer))
         
-        # Verbose Logging
-        if frame % 10 == 0:
-            e_vel = ego_vehicle.get_velocity()
-            e_speed = 3.6 * math.sqrt(e_vel.x**2 + e_vel.y**2 + e_vel.z**2)
-            msg = f"[FRAME {frame:03d}] EGO (spd={e_speed:.1f}kph) target={'MID' if not self._ego_reached_mid else 'P3'} | NPC target={'MID' if not self._npc_reached_mid else 'P3'}"
-            if pbar:
-                pbar.write(msg)
-            else:
-                print(msg)
+        # Verbose Logging (Full frequency for 1:1 log mirroring)
+        e_vel = ego_vehicle.get_velocity()
+        e_speed = 3.6 * math.sqrt(e_vel.x**2 + e_vel.y**2 + e_vel.z**2)
+        msg = f"[FRAME {frame:03d}] EGO (spd={e_speed:.1f}kph) target={'MID' if not self._ego_reached_mid else 'P3'} | NPC target={'MID' if not self._npc_reached_mid else 'P3'}"
+        if pbar:
+            pbar.write(msg)
+        else:
+            print(msg)
 
     def cleanup(self):
         self._destroy_actors()

scripts/__init__.py

@@ -0,0 +1 @@
+# Make the scripts directory an importable Python package

data_inspector.py → scripts/data_inspector.py

@@ -3,7 +3,8 @@ import json
 import numpy as np
 import matplotlib.pyplot as plt
 
-DATA_PATH = "data"
+PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+DATA_PATH = os.path.join(PROJECT_ROOT, "data")
 
 
 # ---------------- LOAD FRAME METADATA ----------------

data_to_mcap.py → scripts/data_to_mcap.py

@@ -197,7 +197,8 @@ def convert_folder(folder_path):
     print(f"  Done! MCAP saved: {output_path} ({os.path.getsize(output_path)/1024/1024:.2f} MB)", flush=True)
 
 def main():
-    root_data = "data"
+    PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    root_data = os.path.join(PROJECT_ROOT, "data")
     if not os.path.exists(root_data):
         print(f"Error: {root_data} directory not found.")
         return

src/main.py

@@ -22,7 +22,7 @@ sys.path.append(os.path.dirname(os.path.dirname(__file__)))
 
 import config
 from scenario_loader import load_scenario, list_scenarios
-from data_to_mcap import convert_folder
+from scripts.data_to_mcap import convert_folder
 
 
 # -----------------------------------------------------------------------