diff --git a/dashboard/app.py b/dashboard/app.py
index 5ce10a7..b466d76 100644
--- a/dashboard/app.py
+++ b/dashboard/app.py
@@ -113,15 +113,72 @@ def run_simulation():
creationflags=subprocess.CREATE_NEW_PROCESS_GROUP,
)
+ import re
+ ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
+
+ full_log = []
+ log_file_path = None
+
yield f"data: [INFO] Starting command: {' '.join(cmd)}\n\n"
- for line in iter(process.stdout.readline, ""):
- if line:
- yield f"data: {line.rstrip()}\n\n"
+ def write_to_log(text):
+ if log_file_path:
+ try:
+ with open(log_file_path, "a", encoding="utf-8") as f:
+ f.write(text + "\n")
+ except Exception:
+ pass
+
+ frame_show_count = 0
+ for raw_line in iter(process.stdout.readline, ""):
+ if raw_line:
+ # Strip ANSI and clean up trailing newlines
+ clean_line = ansi_escape.sub('', raw_line).rstrip('\r\n')
+
+ if not clean_line.strip():
+ continue
+
+ # ------------------------------------------------------
+ # SSE Filtering: Sample high-frequency logs for the UI
+ # ------------------------------------------------------
+ should_yield = True
+ if "[FRAME " in clean_line:
+ frame_show_count += 1
+ if frame_show_count % 10 != 1: # Show 1st, 11th, 21st, etc.
+ should_yield = False
+
+ if should_yield:
+ yield f"data: {clean_line}\n\n"
+
+ # ------------------------------------------------------
+ # Log Mirroring: Always write every cleaned line to disk
+ # ------------------------------------------------------
+ if not log_file_path:
+ full_log.append(clean_line)
+ if "Saving data to: " in clean_line:
+ try:
+ # Extract path: "Saving data to: data\showcase_XXXX ---"
+ # Note: the log says "--- Recorder initialized. Saving data to: data\showcase_2026... ---"
+ path_part = clean_line.split("Saving data to: ")[1]
+ rel_path = path_part.split("---")[0].strip()
+
+ log_file_path = os.path.join(PROJECT_ROOT, rel_path, "console.log")
+ os.makedirs(os.path.dirname(log_file_path), exist_ok=True)
+
+ # Flush backlog
+ for prev in full_log:
+ write_to_log(prev)
+ except Exception:
+ pass
+ else:
+ # Once file path is known, append all new lines
+ write_to_log(clean_line)
process.stdout.close()
return_code = process.wait()
- yield f"data: [PROCESS_COMPLETED] Exit Code: {return_code}\n\n"
+ end_msg = f"[PROCESS_COMPLETED] Exit Code: {return_code}"
+ yield f"data: {end_msg}\n\n"
+ write_to_log(end_msg)
except Exception as e:
yield f"data: [ERROR] Failed to start process: {str(e)}\n\n"
diff --git a/dashboard/static/app.js b/dashboard/static/app.js
index 611c7d3..9dfdc9b 100644
--- a/dashboard/static/app.js
+++ b/dashboard/static/app.js
@@ -108,10 +108,12 @@ document.addEventListener('DOMContentLoaded', () => {
const line = document.createElement('div');
line.className = `log-line ${type}`;
- // Simple heuristic for colored logs
+ // Advanced heuristic for colored logs
if (text.includes('[INFO]')) line.classList.add('info');
if (text.includes('[WARN]')) line.classList.add('warn');
if (text.includes('[ERROR]')) line.classList.add('error');
+ if (text.includes('[SUCCESS]')) line.classList.add('success');
+ if (text.includes('[FRAME ')) line.classList.add('frame-log');
if (text.includes('Traceback') || text.includes('Exception')) line.classList.add('error');
line.textContent = text;
@@ -150,13 +152,23 @@ document.addEventListener('DOMContentLoaded', () => {
if (data.startsWith('[PROCESS_COMPLETED]')) {
appendLog(`> Simulation Finished. ${data}`, 'info');
setLoadingState(false);
+ document.getElementById('sim-progress-container').style.display = 'none';
} else {
- // Only append if it's not a carriage return override (like tqdm progress bars)
- // We do a simple replacement to simulate carriage returns in HTML
+ // Intercept progress bar logs
+ if (data.includes('SIMULATING:')) {
+ const match = data.match(/(\d+)%\|.*\|\s*(\d+)\/(\d+)/);
+ if (match) {
+ document.getElementById('sim-progress-container').style.display = 'block';
+ document.getElementById('sim-progress-fill').style.width = match[1] + '%';
+ document.getElementById('sim-progress-text').textContent = `${match[1]}% (${match[2]} / ${match[3]} frames)`;
+ continue; // Skip appending this line to the terminal box
+ }
+ }
+
+ // Prevent carriage return visual artifacts
if (data.includes('\r')) {
const parts = data.split('\r');
const lastPart = parts[parts.length - 1];
- // Replace the content of the very last line if it exists
const lastLine = terminalOutput.lastElementChild;
if (lastLine && !lastLine.textContent.includes('\n')) {
lastLine.textContent = lastPart;
diff --git a/dashboard/static/style.css b/dashboard/static/style.css
index 844d1c0..ad8b09a 100644
--- a/dashboard/static/style.css
+++ b/dashboard/static/style.css
@@ -445,6 +445,24 @@ input:checked + .slider:before {
.log-line.info { color: var(--accent-color); }
.log-line.warn { color: var(--warning); }
.log-line.error { color: var(--danger); font-weight: bold; }
+.log-line.success { color: var(--success); font-weight: 600; }
+.log-line.frame-log { color: #818cf8; font-family: 'JetBrains Mono', monospace; font-size: 0.8rem; opacity: 0.9; }
+
+/* Progress Bar Styling */
+#sim-progress-container {
+ border-bottom: 1px solid var(--panel-border) !important;
+ animation: slideDown 0.3s cubic-bezier(0.16, 1, 0.3, 1);
+}
+
+#sim-progress-fill {
+ transition: width 0.3s cubic-bezier(0.4, 0, 0.2, 1) !important;
+ background: linear-gradient(90deg, #3b82f6, #2dd4bf) !important;
+}
+
+@keyframes slideDown {
+ from { opacity: 0; transform: translateY(-10px); }
+ to { opacity: 1; transform: translateY(0); }
+}
/* Loader Spinner */
.loader-spinner {
diff --git a/dashboard/templates/index.html b/dashboard/templates/index.html
index 62c85ef..db53e83 100644
--- a/dashboard/templates/index.html
+++ b/dashboard/templates/index.html
@@ -103,6 +103,16 @@
+
+
+
+ SIMULATION STATUS
+ 0% (0 / 0 frames)
+
+
+
> Application initialized. Fetching presets...
diff --git a/intel/context.md b/intel/context.md
index 06ec2d2..8dc507f 100644
--- a/intel/context.md
+++ b/intel/context.md
@@ -27,8 +27,10 @@ Fox/
├── dashboard.bat ← One-click launcher for the GUI orchestrator (Flask)
├── run.bat ← One-click launcher (activates carla312 conda env)
├── config.py ← All tuneable constants (FPS, sensor params, scenario defaults)
-├── data_to_mcap.py ← Converts recorded dataset folders → .mcap files
-├── data_inspector.py ← Utility to inspect / debug recorded datasets
+│
+├── scripts/
+│ ├── data_to_mcap.py ← Converts recorded dataset folders → .mcap files
+│ └── data_inspector.py ← Utility to inspect / debug recorded datasets
│
├── dashboard/ ← Flask backend and web frontend (GUI)
│ ├── app.py ← Web server bridging API requests to run.bat
@@ -233,7 +235,7 @@ All scenarios now encapsulate their own defaults and support CLI injection via `
---
-### `data_to_mcap.py` — MCAP Converter
+### `scripts/data_to_mcap.py` — MCAP Converter
Scans `data/` for subfolders containing `frames.jsonl` and converts each to a `.mcap` file.
Output is written as `data//.mcap` (skips if already exists).
@@ -251,7 +253,7 @@ Output is written as `data//.mcap` (skips if already exists).
**Run:**
```
-python data_to_mcap.py
+python scripts/data_to_mcap.py
```
Processes all unprocessed session folders in `data/` automatically.
@@ -264,7 +266,7 @@ Processes all unprocessed session folders in `data/` automatically.
2. run.bat braking → data/braking_/ (PNG + NPY + JSONL)
3. run.bat cutin → data/cutin_/
4. run.bat obstacle → data/obstacle_/
-5. python data_to_mcap.py → data/*/.mcap
+5. python scripts/data_to_mcap.py → data/*/.mcap
6. Open .mcap in Foxglove Studio
```
diff --git a/intel/showcase.md b/intel/showcase.md
index b17eeff..fb91253 100644
--- a/intel/showcase.md
+++ b/intel/showcase.md
@@ -68,7 +68,7 @@ One of the biggest challenges in CARLA is consistent timing. Rolling friction, g
We turned a 2-step manual process into a single-step autonomous pipeline:
1. `main.py` runs the simulation.
2. `recorder.py` captures frames (Async) + generates `.mp4` previews.
-3. `main.py` cleanup callback triggers `data_to_mcap.py` automatically.
+3. `main.py` cleanup callback triggers `scripts/data_to_mcap.py` automatically.
**Run Command**:
```powershell
diff --git a/scenarios/braking.py b/scenarios/braking.py
index 33094cb..fff9597 100644
--- a/scenarios/braking.py
+++ b/scenarios/braking.py
@@ -14,6 +14,7 @@ import os
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+import math
import carla
from scenarios.base import ScenarioBase
@@ -110,6 +111,20 @@ class BrakingScenario(ScenarioBase):
self._braked = True
print(f"[{self.name}] EMERGENCY BRAKE applied at frame {frame_count}.")
+ # Verbose Logging (Full frequency for 1:1 log mirroring)
+ e_vel = ego_vehicle.get_velocity()
+ e_speed = 3.6 * math.sqrt(e_vel.x**2 + e_vel.y**2 + e_vel.z**2)
+
+ l_dist = -1.0
+ if self._lead_vehicle and self._lead_vehicle.is_alive:
+ l_dist = ego_vehicle.get_location().distance(self._lead_vehicle.get_location())
+
+ msg = f"[FRAME {frame_count:03d}] EGO (spd={e_speed:.1f}kph) | LEAD DIST: {l_dist:.1f}m {'(BRAKING)' if self._braked else ''}"
+ if pbar:
+ pbar.write(msg)
+ else:
+ print(msg)
+
def cleanup(self) -> None:
self._destroy_actors()
print(f"[{self.name}] Cleanup complete.")
diff --git a/scenarios/cutin.py b/scenarios/cutin.py
index 414246c..0035ed7 100644
--- a/scenarios/cutin.py
+++ b/scenarios/cutin.py
@@ -13,6 +13,7 @@ import os
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+import math
from scenarios.base import ScenarioBase
@@ -102,6 +103,20 @@ class CutInScenario(ScenarioBase):
self._cut_triggered = True
print(f"[{self.name}] Lane change triggered at frame {frame_count}.")
+ # Verbose Logging (Full frequency for 1:1 log mirroring)
+ e_vel = ego_vehicle.get_velocity()
+ e_speed = 3.6 * math.sqrt(e_vel.x**2 + e_vel.y**2 + e_vel.z**2)
+
+ n_dist = -1.0
+ if self._npc and self._npc.is_alive:
+ n_dist = ego_vehicle.get_location().distance(self._npc.get_location())
+
+ msg = f"[FRAME {frame_count:03d}] EGO (spd={e_speed:.1f}kph) | NPC DIST: {n_dist:.1f}m {'(CUT-IN)' if self._cut_triggered else ''}"
+ if pbar:
+ pbar.write(msg)
+ else:
+ print(msg)
+
def cleanup(self) -> None:
self._destroy_actors()
print(f"[{self.name}] Cleanup complete.")
diff --git a/scenarios/showcase.py b/scenarios/showcase.py
index 661add8..8e4cbfd 100644
--- a/scenarios/showcase.py
+++ b/scenarios/showcase.py
@@ -122,15 +122,14 @@ class ShowcaseScenario(ScenarioBase):
self._npc.set_target_velocity(fwd * 9.72)
self._npc.apply_control(carla.VehicleControl(throttle=1.0, steer=n_steer))
- # Verbose Logging
- if frame % 10 == 0:
- e_vel = ego_vehicle.get_velocity()
- e_speed = 3.6 * math.sqrt(e_vel.x**2 + e_vel.y**2 + e_vel.z**2)
- msg = f"[FRAME {frame:03d}] EGO (spd={e_speed:.1f}kph) target={'MID' if not self._ego_reached_mid else 'P3'} | NPC target={'MID' if not self._npc_reached_mid else 'P3'}"
- if pbar:
- pbar.write(msg)
- else:
- print(msg)
+ # Verbose Logging (Full frequency for 1:1 log mirroring)
+ e_vel = ego_vehicle.get_velocity()
+ e_speed = 3.6 * math.sqrt(e_vel.x**2 + e_vel.y**2 + e_vel.z**2)
+ msg = f"[FRAME {frame:03d}] EGO (spd={e_speed:.1f}kph) target={'MID' if not self._ego_reached_mid else 'P3'} | NPC target={'MID' if not self._npc_reached_mid else 'P3'}"
+ if pbar:
+ pbar.write(msg)
+ else:
+ print(msg)
def cleanup(self):
self._destroy_actors()
diff --git a/scripts/__init__.py b/scripts/__init__.py
new file mode 100644
index 0000000..77245fb
--- /dev/null
+++ b/scripts/__init__.py
@@ -0,0 +1 @@
+# Make the scripts directory an importable Python package
diff --git a/data_inspector.py b/scripts/data_inspector.py
similarity index 95%
rename from data_inspector.py
rename to scripts/data_inspector.py
index 87c18eb..deb5fe4 100644
--- a/data_inspector.py
+++ b/scripts/data_inspector.py
@@ -3,7 +3,8 @@ import json
import numpy as np
import matplotlib.pyplot as plt
-DATA_PATH = "data"
+PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+DATA_PATH = os.path.join(PROJECT_ROOT, "data")
# ---------------- LOAD FRAME METADATA ----------------
diff --git a/data_to_mcap.py b/scripts/data_to_mcap.py
similarity index 98%
rename from data_to_mcap.py
rename to scripts/data_to_mcap.py
index 4ade4d6..4b736fd 100644
--- a/data_to_mcap.py
+++ b/scripts/data_to_mcap.py
@@ -197,7 +197,8 @@ def convert_folder(folder_path):
print(f" Done! MCAP saved: {output_path} ({os.path.getsize(output_path)/1024/1024:.2f} MB)", flush=True)
def main():
- root_data = "data"
+ PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+ root_data = os.path.join(PROJECT_ROOT, "data")
if not os.path.exists(root_data):
print(f"Error: {root_data} directory not found.")
return
diff --git a/src/main.py b/src/main.py
index b78f01d..cc8590d 100644
--- a/src/main.py
+++ b/src/main.py
@@ -22,7 +22,7 @@ sys.path.append(os.path.dirname(os.path.dirname(__file__)))
import config
from scenario_loader import load_scenario, list_scenarios
-from data_to_mcap import convert_folder
+from scripts.data_to_mcap import convert_folder
# -----------------------------------------------------------------------