diff --git a/dashboard/app.py b/dashboard/app.py index 5ce10a7..b466d76 100644 --- a/dashboard/app.py +++ b/dashboard/app.py @@ -113,15 +113,72 @@ def run_simulation(): creationflags=subprocess.CREATE_NEW_PROCESS_GROUP, ) + import re + ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])') + + full_log = [] + log_file_path = None + yield f"data: [INFO] Starting command: {' '.join(cmd)}\n\n" - for line in iter(process.stdout.readline, ""): - if line: - yield f"data: {line.rstrip()}\n\n" + def write_to_log(text): + if log_file_path: + try: + with open(log_file_path, "a", encoding="utf-8") as f: + f.write(text + "\n") + except Exception: + pass + + frame_show_count = 0 + for raw_line in iter(process.stdout.readline, ""): + if raw_line: + # Strip ANSI and clean up trailing newlines + clean_line = ansi_escape.sub('', raw_line).rstrip('\r\n') + + if not clean_line.strip(): + continue + + # ------------------------------------------------------ + # SSE Filtering: Sample high-frequency logs for the UI + # ------------------------------------------------------ + should_yield = True + if "[FRAME " in clean_line: + frame_show_count += 1 + if frame_show_count % 10 != 1: # Show 1st, 11th, 21st, etc. + should_yield = False + + if should_yield: + yield f"data: {clean_line}\n\n" + + # ------------------------------------------------------ + # Log Mirroring: Always write every cleaned line to disk + # ------------------------------------------------------ + if not log_file_path: + full_log.append(clean_line) + if "Saving data to: " in clean_line: + try: + # Extract path: "Saving data to: data\showcase_XXXX ---" + # Note: the log says "--- Recorder initialized. Saving data to: data\showcase_2026... ---" + path_part = clean_line.split("Saving data to: ")[1] + rel_path = path_part.split("---")[0].strip() + + log_file_path = os.path.join(PROJECT_ROOT, rel_path, "console.log") + os.makedirs(os.path.dirname(log_file_path), exist_ok=True) + + # Flush backlog + for prev in full_log: + write_to_log(prev) + except Exception: + pass + else: + # Once file path is known, append all new lines + write_to_log(clean_line) process.stdout.close() return_code = process.wait() - yield f"data: [PROCESS_COMPLETED] Exit Code: {return_code}\n\n" + end_msg = f"[PROCESS_COMPLETED] Exit Code: {return_code}" + yield f"data: {end_msg}\n\n" + write_to_log(end_msg) except Exception as e: yield f"data: [ERROR] Failed to start process: {str(e)}\n\n" diff --git a/dashboard/static/app.js b/dashboard/static/app.js index 611c7d3..9dfdc9b 100644 --- a/dashboard/static/app.js +++ b/dashboard/static/app.js @@ -108,10 +108,12 @@ document.addEventListener('DOMContentLoaded', () => { const line = document.createElement('div'); line.className = `log-line ${type}`; - // Simple heuristic for colored logs + // Advanced heuristic for colored logs if (text.includes('[INFO]')) line.classList.add('info'); if (text.includes('[WARN]')) line.classList.add('warn'); if (text.includes('[ERROR]')) line.classList.add('error'); + if (text.includes('[SUCCESS]')) line.classList.add('success'); + if (text.includes('[FRAME ')) line.classList.add('frame-log'); if (text.includes('Traceback') || text.includes('Exception')) line.classList.add('error'); line.textContent = text; @@ -150,13 +152,23 @@ document.addEventListener('DOMContentLoaded', () => { if (data.startsWith('[PROCESS_COMPLETED]')) { appendLog(`> Simulation Finished. ${data}`, 'info'); setLoadingState(false); + document.getElementById('sim-progress-container').style.display = 'none'; } else { - // Only append if it's not a carriage return override (like tqdm progress bars) - // We do a simple replacement to simulate carriage returns in HTML + // Intercept progress bar logs + if (data.includes('SIMULATING:')) { + const match = data.match(/(\d+)%\|.*\|\s*(\d+)\/(\d+)/); + if (match) { + document.getElementById('sim-progress-container').style.display = 'block'; + document.getElementById('sim-progress-fill').style.width = match[1] + '%'; + document.getElementById('sim-progress-text').textContent = `${match[1]}% (${match[2]} / ${match[3]} frames)`; + continue; // Skip appending this line to the terminal box + } + } + + // Prevent carriage return visual artifacts if (data.includes('\r')) { const parts = data.split('\r'); const lastPart = parts[parts.length - 1]; - // Replace the content of the very last line if it exists const lastLine = terminalOutput.lastElementChild; if (lastLine && !lastLine.textContent.includes('\n')) { lastLine.textContent = lastPart; diff --git a/dashboard/static/style.css b/dashboard/static/style.css index 844d1c0..ad8b09a 100644 --- a/dashboard/static/style.css +++ b/dashboard/static/style.css @@ -445,6 +445,24 @@ input:checked + .slider:before { .log-line.info { color: var(--accent-color); } .log-line.warn { color: var(--warning); } .log-line.error { color: var(--danger); font-weight: bold; } +.log-line.success { color: var(--success); font-weight: 600; } +.log-line.frame-log { color: #818cf8; font-family: 'JetBrains Mono', monospace; font-size: 0.8rem; opacity: 0.9; } + +/* Progress Bar Styling */ +#sim-progress-container { + border-bottom: 1px solid var(--panel-border) !important; + animation: slideDown 0.3s cubic-bezier(0.16, 1, 0.3, 1); +} + +#sim-progress-fill { + transition: width 0.3s cubic-bezier(0.4, 0, 0.2, 1) !important; + background: linear-gradient(90deg, #3b82f6, #2dd4bf) !important; +} + +@keyframes slideDown { + from { opacity: 0; transform: translateY(-10px); } + to { opacity: 1; transform: translateY(0); } +} /* Loader Spinner */ .loader-spinner { diff --git a/dashboard/templates/index.html b/dashboard/templates/index.html index 62c85ef..db53e83 100644 --- a/dashboard/templates/index.html +++ b/dashboard/templates/index.html @@ -103,6 +103,16 @@ + +
> Application initialized. Fetching presets...
diff --git a/intel/context.md b/intel/context.md index 06ec2d2..8dc507f 100644 --- a/intel/context.md +++ b/intel/context.md @@ -27,8 +27,10 @@ Fox/ ├── dashboard.bat ← One-click launcher for the GUI orchestrator (Flask) ├── run.bat ← One-click launcher (activates carla312 conda env) ├── config.py ← All tuneable constants (FPS, sensor params, scenario defaults) -├── data_to_mcap.py ← Converts recorded dataset folders → .mcap files -├── data_inspector.py ← Utility to inspect / debug recorded datasets +│ +├── scripts/ +│ ├── data_to_mcap.py ← Converts recorded dataset folders → .mcap files +│ └── data_inspector.py ← Utility to inspect / debug recorded datasets │ ├── dashboard/ ← Flask backend and web frontend (GUI) │ ├── app.py ← Web server bridging API requests to run.bat @@ -233,7 +235,7 @@ All scenarios now encapsulate their own defaults and support CLI injection via ` --- -### `data_to_mcap.py` — MCAP Converter +### `scripts/data_to_mcap.py` — MCAP Converter Scans `data/` for subfolders containing `frames.jsonl` and converts each to a `.mcap` file. Output is written as `data//.mcap` (skips if already exists). @@ -251,7 +253,7 @@ Output is written as `data//.mcap` (skips if already exists). **Run:** ``` -python data_to_mcap.py +python scripts/data_to_mcap.py ``` Processes all unprocessed session folders in `data/` automatically. @@ -264,7 +266,7 @@ Processes all unprocessed session folders in `data/` automatically. 2. run.bat braking → data/braking_/ (PNG + NPY + JSONL) 3. run.bat cutin → data/cutin_/ 4. run.bat obstacle → data/obstacle_/ -5. python data_to_mcap.py → data/*/.mcap +5. python scripts/data_to_mcap.py → data/*/.mcap 6. Open .mcap in Foxglove Studio ``` diff --git a/intel/showcase.md b/intel/showcase.md index b17eeff..fb91253 100644 --- a/intel/showcase.md +++ b/intel/showcase.md @@ -68,7 +68,7 @@ One of the biggest challenges in CARLA is consistent timing. Rolling friction, g We turned a 2-step manual process into a single-step autonomous pipeline: 1. `main.py` runs the simulation. 2. `recorder.py` captures frames (Async) + generates `.mp4` previews. -3. `main.py` cleanup callback triggers `data_to_mcap.py` automatically. +3. `main.py` cleanup callback triggers `scripts/data_to_mcap.py` automatically. **Run Command**: ```powershell diff --git a/scenarios/braking.py b/scenarios/braking.py index 33094cb..fff9597 100644 --- a/scenarios/braking.py +++ b/scenarios/braking.py @@ -14,6 +14,7 @@ import os sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import math import carla from scenarios.base import ScenarioBase @@ -110,6 +111,20 @@ class BrakingScenario(ScenarioBase): self._braked = True print(f"[{self.name}] EMERGENCY BRAKE applied at frame {frame_count}.") + # Verbose Logging (Full frequency for 1:1 log mirroring) + e_vel = ego_vehicle.get_velocity() + e_speed = 3.6 * math.sqrt(e_vel.x**2 + e_vel.y**2 + e_vel.z**2) + + l_dist = -1.0 + if self._lead_vehicle and self._lead_vehicle.is_alive: + l_dist = ego_vehicle.get_location().distance(self._lead_vehicle.get_location()) + + msg = f"[FRAME {frame_count:03d}] EGO (spd={e_speed:.1f}kph) | LEAD DIST: {l_dist:.1f}m {'(BRAKING)' if self._braked else ''}" + if pbar: + pbar.write(msg) + else: + print(msg) + def cleanup(self) -> None: self._destroy_actors() print(f"[{self.name}] Cleanup complete.") diff --git a/scenarios/cutin.py b/scenarios/cutin.py index 414246c..0035ed7 100644 --- a/scenarios/cutin.py +++ b/scenarios/cutin.py @@ -13,6 +13,7 @@ import os sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import math from scenarios.base import ScenarioBase @@ -102,6 +103,20 @@ class CutInScenario(ScenarioBase): self._cut_triggered = True print(f"[{self.name}] Lane change triggered at frame {frame_count}.") + # Verbose Logging (Full frequency for 1:1 log mirroring) + e_vel = ego_vehicle.get_velocity() + e_speed = 3.6 * math.sqrt(e_vel.x**2 + e_vel.y**2 + e_vel.z**2) + + n_dist = -1.0 + if self._npc and self._npc.is_alive: + n_dist = ego_vehicle.get_location().distance(self._npc.get_location()) + + msg = f"[FRAME {frame_count:03d}] EGO (spd={e_speed:.1f}kph) | NPC DIST: {n_dist:.1f}m {'(CUT-IN)' if self._cut_triggered else ''}" + if pbar: + pbar.write(msg) + else: + print(msg) + def cleanup(self) -> None: self._destroy_actors() print(f"[{self.name}] Cleanup complete.") diff --git a/scenarios/showcase.py b/scenarios/showcase.py index 661add8..8e4cbfd 100644 --- a/scenarios/showcase.py +++ b/scenarios/showcase.py @@ -122,15 +122,14 @@ class ShowcaseScenario(ScenarioBase): self._npc.set_target_velocity(fwd * 9.72) self._npc.apply_control(carla.VehicleControl(throttle=1.0, steer=n_steer)) - # Verbose Logging - if frame % 10 == 0: - e_vel = ego_vehicle.get_velocity() - e_speed = 3.6 * math.sqrt(e_vel.x**2 + e_vel.y**2 + e_vel.z**2) - msg = f"[FRAME {frame:03d}] EGO (spd={e_speed:.1f}kph) target={'MID' if not self._ego_reached_mid else 'P3'} | NPC target={'MID' if not self._npc_reached_mid else 'P3'}" - if pbar: - pbar.write(msg) - else: - print(msg) + # Verbose Logging (Full frequency for 1:1 log mirroring) + e_vel = ego_vehicle.get_velocity() + e_speed = 3.6 * math.sqrt(e_vel.x**2 + e_vel.y**2 + e_vel.z**2) + msg = f"[FRAME {frame:03d}] EGO (spd={e_speed:.1f}kph) target={'MID' if not self._ego_reached_mid else 'P3'} | NPC target={'MID' if not self._npc_reached_mid else 'P3'}" + if pbar: + pbar.write(msg) + else: + print(msg) def cleanup(self): self._destroy_actors() diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..77245fb --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1 @@ +# Make the scripts directory an importable Python package diff --git a/data_inspector.py b/scripts/data_inspector.py similarity index 95% rename from data_inspector.py rename to scripts/data_inspector.py index 87c18eb..deb5fe4 100644 --- a/data_inspector.py +++ b/scripts/data_inspector.py @@ -3,7 +3,8 @@ import json import numpy as np import matplotlib.pyplot as plt -DATA_PATH = "data" +PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +DATA_PATH = os.path.join(PROJECT_ROOT, "data") # ---------------- LOAD FRAME METADATA ---------------- diff --git a/data_to_mcap.py b/scripts/data_to_mcap.py similarity index 98% rename from data_to_mcap.py rename to scripts/data_to_mcap.py index 4ade4d6..4b736fd 100644 --- a/data_to_mcap.py +++ b/scripts/data_to_mcap.py @@ -197,7 +197,8 @@ def convert_folder(folder_path): print(f" Done! MCAP saved: {output_path} ({os.path.getsize(output_path)/1024/1024:.2f} MB)", flush=True) def main(): - root_data = "data" + PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + root_data = os.path.join(PROJECT_ROOT, "data") if not os.path.exists(root_data): print(f"Error: {root_data} directory not found.") return diff --git a/src/main.py b/src/main.py index b78f01d..cc8590d 100644 --- a/src/main.py +++ b/src/main.py @@ -22,7 +22,7 @@ sys.path.append(os.path.dirname(os.path.dirname(__file__))) import config from scenario_loader import load_scenario, list_scenarios -from data_to_mcap import convert_folder +from scripts.data_to_mcap import convert_folder # -----------------------------------------------------------------------