Browse Source

feat: modernize simulation pipeline with centralized scripts and high-frequency UI telemetry

1843_integration
RUSHIL AMBARISH KADU 2 months ago
parent
commit
201e663327
  1. 65
      dashboard/app.py
  2. 20
      dashboard/static/app.js
  3. 18
      dashboard/static/style.css
  4. 10
      dashboard/templates/index.html
  5. 12
      intel/context.md
  6. 2
      intel/showcase.md
  7. 15
      scenarios/braking.py
  8. 15
      scenarios/cutin.py
  9. 3
      scenarios/showcase.py
  10. 1
      scripts/__init__.py
  11. 3
      scripts/data_inspector.py
  12. 3
      scripts/data_to_mcap.py
  13. 2
      src/main.py

65
dashboard/app.py

@ -113,15 +113,72 @@ def run_simulation():
creationflags=subprocess.CREATE_NEW_PROCESS_GROUP,
)
import re
ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
full_log = []
log_file_path = None
yield f"data: [INFO] Starting command: {' '.join(cmd)}\n\n"
for line in iter(process.stdout.readline, ""):
if line:
yield f"data: {line.rstrip()}\n\n"
def write_to_log(text):
if log_file_path:
try:
with open(log_file_path, "a", encoding="utf-8") as f:
f.write(text + "\n")
except Exception:
pass
frame_show_count = 0
for raw_line in iter(process.stdout.readline, ""):
if raw_line:
# Strip ANSI and clean up trailing newlines
clean_line = ansi_escape.sub('', raw_line).rstrip('\r\n')
if not clean_line.strip():
continue
# ------------------------------------------------------
# SSE Filtering: Sample high-frequency logs for the UI
# ------------------------------------------------------
should_yield = True
if "[FRAME " in clean_line:
frame_show_count += 1
if frame_show_count % 10 != 1: # Show 1st, 11th, 21st, etc.
should_yield = False
if should_yield:
yield f"data: {clean_line}\n\n"
# ------------------------------------------------------
# Log Mirroring: Always write every cleaned line to disk
# ------------------------------------------------------
if not log_file_path:
full_log.append(clean_line)
if "Saving data to: " in clean_line:
try:
# Extract path: "Saving data to: data\showcase_XXXX ---"
# Note: the log says "--- Recorder initialized. Saving data to: data\showcase_2026... ---"
path_part = clean_line.split("Saving data to: ")[1]
rel_path = path_part.split("---")[0].strip()
log_file_path = os.path.join(PROJECT_ROOT, rel_path, "console.log")
os.makedirs(os.path.dirname(log_file_path), exist_ok=True)
# Flush backlog
for prev in full_log:
write_to_log(prev)
except Exception:
pass
else:
# Once file path is known, append all new lines
write_to_log(clean_line)
process.stdout.close()
return_code = process.wait()
yield f"data: [PROCESS_COMPLETED] Exit Code: {return_code}\n\n"
end_msg = f"[PROCESS_COMPLETED] Exit Code: {return_code}"
yield f"data: {end_msg}\n\n"
write_to_log(end_msg)
except Exception as e:
yield f"data: [ERROR] Failed to start process: {str(e)}\n\n"

20
dashboard/static/app.js

@ -108,10 +108,12 @@ document.addEventListener('DOMContentLoaded', () => {
const line = document.createElement('div');
line.className = `log-line ${type}`;
// Simple heuristic for colored logs
// Advanced heuristic for colored logs
if (text.includes('[INFO]')) line.classList.add('info');
if (text.includes('[WARN]')) line.classList.add('warn');
if (text.includes('[ERROR]')) line.classList.add('error');
if (text.includes('[SUCCESS]')) line.classList.add('success');
if (text.includes('[FRAME ')) line.classList.add('frame-log');
if (text.includes('Traceback') || text.includes('Exception')) line.classList.add('error');
line.textContent = text;
@ -150,13 +152,23 @@ document.addEventListener('DOMContentLoaded', () => {
if (data.startsWith('[PROCESS_COMPLETED]')) {
appendLog(`> Simulation Finished. ${data}`, 'info');
setLoadingState(false);
document.getElementById('sim-progress-container').style.display = 'none';
} else {
// Only append if it's not a carriage return override (like tqdm progress bars)
// We do a simple replacement to simulate carriage returns in HTML
// Intercept progress bar logs
if (data.includes('SIMULATING:')) {
const match = data.match(/(\d+)%\|.*\|\s*(\d+)\/(\d+)/);
if (match) {
document.getElementById('sim-progress-container').style.display = 'block';
document.getElementById('sim-progress-fill').style.width = match[1] + '%';
document.getElementById('sim-progress-text').textContent = `${match[1]}% (${match[2]} / ${match[3]} frames)`;
continue; // Skip appending this line to the terminal box
}
}
// Prevent carriage return visual artifacts
if (data.includes('\r')) {
const parts = data.split('\r');
const lastPart = parts[parts.length - 1];
// Replace the content of the very last line if it exists
const lastLine = terminalOutput.lastElementChild;
if (lastLine && !lastLine.textContent.includes('\n')) {
lastLine.textContent = lastPart;

18
dashboard/static/style.css

@ -445,6 +445,24 @@ input:checked + .slider:before {
.log-line.info { color: var(--accent-color); }
.log-line.warn { color: var(--warning); }
.log-line.error { color: var(--danger); font-weight: bold; }
.log-line.success { color: var(--success); font-weight: 600; }
.log-line.frame-log { color: #818cf8; font-family: 'JetBrains Mono', monospace; font-size: 0.8rem; opacity: 0.9; }
/* Progress Bar Styling */
#sim-progress-container {
border-bottom: 1px solid var(--panel-border) !important;
animation: slideDown 0.3s cubic-bezier(0.16, 1, 0.3, 1);
}
#sim-progress-fill {
transition: width 0.3s cubic-bezier(0.4, 0, 0.2, 1) !important;
background: linear-gradient(90deg, #3b82f6, #2dd4bf) !important;
}
@keyframes slideDown {
from { opacity: 0; transform: translateY(-10px); }
to { opacity: 1; transform: translateY(0); }
}
/* Loader Spinner */
.loader-spinner {

10
dashboard/templates/index.html

@ -103,6 +103,16 @@
</button>
</div>
</div>
<!-- Interactive Progress Bar -->
<div id="sim-progress-container" style="display: none; padding: 12px 15px; border-bottom: 1px solid rgba(255,255,255,0.05); background: rgba(0,0,0,0.15);">
<div style="display: flex; justify-content: space-between; margin-bottom: 6px; font-family: 'JetBrains Mono', monospace; font-size: 0.8rem; color: #a0aec0; letter-spacing: 0.5px;">
<span>SIMULATION STATUS</span>
<span id="sim-progress-text">0% (0 / 0 frames)</span>
</div>
<div style="width: 100%; background: #1a202c; border-radius: 4px; overflow: hidden; height: 10px; box-shadow: inset 0 1px 3px rgba(0,0,0,0.5);">
<div id="sim-progress-fill" style="width: 0%; height: 100%; background: linear-gradient(90deg, #3b82f6, #60a5fa); box-shadow: 0 0 10px rgba(96,165,250,0.5); transition: width 0.15s ease-out;"></div>
</div>
</div>
<div class="terminal-body" id="terminal-output">
<div class="log-line system">> Application initialized. Fetching presets...</div>
</div>

12
intel/context.md

@ -27,8 +27,10 @@ Fox/
├── dashboard.bat ← One-click launcher for the GUI orchestrator (Flask)
├── run.bat ← One-click launcher (activates carla312 conda env)
├── config.py ← All tuneable constants (FPS, sensor params, scenario defaults)
├── data_to_mcap.py ← Converts recorded dataset folders → .mcap files
├── data_inspector.py ← Utility to inspect / debug recorded datasets
├── scripts/
│ ├── data_to_mcap.py ← Converts recorded dataset folders → .mcap files
│ └── data_inspector.py ← Utility to inspect / debug recorded datasets
├── dashboard/ ← Flask backend and web frontend (GUI)
│ ├── app.py ← Web server bridging API requests to run.bat
@ -233,7 +235,7 @@ All scenarios now encapsulate their own defaults and support CLI injection via `
---
### `data_to_mcap.py` — MCAP Converter
### `scripts/data_to_mcap.py` — MCAP Converter
Scans `data/` for subfolders containing `frames.jsonl` and converts each to a `.mcap` file.
Output is written as `data/<session>/<session>.mcap` (skips if already exists).
@ -251,7 +253,7 @@ Output is written as `data/<session>/<session>.mcap` (skips if already exists).
**Run:**
```
python data_to_mcap.py
python scripts/data_to_mcap.py
```
Processes all unprocessed session folders in `data/` automatically.
@ -264,7 +266,7 @@ Processes all unprocessed session folders in `data/` automatically.
2. run.bat braking → data/braking_<ts>/ (PNG + NPY + JSONL)
3. run.bat cutin → data/cutin_<ts>/
4. run.bat obstacle → data/obstacle_<ts>/
5. python data_to_mcap.py → data/*/<session>.mcap
5. python scripts/data_to_mcap.py → data/*/<session>.mcap
6. Open .mcap in Foxglove Studio
```

2
intel/showcase.md

@ -68,7 +68,7 @@ One of the biggest challenges in CARLA is consistent timing. Rolling friction, g
We turned a 2-step manual process into a single-step autonomous pipeline:
1. `main.py` runs the simulation.
2. `recorder.py` captures frames (Async) + generates `.mp4` previews.
3. `main.py` cleanup callback triggers `data_to_mcap.py` automatically.
3. `main.py` cleanup callback triggers `scripts/data_to_mcap.py` automatically.
**Run Command**:
```powershell

15
scenarios/braking.py

@ -14,6 +14,7 @@ import os
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import math
import carla
from scenarios.base import ScenarioBase
@ -110,6 +111,20 @@ class BrakingScenario(ScenarioBase):
self._braked = True
print(f"[{self.name}] EMERGENCY BRAKE applied at frame {frame_count}.")
# Verbose Logging (Full frequency for 1:1 log mirroring)
e_vel = ego_vehicle.get_velocity()
e_speed = 3.6 * math.sqrt(e_vel.x**2 + e_vel.y**2 + e_vel.z**2)
l_dist = -1.0
if self._lead_vehicle and self._lead_vehicle.is_alive:
l_dist = ego_vehicle.get_location().distance(self._lead_vehicle.get_location())
msg = f"[FRAME {frame_count:03d}] EGO (spd={e_speed:.1f}kph) | LEAD DIST: {l_dist:.1f}m {'(BRAKING)' if self._braked else ''}"
if pbar:
pbar.write(msg)
else:
print(msg)
def cleanup(self) -> None:
self._destroy_actors()
print(f"[{self.name}] Cleanup complete.")

15
scenarios/cutin.py

@ -13,6 +13,7 @@ import os
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import math
from scenarios.base import ScenarioBase
@ -102,6 +103,20 @@ class CutInScenario(ScenarioBase):
self._cut_triggered = True
print(f"[{self.name}] Lane change triggered at frame {frame_count}.")
# Verbose Logging (Full frequency for 1:1 log mirroring)
e_vel = ego_vehicle.get_velocity()
e_speed = 3.6 * math.sqrt(e_vel.x**2 + e_vel.y**2 + e_vel.z**2)
n_dist = -1.0
if self._npc and self._npc.is_alive:
n_dist = ego_vehicle.get_location().distance(self._npc.get_location())
msg = f"[FRAME {frame_count:03d}] EGO (spd={e_speed:.1f}kph) | NPC DIST: {n_dist:.1f}m {'(CUT-IN)' if self._cut_triggered else ''}"
if pbar:
pbar.write(msg)
else:
print(msg)
def cleanup(self) -> None:
self._destroy_actors()
print(f"[{self.name}] Cleanup complete.")

3
scenarios/showcase.py

@ -122,8 +122,7 @@ class ShowcaseScenario(ScenarioBase):
self._npc.set_target_velocity(fwd * 9.72)
self._npc.apply_control(carla.VehicleControl(throttle=1.0, steer=n_steer))
# Verbose Logging
if frame % 10 == 0:
# Verbose Logging (Full frequency for 1:1 log mirroring)
e_vel = ego_vehicle.get_velocity()
e_speed = 3.6 * math.sqrt(e_vel.x**2 + e_vel.y**2 + e_vel.z**2)
msg = f"[FRAME {frame:03d}] EGO (spd={e_speed:.1f}kph) target={'MID' if not self._ego_reached_mid else 'P3'} | NPC target={'MID' if not self._npc_reached_mid else 'P3'}"

1
scripts/__init__.py

@ -0,0 +1 @@
# Make the scripts directory an importable Python package

3
data_inspector.py → scripts/data_inspector.py

@ -3,7 +3,8 @@ import json
import numpy as np
import matplotlib.pyplot as plt
DATA_PATH = "data"
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
DATA_PATH = os.path.join(PROJECT_ROOT, "data")
# ---------------- LOAD FRAME METADATA ----------------

3
data_to_mcap.py → scripts/data_to_mcap.py

@ -197,7 +197,8 @@ def convert_folder(folder_path):
print(f" Done! MCAP saved: {output_path} ({os.path.getsize(output_path)/1024/1024:.2f} MB)", flush=True)
def main():
root_data = "data"
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
root_data = os.path.join(PROJECT_ROOT, "data")
if not os.path.exists(root_data):
print(f"Error: {root_data} directory not found.")
return

2
src/main.py

@ -22,7 +22,7 @@ sys.path.append(os.path.dirname(os.path.dirname(__file__)))
import config
from scenario_loader import load_scenario, list_scenarios
from data_to_mcap import convert_folder
from scripts.data_to_mcap import convert_folder
# -----------------------------------------------------------------------

Loading…
Cancel
Save