diff --git a/config.py b/config.py index f0731cb..ee241fb 100644 --- a/config.py +++ b/config.py @@ -26,4 +26,7 @@ MAX_FRAMES = 200 # ----------------------------------------------------------------------- DEFAULT_SCENARIO = "braking" DEFAULT_EGO_MODEL = "vehicle.tesla.model3" -DEFAULT_WEATHER = "ClearNoon" \ No newline at end of file +DEFAULT_WEATHER = "ClearNoon" + +# Simulator Path +CARLA_EXECUTABLE_PATH = r"D:\CARLA\CARLA_0.9.16\CarlaUE4.exe" \ No newline at end of file diff --git a/dashboard/app.py b/dashboard/app.py index b466d76..4f37a7d 100644 --- a/dashboard/app.py +++ b/dashboard/app.py @@ -13,6 +13,8 @@ import os import sys from flask import Flask, render_template, request, jsonify, Response import subprocess +import time +import psutil # ------------------------------------------------------------------ # Path bootstrapping — make the project root importable regardless @@ -36,6 +38,7 @@ app = Flask( static_folder=os.path.join(DASHBOARD_DIR, "static"), ) +active_simulation_process = None # ── Routes ──────────────────────────────────────────────────────── @@ -76,6 +79,95 @@ def get_scenario_params(scenario_name): return jsonify({"success": False, "error": str(e)}) +@app.route("/api/simulator/status", methods=["GET"]) +def simulator_status(): + status = "offline" + is_paused = False + is_running = False + try: + for proc in psutil.process_iter(['name']): + if proc.info['name'] and 'CarlaUE4' in proc.info['name']: + is_running = True + break + except Exception: + pass + + if is_running: + status = "loading" + try: + import carla + client = carla.Client('localhost', 2000) + client.set_timeout(10.0) + client.get_server_version() # Will raise if not ready + status = "ready" + is_paused = client.get_world().get_settings().synchronous_mode + except Exception: + pass + + return jsonify({"status": status, "is_paused": is_paused}) + +@app.route("/api/simulator/idle", methods=["POST"]) +def simulator_idle(): + try: + import carla + client = carla.Client('localhost', 2000) + client.set_timeout(5.0) + world = client.get_world() + settings = world.get_settings() + settings.synchronous_mode = True + world.apply_settings(settings) + return jsonify({"success": True}) + except Exception as e: + return jsonify({"success": False, "error": str(e)}) + +@app.route("/api/simulator/wake", methods=["POST"]) +def simulator_wake(): + try: + import carla + client = carla.Client('localhost', 2000) + client.set_timeout(5.0) + world = client.get_world() + settings = world.get_settings() + settings.synchronous_mode = False + settings.fixed_delta_seconds = None + world.apply_settings(settings) + return jsonify({"success": True}) + except Exception as e: + return jsonify({"success": False, "error": str(e)}) + + +@app.route("/api/simulator/launch", methods=["POST"]) +def simulator_launch(): + try: + exe_path = getattr(config, "CARLA_EXECUTABLE_PATH", r"D:\CARLA\CARLA_0.9.16\CarlaUE4.exe") + if not os.path.exists(exe_path): + return jsonify({"success": False, "error": f"Executable not found at {exe_path}"}) + # Launch detached + subprocess.Popen([exe_path], creationflags=subprocess.CREATE_NEW_PROCESS_GROUP) + return jsonify({"success": True}) + except Exception as e: + return jsonify({"success": False, "error": str(e)}) + + +@app.route("/api/stop", methods=["POST"]) +def stop_simulation(): + global active_simulation_process + + flag_path = os.path.join(PROJECT_ROOT, "tmp", "stop.flag") + os.makedirs(os.path.dirname(flag_path), exist_ok=True) + with open(flag_path, "w") as f: + f.write("stop") + + if active_simulation_process is not None: + try: + active_simulation_process.wait(timeout=5) + except subprocess.TimeoutExpired: + active_simulation_process.terminate() + active_simulation_process = None + + return jsonify({"success": True, "message": "Simulation stopped."}) + + @app.route("/api/run", methods=["POST"]) def run_simulation(): data = request.json or {} @@ -97,6 +189,7 @@ def run_simulation(): cmd.extend(["--params", f'"{params}"']) def generate(): + global active_simulation_process try: # cwd = project root so that run.bat is reachable env = os.environ.copy() @@ -112,6 +205,7 @@ def run_simulation(): env=env, creationflags=subprocess.CREATE_NEW_PROCESS_GROUP, ) + active_simulation_process = process import re ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])') diff --git a/dashboard/static/app.js b/dashboard/static/app.js index 9dfdc9b..09d66ca 100644 --- a/dashboard/static/app.js +++ b/dashboard/static/app.js @@ -8,8 +8,106 @@ document.addEventListener('DOMContentLoaded', () => { const launchSpinner = document.getElementById('launch-spinner'); const terminalOutput = document.getElementById('terminal-output'); const clearTerminalBtn = document.getElementById('clear-terminal'); + + // New Elements + const stopBtn = document.getElementById('stop-btn'); + const simStatusDot = document.getElementById('sim-status-dot'); + const simStatusText = document.getElementById('sim-status-text'); + const launchSimBtn = document.getElementById('launch-sim-btn'); + + // Idle Toggle Elements + const idleToggleContainer = document.getElementById('idle-toggle-container'); + const gpuIdleToggle = document.getElementById('gpu-idle-toggle'); let currentEventSource = null; + let idleTimer = null; + let isSimulatorRunning = false; + let isSimulationActive = false; // user currently running a scenario + + // Auto Idle Tracker + function resetIdleTimer() { + if (idleTimer) clearTimeout(idleTimer); + // If simulation is not active and simulator is ready but NOT paused + if (!isSimulationActive && isSimulatorRunning && gpuIdleToggle && !gpuIdleToggle.checked) { + idleTimer = setTimeout(() => { + console.log("Auto-idling simulator due to 30s inactivity..."); + gpuIdleToggle.checked = true; + gpuIdleToggle.dispatchEvent(new Event('change')); + }, 30000); + } + } + + if (gpuIdleToggle) { + gpuIdleToggle.addEventListener('change', (e) => { + const isIdle = e.target.checked; + const endpoint = isIdle ? '/api/simulator/idle' : '/api/simulator/wake'; + fetch(endpoint, { method: 'POST' }); + if (!isIdle) { + resetIdleTimer(); + } else { + if (idleTimer) clearTimeout(idleTimer); + } + }); + } + + // Simulator Status Polling + function updateSimulatorStatus() { + fetch('/api/simulator/status') + .then(res => res.json()) + .then(data => { + const status = data.status; // offline, loading, ready + simStatusDot.className = 'dot'; + + if (status === 'ready') { + if (!isSimulatorRunning) { + isSimulatorRunning = true; + resetIdleTimer(); // start tracker as soon as it becomes ready + } + if (idleToggleContainer) idleToggleContainer.style.display = 'flex'; + + if (gpuIdleToggle && gpuIdleToggle.checked !== data.is_paused) { + gpuIdleToggle.checked = data.is_paused; + } + + if (data.is_paused) { + simStatusDot.classList.add('yellow'); + simStatusText.textContent = 'Ready (Idle)'; + } else { + simStatusDot.classList.add('green'); + simStatusText.textContent = 'Ready'; + } + launchSimBtn.style.display = 'none'; + } else if (status === 'loading') { + isSimulatorRunning = false; + if (idleToggleContainer) idleToggleContainer.style.display = 'none'; + simStatusDot.classList.add('yellow'); + simStatusText.textContent = 'Loading...'; + launchSimBtn.style.display = 'none'; + } else { + isSimulatorRunning = false; + if (idleToggleContainer) idleToggleContainer.style.display = 'none'; + if (idleTimer) clearTimeout(idleTimer); + + simStatusDot.classList.add('red'); + simStatusText.textContent = 'Offline'; + launchSimBtn.style.display = 'block'; + } + }) + .catch(() => { + simStatusDot.className = 'dot red'; + simStatusText.textContent = 'Error'; + }); + } + + updateSimulatorStatus(); + setInterval(updateSimulatorStatus, 1000); + + // Launch CarlaUE4 action + launchSimBtn.addEventListener('click', () => { + simStatusText.textContent = 'Starting...'; + simStatusDot.className = 'dot yellow'; + fetch('/api/simulator/launch', { method: 'POST' }); + }); // Fetch Initial Config fetch('/api/config') @@ -60,6 +158,15 @@ document.addEventListener('DOMContentLoaded', () => { launchForm.addEventListener('submit', (e) => { e.preventDefault(); + isSimulationActive = true; + if (idleTimer) clearTimeout(idleTimer); + + // Ensure simulator is awake before scenario runs just in case + if (gpuIdleToggle && gpuIdleToggle.checked) { + gpuIdleToggle.checked = false; + fetch('/api/simulator/wake', { method: 'POST' }); + } + // UI Loading State setLoadingState(true); appendLog(`\n> --- NEW SIMULATION RUN ---`, 'info'); @@ -102,6 +209,25 @@ document.addEventListener('DOMContentLoaded', () => { terminalOutput.innerHTML = ''; appendLog('> Terminal cleared.', 'system'); }); + + // Stop Simulation + if (stopBtn) { + stopBtn.addEventListener('click', () => { + stopBtn.disabled = true; + stopBtn.querySelector('.btn-text').textContent = 'Stopping...'; + appendLog('\n> Sending stop signal to simulation. Waiting for graceful exit...', 'warn'); + + fetch('/api/stop', { method: 'POST' }) + .then(res => res.json()) + .then(data => { + console.log('Stop requested:', data.message); + }) + .catch(err => { + appendLog('> Failed to send stop signal: ' + err.message, 'error'); + stopBtn.disabled = false; + }); + }); + } // Utility: Append log to terminal function appendLog(text, type = '') { @@ -117,8 +243,16 @@ document.addEventListener('DOMContentLoaded', () => { if (text.includes('Traceback') || text.includes('Exception')) line.classList.add('error'); line.textContent = text; + + // Smart Scroll Lock + // Check if the user is currently at the bottom of the scroll view + const isAtBottom = terminalOutput.scrollHeight - terminalOutput.scrollTop - terminalOutput.clientHeight < 50; + terminalOutput.appendChild(line); - terminalOutput.scrollTop = terminalOutput.scrollHeight; + + if (isAtBottom) { + terminalOutput.scrollTop = terminalOutput.scrollHeight; + } } // Utility: Post JSON and stream response @@ -153,6 +287,9 @@ document.addEventListener('DOMContentLoaded', () => { appendLog(`> Simulation Finished. ${data}`, 'info'); setLoadingState(false); document.getElementById('sim-progress-container').style.display = 'none'; + + isSimulationActive = false; + resetIdleTimer(); } else { // Intercept progress bar logs if (data.includes('SIMULATING:')) { @@ -189,11 +326,17 @@ document.addEventListener('DOMContentLoaded', () => { function setLoadingState(isLoading) { launchBtn.disabled = isLoading; - launchText.style.display = isLoading ? 'none' : 'block'; - launchSpinner.style.display = isLoading ? 'block' : 'none'; + launchBtn.style.display = isLoading ? 'none' : 'block'; + if (stopBtn) { + stopBtn.style.display = isLoading ? 'block' : 'none'; + } if (!isLoading) { launchText.textContent = "Launch Another Simulation"; + if (stopBtn) { + stopBtn.disabled = false; + stopBtn.querySelector('.btn-text').textContent = 'Stop Simulation'; + } } } diff --git a/dashboard/static/style.css b/dashboard/static/style.css index ad8b09a..b48c0a6 100644 --- a/dashboard/static/style.css +++ b/dashboard/static/style.css @@ -495,3 +495,90 @@ input:checked + .slider:before { 0% { opacity: 0; transform: translateY(20px); } 100% { opacity: 1; transform: translateY(0); } } + +/* Simulator Status Card */ +.status-card { + background: var(--input-bg); + border: 1px solid var(--input-border); + padding: 1rem; + border-radius: 8px; + display: flex; + flex-direction: column; + align-items: center; +} + +.status-indicator { + display: flex; + align-items: center; + gap: 0.5rem; + width: 100%; +} + +.btn-secondary { + background: rgba(255, 255, 255, 0.1); + color: var(--text-primary); +} + +.btn-secondary:hover { + background: rgba(255, 255, 255, 0.2); +} + +.btn-danger { + background: linear-gradient(135deg, #f85149, #b31d28); + color: white; +} + +.btn-danger:hover { + box-shadow: 0 4px 14px rgba(248, 81, 73, 0.4); + transform: translateY(-2px); +} + +.btn-danger:active { + transform: translateY(0); +} + +.btn-danger:disabled { + background: #6e2a2a; + color: #a0aec0; + cursor: not-allowed; + transform: none; + box-shadow: none; +} + +.btn-small { + padding: 0.5rem 1rem; + font-size: 0.85rem; +} + +/* Idle Toggle CSS */ +.idle-toggle-container { + display: flex; + justify-content: space-between; + align-items: center; + width: 100%; + margin-top: 12px; + padding-top: 12px; + border-top: 1px solid rgba(255,255,255,0.05); +} + +.idle-label { + font-size: 0.8rem; + color: var(--text-muted); +} + +.small-switch { + width: 34px; + height: 18px; +} + +.small-switch .slider:before { + height: 12px; + width: 12px; + bottom: 3px; + left: 3px; +} + +.small-switch input:checked + .slider:before { + transform: translateX(16px); +} + diff --git a/dashboard/templates/index.html b/dashboard/templates/index.html index db53e83..5b0b109 100644 --- a/dashboard/templates/index.html +++ b/dashboard/templates/index.html @@ -23,6 +23,25 @@ +
+
SIMULATOR STATUS
+
+
+ + Offline +
+ + + +
+
+
SCENARIOS
@@ -79,11 +98,14 @@
-
- +
diff --git a/gemini.md b/gemini.md index 25a0968..e812e53 100644 --- a/gemini.md +++ b/gemini.md @@ -60,6 +60,8 @@ This document is the consolidated source of truth for AI agents working on the F ## 🖥️ 4. GUI Dashboard Architecture - **Backend:** Flask (`app.py`) running on port 5000. +- **Simulator Lifecycle:** Dashboard polls `/api/simulator/status` to detect if CARLA is Offline/Ready. Provides one-click initialization via the UI. +- **GPU Resource Control:** Incorporates "Idle Mode" (Synchronous Throttling) to drop CARLA GPU usage to ~0% when inactive or while Shenron processing is active. - **Streaming:** Uses **Server-Sent Events (SSE)** to stream `stdout` to the browser. - **Execution:** Spawns `run.bat` as a subprocess. - **Crucial Rule:** Always set `PYTHONUNBUFFERED=1` to prevent log delays in the GUI console. @@ -75,6 +77,7 @@ This document is the consolidated source of truth for AI agents working on the F ### Common Pitfalls - **Stale Physics:** Always `world.tick()` once after spawning the Ego but before starting scenario logic. +- **Graceful Shutdown:** Use the "Stop Simulation" button in the dashboard. It writes a `tmp/stop.flag` which the orchestrator checks to ensure all recorders and data converters (Shenron/MCAP) finalize correctly. - **Progress Bar Clash:** Use `pbar.write()` for logs if a `tqdm` progress bar is active. - **Asynchronous I/O:** Use `ThreadPoolExecutor` for disk writes (images) to prevent simulation frame drops. @@ -88,4 +91,4 @@ This document is the consolidated source of truth for AI agents working on the F --- -*Generated by Antigravity AI | Last Updated: 2026-04-01 | Refer to .cursorrules for start-of-turn instructions.* +*Generated by Antigravity AI | Last Updated: 2026-04-10 | Refer to .cursorrules for start-of-turn instructions.* diff --git a/intel/internal/context.md b/intel/internal/context.md index 08d47b2..5a7e9ee 100644 --- a/intel/internal/context.md +++ b/intel/internal/context.md @@ -58,6 +58,8 @@ Fox/ │ ├── lidar/ ← frame_XXXXXX.npy (shape: [N, 4] — x/y/z/intensity) │ └── frames.jsonl ← One JSON record per frame (metadata + scenario info) │ +├── tmp/ ← Staging area for IPC flags (e.g., stop.flag) +│ └── intel/ ├── radar/ ← [CRITICAL] Physics, Material RCS, and Debug logs │ ├── Shenron_debug.md ← The "Source of Truth" for radar calibration @@ -121,7 +123,7 @@ python src/main.py --list-scenarios 6. `Recorder(scenario_name=scenario.name)` → creates `data/_/` 7. **Settle Physics**: Call `world.tick()` once to synchronize Ego location before setup 8. `scenario.setup(world, ego, traffic_manager)` -9. **Main loop** per frame: `world.tick()` → `sensor_manager.get_data()` → +9. **Main loop** per frame: `world.tick()` → **IPC Stop Check (`tmp/stop.flag`)** → `sensor_manager.get_data()` → `recorder.save(..., extra_meta=scenario.get_scenario_metadata())` → `scenario.step(frame, ego, pbar)` 10. `finally`: `scenario.cleanup()` → `sensor_manager.destroy()` → restore async mode @@ -349,4 +351,4 @@ When working on this repository, prioritize documentation based on your specific --- -*Last updated: 2026-04-06 | Pipeline version: Scenario-Centric Deterministic Architecture* +*Last updated: 2026-04-10 | Pipeline version: Scenario-Centric Deterministic Architecture* diff --git a/intel/scenarios/dashboard.md b/intel/scenarios/dashboard.md index 6ec0c65..6673713 100644 --- a/intel/scenarios/dashboard.md +++ b/intel/scenarios/dashboard.md @@ -62,9 +62,23 @@ When the `/api/run` endpoint is hit: --- -## 4. Extension Guidelines for AI - -When asked to extend the Dashboard, observe the following patterns: -- **Adding new CLI flags:** Update the `POST /api/run` payload parser in `app.py`, extend the `cmd.extend()` builder, and add the corresponding HTML `` in `index.html`. -- **Adding new data streams:** The SSE generator currently pipes raw text. To parse structured data (like active ego speed) for live graphs, consider formatting the intermediate Python prints as JSON and having `app.js` parse the `event.data` payload conditionally. -- **Pathing:** Remember that `app.py` lives in a sub-folder. Operations manipulating the CARLA simulation *must* be rooted up two levels using the `PROJECT_ROOT` pointer established in the file. +## 5. Simulator Lifecycle & GPU Control + +The dashboard now incorporates active management of the `CarlaUE4.exe` process and its hardware utilization. + +### A. Lifecycle Monitoring +- **Polling:** The backend uses `psutil` to detect if the simulator process is active. +- **Initialization:** If offline, a one-click "Initialize CarlaUE4" button allows the user to launch the simulator directly from the browser. +- **Ping Validation:** The dashboard "Ready" state is only achieved once the backend successfully pings the CARLA server (localhost:2000) and receives a valid version response. + +### B. Graceful Stop Protocol (IPC) +Instead of killing the simulation process—which can lead to corrupted radar/lidar data—the "Stop Simulation" button triggers an Inter-Process Communication (IPC) signal: +1. Dashboard writes an empty flag file to `tmp/stop.flag`. +2. The orchestrator (`src/main.py`) checks for this file's existence at the start of every frame tick. +3. If detected, the orchestrator breaks the main loop, allowing all `finally` cleanups, recorder closures, and MCAP conversions to finish elegantly. + +### C. GPU Resource Control (Idle Mode) +To prevent CARLA from consuming 100% GPU while waiting for user interaction or during Shenron radar post-processing: +- **Synchronous Throttling:** The dashboard can toggle CARLA into `synchronous_mode`. +- **Idle State:** Without a `world.tick()` call from the client, Unreal Engine pauses its rendering and physics loop, dropping GPU usage to ~0%. +- **Auto-Idle:** The dashboard incorporates a 30-second inactivity timer that automatically pauses the simulator to save power. diff --git a/src/main.py b/src/main.py index b1217aa..a801caa 100644 --- a/src/main.py +++ b/src/main.py @@ -84,6 +84,17 @@ def parse_args(): def main(): args = parse_args() + + # ------------------------------------------------------------------ + # 0. Clean up stale stop flags + # ------------------------------------------------------------------ + flag_path = os.path.join(os.path.dirname(__file__), "..", "tmp", "stop.flag") + if os.path.exists(flag_path): + try: + os.remove(flag_path) + print("[INFO] Cleaned up stale stop.flag") + except Exception as e: + pass if args.list_scenarios: print("Available scenarios:") @@ -245,6 +256,10 @@ def main(): pbar = tqdm(total=max_frames, desc=f"SIMULATING: {scenario.name}", unit=" frames", colour='cyan') while frame_count < max_frames: + if os.path.exists(flag_path): + print("\n[INFO] Stop flag detected! Breaking simulation loop for graceful shutdown...") + break + world.tick() frame_count += 1 @@ -293,10 +308,11 @@ def main(): print(f"[AUTO-MCAP] Triggering seamless conversion for: {recorder.base_path}") convert_folder(recorder.base_path) - # Restore async mode - settings.synchronous_mode = False - settings.fixed_delta_seconds = None - world.apply_settings(settings) + # ------------------------------------------------------------------ + # EXPLICIT IDLE MODE: Do NOT restore async mode here. + # Leaving the simulator in synchronous_mode freezes the GPU load + # to ~0%, allowing Shenron processes full GPU headroom. + # ------------------------------------------------------------------ print("[INFO] Done")