feat(dashboard): add simulator lifecycle management, graceful shutdown, and gpu control

config.py

@@ -27,3 +27,6 @@ MAX_FRAMES = 200
 DEFAULT_SCENARIO  = "braking"
 DEFAULT_EGO_MODEL = "vehicle.tesla.model3"
 DEFAULT_WEATHER   = "ClearNoon"
+
+# Simulator Path
+CARLA_EXECUTABLE_PATH = r"D:\CARLA\CARLA_0.9.16\CarlaUE4.exe"

dashboard/app.py

@@ -13,6 +13,8 @@ import os
 import sys
 from flask import Flask, render_template, request, jsonify, Response
 import subprocess
+import time
+import psutil
 
 # ------------------------------------------------------------------
 # Path bootstrapping — make the project root importable regardless
@@ -36,6 +38,7 @@ app = Flask(
     static_folder=os.path.join(DASHBOARD_DIR, "static"),
 )
 
+active_simulation_process = None
 
 # ── Routes ────────────────────────────────────────────────────────
 
@@ -76,6 +79,95 @@ def get_scenario_params(scenario_name):
         return jsonify({"success": False, "error": str(e)})
 
 
+@app.route("/api/simulator/status", methods=["GET"])
+def simulator_status():
+    status = "offline"
+    is_paused = False
+    is_running = False
+    try:
+        for proc in psutil.process_iter(['name']):
+            if proc.info['name'] and 'CarlaUE4' in proc.info['name']:
+                is_running = True
+                break
+    except Exception:
+        pass
+    
+    if is_running:
+        status = "loading"
+        try:
+            import carla
+            client = carla.Client('localhost', 2000)
+            client.set_timeout(10.0)
+            client.get_server_version() # Will raise if not ready
+            status = "ready"
+            is_paused = client.get_world().get_settings().synchronous_mode
+        except Exception:
+            pass
+            
+    return jsonify({"status": status, "is_paused": is_paused})
+
+@app.route("/api/simulator/idle", methods=["POST"])
+def simulator_idle():
+    try:
+        import carla
+        client = carla.Client('localhost', 2000)
+        client.set_timeout(5.0)
+        world = client.get_world()
+        settings = world.get_settings()
+        settings.synchronous_mode = True
+        world.apply_settings(settings)
+        return jsonify({"success": True})
+    except Exception as e:
+        return jsonify({"success": False, "error": str(e)})
+
+@app.route("/api/simulator/wake", methods=["POST"])
+def simulator_wake():
+    try:
+        import carla
+        client = carla.Client('localhost', 2000)
+        client.set_timeout(5.0)
+        world = client.get_world()
+        settings = world.get_settings()
+        settings.synchronous_mode = False
+        settings.fixed_delta_seconds = None
+        world.apply_settings(settings)
+        return jsonify({"success": True})
+    except Exception as e:
+        return jsonify({"success": False, "error": str(e)})
+
+
+@app.route("/api/simulator/launch", methods=["POST"])
+def simulator_launch():
+    try:
+        exe_path = getattr(config, "CARLA_EXECUTABLE_PATH", r"D:\CARLA\CARLA_0.9.16\CarlaUE4.exe")
+        if not os.path.exists(exe_path):
+            return jsonify({"success": False, "error": f"Executable not found at {exe_path}"})
+        # Launch detached
+        subprocess.Popen([exe_path], creationflags=subprocess.CREATE_NEW_PROCESS_GROUP)
+        return jsonify({"success": True})
+    except Exception as e:
+        return jsonify({"success": False, "error": str(e)})
+
+
+@app.route("/api/stop", methods=["POST"])
+def stop_simulation():
+    global active_simulation_process
+    
+    flag_path = os.path.join(PROJECT_ROOT, "tmp", "stop.flag")
+    os.makedirs(os.path.dirname(flag_path), exist_ok=True)
+    with open(flag_path, "w") as f:
+        f.write("stop")
+        
+    if active_simulation_process is not None:
+        try:
+            active_simulation_process.wait(timeout=5)
+        except subprocess.TimeoutExpired:
+            active_simulation_process.terminate()
+        active_simulation_process = None
+        
+    return jsonify({"success": True, "message": "Simulation stopped."})
+
+
 @app.route("/api/run", methods=["POST"])
 def run_simulation():
     data      = request.json or {}
@@ -97,6 +189,7 @@ def run_simulation():
         cmd.extend(["--params", f'"{params}"'])
 
     def generate():
+        global active_simulation_process
         try:
             # cwd = project root so that run.bat is reachable
             env = os.environ.copy()
@@ -112,6 +205,7 @@ def run_simulation():
                 env=env,
                 creationflags=subprocess.CREATE_NEW_PROCESS_GROUP,
             )
+            active_simulation_process = process
 
             import re
             ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')

dashboard/static/app.js

@@ -9,7 +9,105 @@ document.addEventListener('DOMContentLoaded', () => {
     const terminalOutput = document.getElementById('terminal-output');
     const clearTerminalBtn = document.getElementById('clear-terminal');
     
+    // New Elements
+    const stopBtn = document.getElementById('stop-btn');
+    const simStatusDot = document.getElementById('sim-status-dot');
+    const simStatusText = document.getElementById('sim-status-text');
+    const launchSimBtn = document.getElementById('launch-sim-btn');
+    
+    // Idle Toggle Elements
+    const idleToggleContainer = document.getElementById('idle-toggle-container');
+    const gpuIdleToggle = document.getElementById('gpu-idle-toggle');
+
     let currentEventSource = null;
+    let idleTimer = null;
+    let isSimulatorRunning = false;
+    let isSimulationActive = false; // user currently running a scenario
+
+    // Auto Idle Tracker
+    function resetIdleTimer() {
+        if (idleTimer) clearTimeout(idleTimer);
+        // If simulation is not active and simulator is ready but NOT paused
+        if (!isSimulationActive && isSimulatorRunning && gpuIdleToggle && !gpuIdleToggle.checked) {
+            idleTimer = setTimeout(() => {
+                console.log("Auto-idling simulator due to 30s inactivity...");
+                gpuIdleToggle.checked = true;
+                gpuIdleToggle.dispatchEvent(new Event('change'));
+            }, 30000);
+        }
+    }
+
+    if (gpuIdleToggle) {
+        gpuIdleToggle.addEventListener('change', (e) => {
+            const isIdle = e.target.checked;
+            const endpoint = isIdle ? '/api/simulator/idle' : '/api/simulator/wake';
+            fetch(endpoint, { method: 'POST' });
+            if (!isIdle) {
+                resetIdleTimer(); 
+            } else {
+                if (idleTimer) clearTimeout(idleTimer);
+            }
+        });
+    }
+
+    // Simulator Status Polling
+    function updateSimulatorStatus() {
+        fetch('/api/simulator/status')
+            .then(res => res.json())
+            .then(data => {
+                const status = data.status; // offline, loading, ready
+                simStatusDot.className = 'dot';
+                
+                if (status === 'ready') {
+                    if (!isSimulatorRunning) {
+                        isSimulatorRunning = true;
+                        resetIdleTimer(); // start tracker as soon as it becomes ready
+                    }
+                    if (idleToggleContainer) idleToggleContainer.style.display = 'flex';
+                    
+                    if (gpuIdleToggle && gpuIdleToggle.checked !== data.is_paused) {
+                        gpuIdleToggle.checked = data.is_paused;
+                    }
+                    
+                    if (data.is_paused) {
+                        simStatusDot.classList.add('yellow');
+                        simStatusText.textContent = 'Ready (Idle)';
+                    } else {
+                        simStatusDot.classList.add('green');
+                        simStatusText.textContent = 'Ready';
+                    }
+                    launchSimBtn.style.display = 'none';
+                } else if (status === 'loading') {
+                    isSimulatorRunning = false;
+                    if (idleToggleContainer) idleToggleContainer.style.display = 'none';
+                    simStatusDot.classList.add('yellow');
+                    simStatusText.textContent = 'Loading...';
+                    launchSimBtn.style.display = 'none';
+                } else {
+                    isSimulatorRunning = false;
+                    if (idleToggleContainer) idleToggleContainer.style.display = 'none';
+                    if (idleTimer) clearTimeout(idleTimer);
+                    
+                    simStatusDot.classList.add('red');
+                    simStatusText.textContent = 'Offline';
+                    launchSimBtn.style.display = 'block';
+                }
+            })
+            .catch(() => {
+                simStatusDot.className = 'dot red';
+                simStatusText.textContent = 'Error';
+            });
+    }
+
+    updateSimulatorStatus();
+    setInterval(updateSimulatorStatus, 1000);
+
+    // Launch CarlaUE4 action
+    launchSimBtn.addEventListener('click', () => {
+        simStatusText.textContent = 'Starting...';
+        simStatusDot.className = 'dot yellow';
+        fetch('/api/simulator/launch', { method: 'POST' });
+    });
 
     // Fetch Initial Config
     fetch('/api/config')
@@ -60,6 +158,15 @@ document.addEventListener('DOMContentLoaded', () => {
     launchForm.addEventListener('submit', (e) => {
         e.preventDefault();
         
+        isSimulationActive = true;
+        if (idleTimer) clearTimeout(idleTimer);
+        
+        // Ensure simulator is awake before scenario runs just in case
+        if (gpuIdleToggle && gpuIdleToggle.checked) {
+            gpuIdleToggle.checked = false;
+            fetch('/api/simulator/wake', { method: 'POST' });
+        }
+        
         // UI Loading State
         setLoadingState(true);
         appendLog(`\n> --- NEW SIMULATION RUN ---`, 'info');
@@ -103,6 +210,25 @@ document.addEventListener('DOMContentLoaded', () => {
         appendLog('> Terminal cleared.', 'system');
     });
     
+    // Stop Simulation
+    if (stopBtn) {
+        stopBtn.addEventListener('click', () => {
+            stopBtn.disabled = true;
+            stopBtn.querySelector('.btn-text').textContent = 'Stopping...';
+            appendLog('\n> Sending stop signal to simulation. Waiting for graceful exit...', 'warn');
+            
+            fetch('/api/stop', { method: 'POST' })
+                .then(res => res.json())
+                .then(data => {
+                    console.log('Stop requested:', data.message);
+                })
+                .catch(err => {
+                    appendLog('> Failed to send stop signal: ' + err.message, 'error');
+                    stopBtn.disabled = false;
+                });
+        });
+    }
+
     // Utility: Append log to terminal
     function appendLog(text, type = '') {
         const line = document.createElement('div');
@@ -117,8 +243,16 @@ document.addEventListener('DOMContentLoaded', () => {
         if (text.includes('Traceback') || text.includes('Exception')) line.classList.add('error');
 
         line.textContent = text;
+        
+        // Smart Scroll Lock
+        // Check if the user is currently at the bottom of the scroll view
+        const isAtBottom = terminalOutput.scrollHeight - terminalOutput.scrollTop - terminalOutput.clientHeight < 50;
+        
         terminalOutput.appendChild(line);
-        terminalOutput.scrollTop = terminalOutput.scrollHeight;
+        
+        if (isAtBottom) {
+            terminalOutput.scrollTop = terminalOutput.scrollHeight;
+        }
     }
 
     // Utility: Post JSON and stream response
@@ -153,6 +287,9 @@ document.addEventListener('DOMContentLoaded', () => {
                             appendLog(`> Simulation Finished. ${data}`, 'info');
                             setLoadingState(false);
                             document.getElementById('sim-progress-container').style.display = 'none';
+                            
+                            isSimulationActive = false;
+                            resetIdleTimer();
                         } else {
                             // Intercept progress bar logs
                             if (data.includes('SIMULATING:')) {
@@ -189,11 +326,17 @@ document.addEventListener('DOMContentLoaded', () => {
 
     function setLoadingState(isLoading) {
         launchBtn.disabled = isLoading;
-        launchText.style.display = isLoading ? 'none' : 'block';
-        launchSpinner.style.display = isLoading ? 'block' : 'none';
+        launchBtn.style.display = isLoading ? 'none' : 'block';
+        if (stopBtn) {
+            stopBtn.style.display = isLoading ? 'block' : 'none';
+        }
         
         if (!isLoading) {
             launchText.textContent = "Launch Another Simulation";
+            if (stopBtn) {
+                stopBtn.disabled = false;
+                stopBtn.querySelector('.btn-text').textContent = 'Stop Simulation';
+            }
         }
     }
 

dashboard/static/style.css

@@ -495,3 +495,90 @@ input:checked + .slider:before {
     0% { opacity: 0; transform: translateY(20px); }
     100% { opacity: 1; transform: translateY(0); }
 }
+
+/* Simulator Status Card */
+.status-card {
+    background: var(--input-bg);
+    border: 1px solid var(--input-border);
+    padding: 1rem;
+    border-radius: 8px;
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+}
+
+.status-indicator {
+    display: flex;
+    align-items: center;
+    gap: 0.5rem;
+    width: 100%;
+}
+
+.btn-secondary {
+    background: rgba(255, 255, 255, 0.1);
+    color: var(--text-primary);
+}
+
+.btn-secondary:hover {
+    background: rgba(255, 255, 255, 0.2);
+}
+
+.btn-danger {
+    background: linear-gradient(135deg, #f85149, #b31d28);
+    color: white;
+}
+
+.btn-danger:hover {
+    box-shadow: 0 4px 14px rgba(248, 81, 73, 0.4);
+    transform: translateY(-2px);
+}
+
+.btn-danger:active {
+    transform: translateY(0);
+}
+
+.btn-danger:disabled {
+    background: #6e2a2a;
+    color: #a0aec0;
+    cursor: not-allowed;
+    transform: none;
+    box-shadow: none;
+}
+
+.btn-small {
+    padding: 0.5rem 1rem;
+    font-size: 0.85rem;
+}
+
+/* Idle Toggle CSS */
+.idle-toggle-container {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    width: 100%;
+    margin-top: 12px;
+    padding-top: 12px;
+    border-top: 1px solid rgba(255,255,255,0.05);
+}
+
+.idle-label {
+    font-size: 0.8rem;
+    color: var(--text-muted);
+}
+
+.small-switch {
+    width: 34px;
+    height: 18px;
+}
+
+.small-switch .slider:before {
+    height: 12px;
+    width: 12px;
+    bottom: 3px;
+    left: 3px;
+}
+
+.small-switch input:checked + .slider:before {
+    transform: translateX(16px);
+}
+

dashboard/templates/index.html

@@ -23,6 +23,25 @@
                 </div>
             </div>
             
+            <div class="nav-section" style="margin-bottom: 2rem;">
+                <div class="section-title">SIMULATOR STATUS</div>
+                <div class="status-card" id="sim-status-card">
+                    <div class="status-indicator">
+                        <span class="dot red" id="sim-status-dot"></span>
+                        <span id="sim-status-text" style="font-weight: 500; font-size: 0.9rem;">Offline</span>
+                    </div>
+                    <button id="launch-sim-btn" class="btn btn-secondary btn-small" style="width: 100%; margin-top: 10px; padding: 0.6rem;">Initialize CarlaUE4</button>
+                    <!-- Idle Toggle -->
+                    <div id="idle-toggle-container" class="idle-toggle-container" style="display: none;">
+                        <span class="idle-label">GPU Idle Mode</span>
+                        <label class="switch small-switch">
+                            <input type="checkbox" id="gpu-idle-toggle">
+                            <span class="slider round"></span>
+                        </label>
+                    </div>
+                </div>
+            </div>
+            
             <div class="nav-section">
                 <div class="section-title">SCENARIOS</div>
                 <div id="scenario-list" class="scenario-list">
@@ -79,11 +98,14 @@
                         </div>
                     </div>
 
-                    <div class="form-actions full-width">
-                        <button type="submit" id="launch-btn" class="btn btn-primary pulse-hover">
+                    <div class="form-actions full-width" style="display: flex; gap: 1rem;">
+                        <button type="submit" id="launch-btn" class="btn btn-primary pulse-hover" style="flex: 1;">
                             <span class="btn-text" id="launch-text">Launch Simulation</span>
                             <div class="loader-spinner" id="launch-spinner" style="display: none;"></div>
                         </button>
+                        <button type="button" id="stop-btn" class="btn btn-danger" style="display: none; flex: 1;">
+                            <span class="btn-text">Stop Simulation</span>
+                        </button>
                     </div>
                 </form>
             </section>

gemini.md

@@ -60,6 +60,8 @@ This document is the consolidated source of truth for AI agents working on the F
 ## 🖥️ 4. GUI Dashboard Architecture
 
 - **Backend:** Flask (`app.py`) running on port 5000.
+- **Simulator Lifecycle:** Dashboard polls `/api/simulator/status` to detect if CARLA is Offline/Ready. Provides one-click initialization via the UI.
+- **GPU Resource Control:** Incorporates "Idle Mode" (Synchronous Throttling) to drop CARLA GPU usage to ~0% when inactive or while Shenron processing is active.
 - **Streaming:** Uses **Server-Sent Events (SSE)** to stream `stdout` to the browser.
 - **Execution:** Spawns `run.bat` as a subprocess.
 - **Crucial Rule:** Always set `PYTHONUNBUFFERED=1` to prevent log delays in the GUI console.
@@ -75,6 +77,7 @@ This document is the consolidated source of truth for AI agents working on the F
 
 ### Common Pitfalls
 - **Stale Physics:** Always `world.tick()` once after spawning the Ego but before starting scenario logic.
+- **Graceful Shutdown:** Use the "Stop Simulation" button in the dashboard. It writes a `tmp/stop.flag` which the orchestrator checks to ensure all recorders and data converters (Shenron/MCAP) finalize correctly.
 - **Progress Bar Clash:** Use `pbar.write()` for logs if a `tqdm` progress bar is active.
 - **Asynchronous I/O:** Use `ThreadPoolExecutor` for disk writes (images) to prevent simulation frame drops.
 
@@ -88,4 +91,4 @@ This document is the consolidated source of truth for AI agents working on the F
 
 ---
 
-*Generated by Antigravity AI | Last Updated: 2026-04-01 | Refer to .cursorrules for start-of-turn instructions.*
+*Generated by Antigravity AI | Last Updated: 2026-04-10 | Refer to .cursorrules for start-of-turn instructions.*

intel/internal/context.md

@@ -58,6 +58,8 @@ Fox/
 │       ├── lidar/           ← frame_XXXXXX.npy  (shape: [N, 4] — x/y/z/intensity)
 │       └── frames.jsonl     ← One JSON record per frame (metadata + scenario info)
 │
+├── tmp/                     ← Staging area for IPC flags (e.g., stop.flag)
+│
 └── intel/
     ├── radar/              ← [CRITICAL] Physics, Material RCS, and Debug logs
     │   ├── Shenron_debug.md      ← The "Source of Truth" for radar calibration
@@ -121,7 +123,7 @@ python src/main.py --list-scenarios
 6. `Recorder(scenario_name=scenario.name)` → creates `data/<scenario>_<ts>/`
 7. **Settle Physics**: Call `world.tick()` once to synchronize Ego location before setup
 8. `scenario.setup(world, ego, traffic_manager)`
-9. **Main loop** per frame: `world.tick()` → `sensor_manager.get_data()` →
+9. **Main loop** per frame: `world.tick()` → **IPC Stop Check (`tmp/stop.flag`)** → `sensor_manager.get_data()` →
    `recorder.save(..., extra_meta=scenario.get_scenario_metadata())` → `scenario.step(frame, ego, pbar)`
 10. `finally`: `scenario.cleanup()` → `sensor_manager.destroy()` → restore async mode
 
@@ -349,4 +351,4 @@ When working on this repository, prioritize documentation based on your specific
 
 ---
 
-*Last updated: 2026-04-06 | Pipeline version: Scenario-Centric Deterministic Architecture*
+*Last updated: 2026-04-10 | Pipeline version: Scenario-Centric Deterministic Architecture*

intel/scenarios/dashboard.md

@@ -62,9 +62,23 @@ When the `/api/run` endpoint is hit:
 
 ---
 
-## 4. Extension Guidelines for AI
-
-When asked to extend the Dashboard, observe the following patterns:
-- **Adding new CLI flags:** Update the `POST /api/run` payload parser in `app.py`, extend the `cmd.extend()` builder, and add the corresponding HTML `<input>` in `index.html`.
-- **Adding new data streams:** The SSE generator currently pipes raw text. To parse structured data (like active ego speed) for live graphs, consider formatting the intermediate Python prints as JSON and having `app.js` parse the `event.data` payload conditionally.
-- **Pathing:** Remember that `app.py` lives in a sub-folder. Operations manipulating the CARLA simulation *must* be rooted up two levels using the `PROJECT_ROOT` pointer established in the file.
+## 5. Simulator Lifecycle & GPU Control
+
+The dashboard now incorporates active management of the `CarlaUE4.exe` process and its hardware utilization.
+
+### A. Lifecycle Monitoring
+- **Polling:** The backend uses `psutil` to detect if the simulator process is active.
+- **Initialization:** If offline, a one-click "Initialize CarlaUE4" button allows the user to launch the simulator directly from the browser.
+- **Ping Validation:** The dashboard "Ready" state is only achieved once the backend successfully pings the CARLA server (localhost:2000) and receives a valid version response.
+
+### B. Graceful Stop Protocol (IPC)
+Instead of killing the simulation process—which can lead to corrupted radar/lidar data—the "Stop Simulation" button triggers an Inter-Process Communication (IPC) signal:
+1. Dashboard writes an empty flag file to `tmp/stop.flag`.
+2. The orchestrator (`src/main.py`) checks for this file's existence at the start of every frame tick.
+3. If detected, the orchestrator breaks the main loop, allowing all `finally` cleanups, recorder closures, and MCAP conversions to finish elegantly.
+
+### C. GPU Resource Control (Idle Mode)
+To prevent CARLA from consuming 100% GPU while waiting for user interaction or during Shenron radar post-processing:
+- **Synchronous Throttling:** The dashboard can toggle CARLA into `synchronous_mode`.
+- **Idle State:** Without a `world.tick()` call from the client, Unreal Engine pauses its rendering and physics loop, dropping GPU usage to ~0%.
+- **Auto-Idle:** The dashboard incorporates a 30-second inactivity timer that automatically pauses the simulator to save power.

src/main.py

@@ -85,6 +85,17 @@ def parse_args():
 def main():
     args = parse_args()
     
+    # ------------------------------------------------------------------
+    # 0. Clean up stale stop flags
+    # ------------------------------------------------------------------
+    flag_path = os.path.join(os.path.dirname(__file__), "..", "tmp", "stop.flag")
+    if os.path.exists(flag_path):
+        try:
+            os.remove(flag_path)
+            print("[INFO] Cleaned up stale stop.flag")
+        except Exception as e:
+            pass
+
     if args.list_scenarios:
         print("Available scenarios:")
         for s in list_scenarios():
@@ -245,6 +256,10 @@ def main():
         pbar = tqdm(total=max_frames, desc=f"SIMULATING: {scenario.name}", unit=" frames", colour='cyan')
         
         while frame_count < max_frames:
+            if os.path.exists(flag_path):
+                print("\n[INFO] Stop flag detected! Breaking simulation loop for graceful shutdown...")
+                break
+                
             world.tick()
             frame_count += 1
             
@@ -293,10 +308,11 @@ def main():
                 print(f"[AUTO-MCAP] Triggering seamless conversion for: {recorder.base_path}")
                 convert_folder(recorder.base_path)
 
-        # Restore async mode
-        settings.synchronous_mode   = False
-        settings.fixed_delta_seconds = None
-        world.apply_settings(settings)
+        # ------------------------------------------------------------------
+        # EXPLICIT IDLE MODE: Do NOT restore async mode here. 
+        # Leaving the simulator in synchronous_mode freezes the GPU load 
+        # to ~0%, allowing Shenron processes full GPU headroom.
+        # ------------------------------------------------------------------
 
         print("[INFO] Done")