From bd466a356823db39cca160b8a5694a58efdd4bb5 Mon Sep 17 00:00:00 2001 From: rakadu1 Date: Tue, 5 May 2026 17:02:26 +0530 Subject: [PATCH] refactor: unify Shenron radar processing pipeline Established a common orchestration framework to eliminate logic duplication between production data generation and the iterative testbench. Key changes: - Created 'ShenronOrchestrator' in scripts/ISOLATE/shenron_orchestrator.py to serve as the single source of truth for the processing loop. - Refactored 'generate_shenron.py' to use the orchestrator, ensuring production data benefit from research-level DSP improvements. - Refactored 'test_shenron.py' to use the orchestrator, guaranteeing that debug iterations are bit-identical to production outputs. - Centralized LiDAR padding, model execution, and metrology/metric serialization logic. - Preserved Dashboard SSE telemetry patterns ([SHENRON_INIT/STEP]) to maintain full UI compatibility. This restructuring ensures that any iterative changes made in the 'test_shenron' lab are automatically and safely inherited by the Dashboard's automated simulation pipeline. --- scripts/ISOLATE/shenron_orchestrator.py | 129 ++++++++++++++++++++++++ scripts/generate_shenron.py | 117 ++------------------- scripts/test_shenron.py | 63 ++---------- 3 files changed, 146 insertions(+), 163 deletions(-) create mode 100644 scripts/ISOLATE/shenron_orchestrator.py diff --git a/scripts/ISOLATE/shenron_orchestrator.py b/scripts/ISOLATE/shenron_orchestrator.py new file mode 100644 index 0000000..48ac771 --- /dev/null +++ b/scripts/ISOLATE/shenron_orchestrator.py @@ -0,0 +1,129 @@ +import os +import sys +import time +import numpy as np +import json +from pathlib import Path + +# Ensure we can import the model wrapper +sys.path.append(str(Path(__file__).parent)) +try: + from model_wrapper import ShenronRadarModel +except ImportError: + # Fallback if called from a different context + from scripts.ISOLATE.model_wrapper import ShenronRadarModel + +class ShenronOrchestrator: + """ + Unified orchestration engine for physics-based radar synthesis. + Ensures parity between production generation (dashboard) and iterative testbench (test_shenron). + """ + + def __init__(self, radar_types=['awrl1432', 'radarbook']): + self.radar_types = radar_types + self.models = {} + + def init_models(self, output_root: Path): + """Initializes models and prepares directory structure.""" + specs = {} + for r_type in self.radar_types: + try: + print(f" - Initializing Shenron {r_type} engine...") + model = ShenronRadarModel(radar_type=r_type) + self.models[r_type] = model + + # Setup Folders + radar_dir = output_root / r_type + radar_dir.mkdir(exist_ok=True, parents=True) + + met_base = radar_dir / "metrology" + for sub in ["rd", "ra", "cfar"]: + (met_base / sub).mkdir(parents=True, exist_ok=True) + + # Save physical axes (static per session) + np.save(met_base / "range_axis.npy", model.processor.rangeAxis) + np.save(met_base / "angle_axis.npy", model.processor.angleAxis) + + # Get hardware specs + specs[r_type] = model.get_radar_specs() + + # Save specs for MCAP converter downstream + hw_specs = { + 'f': float(model.radar_obj.f), + 'chirp_rep': float(model.radar_obj.chirp_rep), + 'max_velocity': float((3e8 / model.radar_obj.f) / (4 * model.radar_obj.chirp_rep)), + } + with open(met_base / "radar_specs.json", "w") as sf: + json.dump(hw_specs, sf) + + except Exception as e: + print(f" [WARNING] Failed to init {r_type}: {e}") + continue + return specs + + def process_frame(self, lidar_file: Path, output_root: Path, save_adc=False): + """Processes a single LiDAR frame through all active radar models.""" + try: + data = np.load(lidar_file) + except Exception as e: + print(f" [ERROR] Failed to load {lidar_file.name}: {e}") + return None + + # Standard Padding: Ensure [x, y, z, intensity, cos_inc_angle, obj, tag] + if data.shape[1] == 6: + padded_data = np.zeros((data.shape[0], 7), dtype=np.float32) + padded_data[:, 0:3] = data[:, 0:3] + padded_data[:, 4:7] = data[:, 3:6] + data = padded_data + + frame_results = {} + + for r_type, model in self.models.items(): + try: + # 1. Physics Processing + rich_pcd = model.process(data) + + # 2. Save Pointcloud + output_file = output_root / r_type / lidar_file.name + np.save(output_file, rich_pcd) + + # 3. Optional ADC Saving + if save_adc and hasattr(model, "last_adc") and model.last_adc is not None: + adc_folder = output_root / r_type / "adc_raw" + adc_folder.mkdir(parents=True, exist_ok=True) + np.save(adc_folder / lidar_file.name, model.last_adc) + + # 4. Save Metrology (.npy) + met = model.get_last_metrology() + met_base = output_root / r_type / "metrology" + if met: + frame_name = lidar_file.stem + np.save(met_base / "rd" / f"{frame_name}.npy", met['rd_heatmap']) + np.save(met_base / "ra" / f"{frame_name}.npy", met['ra_heatmap']) + np.save(met_base / "cfar" / f"{frame_name}.npy", met['threshold_matrix']) + + # 5. Extract and Save Metrics + metrics = model.get_signal_metrics() + if metrics: + # Clean for JSON (handle NaN/Inf) + clean_metrics = {} + for k, v in metrics.items(): + if isinstance(v, float) and (np.isnan(v) or np.isinf(v)): + clean_metrics[k] = 0.0 + else: + clean_metrics[k] = v + + # Append to metrics log + metrics_file = met_base / "metrics.jsonl" + with open(metrics_file, "a") as f: + f.write(json.dumps({"frame": lidar_file.stem, **clean_metrics}) + "\n") + + # Add point count for telemetry + clean_metrics["pts"] = int(rich_pcd.shape[0]) if hasattr(rich_pcd, 'shape') else 0 + frame_results[r_type] = clean_metrics + + except Exception as e: + print(f" [ERROR] {r_type} processing failed for {lidar_file.name}: {e}") + continue + + return frame_results diff --git a/scripts/generate_shenron.py b/scripts/generate_shenron.py index 3c4cacc..594a626 100644 --- a/scripts/generate_shenron.py +++ b/scripts/generate_shenron.py @@ -55,50 +55,19 @@ def process_session(session_path): print(f" [SKIP] No .npy files in 'lidar' folder.") return - radar_types = ['awrl1432', 'radarbook'] - models = {} + from scripts.ISOLATE.shenron_orchestrator import ShenronOrchestrator + + orchestrator = ShenronOrchestrator(radar_types=['awrl1432', 'radarbook']) # ----------------------------------------------------------------------- - # DIAGNOSTIC: Step 1 - Model Initialization + # TELEMETRY: Init Phase # ----------------------------------------------------------------------- print(f" [DIAGNOSTIC] Step 1: Initializing models...", flush=True) - for r_type in radar_types: - try: - print(f" - Loading physics weights for {r_type}...", flush=True) - models[r_type] = ShenronRadarModel(radar_type=r_type) - (session_path / r_type).mkdir(exist_ok=True) - - # Create Metrology folders - met_base = session_path / r_type / "metrology" - for sub in ["rd", "ra", "cfar"]: - (met_base / sub).mkdir(parents=True, exist_ok=True) - - # Save physical axes once per session - np.save(met_base / "range_axis.npy", models[r_type].processor.rangeAxis) - np.save(met_base / "angle_axis.npy", models[r_type].processor.angleAxis) - - # Save radar hardware specs for downstream MCAP visualization - radar_hw_specs = { - 'f': float(models[r_type].radar_obj.f), - 'chirp_rep': float(models[r_type].radar_obj.chirp_rep), - 'max_velocity': float((3e8 / models[r_type].radar_obj.f) / (4 * models[r_type].radar_obj.chirp_rep)), - } - with open(met_base / "radar_specs.json", "w") as sf: - json.dump(radar_hw_specs, sf) - except Exception as e: - print(f" [WARNING] Failed to init {r_type}: {e}") - continue + radar_specs = orchestrator.init_models(session_path) - # ----------------------------------------------------------------------- - # TELEMETRY: Emit structured init payload - # ----------------------------------------------------------------------- print(f" [DIAGNOSTIC] Step 2: Collecting metadata...", flush=True) gpu_info = _get_gpu_info() - radar_specs = {} - for r_type, model in models.items(): - radar_specs[r_type] = model.get_radar_specs() - telemetry_init = { "gpu": gpu_info, "radars": radar_specs, @@ -117,78 +86,12 @@ def process_session(session_path): for frame_idx, lidar_file in enumerate(lidar_files): frame_start = time.time() - # 1. Load Semantic LiDAR data - try: - data = np.load(lidar_file) - except Exception as e: - print(f"\n [ERROR] Failed to load {lidar_file.name}: {e}") - continue + # Process through the unified orchestrator + frame_results = orchestrator.process_frame(lidar_file, session_path, save_adc=True) - if data.shape[1] == 6: - padded_data = np.zeros((data.shape[0], 7), dtype=np.float32) - padded_data[:, 0:3] = data[:, 0:3] - padded_data[:, 4:7] = data[:, 3:6] - data = padded_data - - frame_metrics = {} - - for r_type, model in models.items(): - # Ensure a folder exists for raw ADC samples - adc_folder = session_path / r_type / "adc_raw" - adc_folder.mkdir(parents=True, exist_ok=True) - try: - # 2. Process through the physics-based model - # print(f" [DEBUG] Processing {r_type} frame {frame_idx+1}...", end='\r', flush=True) - rich_pcd = model.process(data) - # 2. Save raw ADC data (saved by the model in .last_adc) - if hasattr(model, "last_adc") and model.last_adc is not None: - adc_path = session_path / r_type / "adc_raw" / lidar_file.name - np.save(adc_path, model.last_adc) - - # 3. Save to disk - output_file = session_path / r_type / lidar_file.name - np.save(output_file, rich_pcd) - - # 4. Save Metrology Heatmaps - met_base = session_path / r_type / "metrology" - met = model.get_last_metrology() - if met: - frame_name = lidar_file.stem - ra_map = met['ra_heatmap'] - np.save(met_base / "rd" / f"{frame_name}.npy", met['rd_heatmap']) - np.save(met_base / "ra" / f"{frame_name}.npy", ra_map) - np.save(met_base / "cfar" / f"{frame_name}.npy", met['threshold_matrix']) - - # --- SANITY CHECK: Azimuth Variance --- - # Detects if RA heatmap is "peaky" (good) vs "rings/flat" (broken phase) - # We use the normalized variance from the model for consistency - pass # Handled by model.get_signal_metrics below - - # 5. Save Signal Metrics - try: - metrics = model.get_signal_metrics() - if metrics: - # Clean metrics for JSON (handle NaN/Inf) - clean_metrics = frame_metrics.get(r_type, {}) - for k, v in metrics.items(): - if isinstance(v, float) and (np.isnan(v) or np.isinf(v)): - clean_metrics[k] = 0.0 - else: - clean_metrics[k] = v - - metrics_file = met_base / "metrics.jsonl" - with open(metrics_file, "a") as f: - f.write(json.dumps({"frame": lidar_file.stem, **clean_metrics}) + "\n") - - # CAPTURE UNIQUE POINT COUNT FOR TELEMETRY - clean_metrics["pts"] = int(rich_pcd.shape[0]) if hasattr(rich_pcd, 'shape') else 0 - frame_metrics[r_type] = clean_metrics - except Exception as e: - pass # Metrics failure shouldn't crash the loop + if frame_results is None: + continue - except Exception as e: - print(f"\n [ERROR] Failed to process {lidar_file.name} for {r_type}: {e}") - # Timing frame_elapsed = time.time() - frame_start frame_times.append(frame_elapsed) @@ -210,7 +113,7 @@ def process_session(session_path): "metrics": {} } - for r_type, m in frame_metrics.items(): + for r_type, m in frame_results.items(): telemetry_frame["metrics"][r_type] = { "snr": round(m.get("peak_snr_db", 0), 1), "pts": m.get("pts", 0), diff --git a/scripts/test_shenron.py b/scripts/test_shenron.py index 18a9120..363c390 100644 --- a/scripts/test_shenron.py +++ b/scripts/test_shenron.py @@ -164,26 +164,10 @@ def run_testbench(iter_name): # 1. GENERATE SYNTHETIC DATA print("\n[Stage 1]: Processing Physics models...") - models = {} - for r_type in radar_types: - try: - print(f" -> Initializing {r_type} engine...") - models[r_type] = ShenronRadarModel(radar_type=r_type) - (iter_dir / r_type).mkdir(exist_ok=True) - - # Create Metrology folders - met_base = iter_dir / r_type / "metrology" - for sub in ["rd", "ra", "cfar"]: - (met_base / sub).mkdir(parents=True, exist_ok=True) - - except Exception as e: - print(f" -> [WARNING] Failed to init {r_type}: {e}") - continue - - # Save physical axes once per radar type (same for every frame — config-derived) - met_base = iter_dir / r_type / "metrology" - np.save(met_base / "range_axis.npy", models[r_type].processor.rangeAxis) - np.save(met_base / "angle_axis.npy", models[r_type].processor.angleAxis) + from scripts.ISOLATE.shenron_orchestrator import ShenronOrchestrator + + orchestrator = ShenronOrchestrator(radar_types=radar_types) + radar_specs = orchestrator.init_models(iter_dir) lidar_files = sorted(list(lidar_dir.glob("*.npy"))) if args.frames and args.frames > 0: @@ -191,40 +175,7 @@ def run_testbench(iter_name): lidar_files = lidar_files[:args.frames] for lidar_file in tqdm.tqdm(lidar_files, desc=" Simulating Radars", unit="frame"): - data = np.load(lidar_file) - # Pad to [x, y, z, intensity, cos_inc_angle, obj, tag] if needed - if data.shape[1] == 6: - padded_data = np.zeros((data.shape[0], 7), dtype=np.float32) - padded_data[:, 0:3] = data[:, 0:3] - padded_data[:, 4:7] = data[:, 3:6] - data = padded_data - - for r_type, model in models.items(): - try: - rich_pcd = model.process(data) - out_path = iter_dir / r_type / lidar_file.name - np.save(out_path, rich_pcd) - - # --- PHASES 1 & 3: Save Raw Metrology (.npy) --- - met = model.get_last_metrology() - if met: - frame_name = lidar_file.stem # e.g., frame_000200 - ra_map = met['ra_heatmap'] - np.save(iter_dir / r_type / "metrology" / "rd" / f"{frame_name}.npy", met['rd_heatmap']) - np.save(iter_dir / r_type / "metrology" / "ra" / f"{frame_name}.npy", ra_map) - np.save(iter_dir / r_type / "metrology" / "cfar" / f"{frame_name}.npy", met['threshold_matrix']) - - # --- SANITY CHECK: Azimuth Variance --- - # Detects if RA heatmap is "peaky" (good) vs "rings/flat" (broken phase) - pass # Handled by model.get_signal_metrics below - - # Log Metrics - metrics = model.get_signal_metrics() - with open(iter_dir / r_type / "metrology" / "metrics.jsonl", "a") as mf: - mf.write(json.dumps({"frame": frame_name, **metrics}) + "\n") - - except Exception as e: - print(f"[ERROR] Frame {lidar_file.name} failed for {r_type}: {e}") + orchestrator.process_frame(lidar_file, iter_dir, save_adc=False) # 2. GENERATE MCAP print("\n[Stage 2]: Weaving MCAP Comparison Package...") @@ -300,8 +251,8 @@ def run_testbench(iter_name): cached_axes[r_type] = None # Initialize the stateful Matplotlib renderers for extreme throughput - f_cfg = models[r_type].radar_obj.f if r_type in models else 77e9 - chirp_rep_cfg = models[r_type].radar_obj.chirp_rep if r_type in models else 3e-5 + f_cfg = orchestrator.models[r_type].radar_obj.f if r_type in orchestrator.models else 77e9 + chirp_rep_cfg = orchestrator.models[r_type].radar_obj.chirp_rep if r_type in orchestrator.models else 3e-5 max_vel_cfg = (3e8 / f_cfg) / (4 * chirp_rep_cfg) max_r_cfg = cached_axes[r_type]['range_axis'][-1] if cached_axes[r_type] else 150 display_limit_cfg = 120.0