|
| 1 | +import argparse |
| 2 | +import subprocess |
| 3 | +import shutil |
| 4 | +import sqlite3 |
| 5 | +import statistics |
| 6 | +import fnmatch |
| 7 | +import site |
| 8 | +import sys |
| 9 | +import os |
| 10 | + |
| 11 | + |
| 12 | +PROFILES = [ |
| 13 | + {"name": "ovrtx_constant_diffuse_oldpipe", "preset": "ovrtx_renderer,simple_shading_constant_diffuse", "settings": { "min-pipe": False }}, |
| 14 | + {"name": "ovrtx_constant_diffuse_newpipe", "preset": "ovrtx_renderer,simple_shading_constant_diffuse", "settings": { "min-pipe": True }}, |
| 15 | + {"name": "ovrtx_diffuse_mdl_oldpipe", "preset": "ovrtx_renderer,simple_shading_diffuse_mdl", "settings": { "min-pipe": False }}, |
| 16 | + {"name": "ovrtx_diffuse_mdl_newpipe", "preset": "ovrtx_renderer,simple_shading_diffuse_mdl", "settings": { "min-pipe": True }}, |
| 17 | + {"name": "ovrtx_full_mdl_oldpipe", "preset": "ovrtx_renderer,simple_shading_full_mdl", "settings": { "min-pipe": False }}, |
| 18 | + {"name": "ovrtx_full_mdl_newpipe", "preset": "ovrtx_renderer,simple_shading_full_mdl", "settings": { "min-pipe": True }}, |
| 19 | + |
| 20 | + {"name": "newton_lbvh_lbvh", "preset": "newton_renderer,rgb", "settings": { "tlas": "lbvh", "blas": "lbvh" }}, |
| 21 | + {"name": "newton_lbvh_sah", "preset": "newton_renderer,rgb", "settings": { "tlas": "lbvh", "blas": "sah" }}, |
| 22 | + {"name": "newton_lbvh_cubql", "preset": "newton_renderer,rgb", "settings": { "tlas": "lbvh", "blas": "cubql" }}, |
| 23 | + {"name": "newton_sah_lbvh", "preset": "newton_renderer,rgb", "settings": { "tlas": "sah", "blas": "lbvh" }}, |
| 24 | + {"name": "newton_sah_sah", "preset": "newton_renderer,rgb", "settings": { "tlas": "sah", "blas": "sah" }}, |
| 25 | + {"name": "newton_sah_cubql", "preset": "newton_renderer,rgb", "settings": { "tlas": "sah", "blas": "cubql" }}, |
| 26 | +] |
| 27 | + |
| 28 | +TASK_NAME = "Isaac-RenderBenchmark-Franka-Cabinet" |
| 29 | +OUTPUT_PATH = "benchmarks" |
| 30 | +FRAME_PADDING = 5 |
| 31 | + |
| 32 | + |
| 33 | +def get_profile(name: str) -> dict | None: |
| 34 | + for profile in PROFILES: |
| 35 | + if profile["name"] == name: |
| 36 | + return profile |
| 37 | + return None |
| 38 | + |
| 39 | +def parse_profile(filename: str, renderer: str, num_frames: int): |
| 40 | + cursor = sqlite3.connect(filename) |
| 41 | + |
| 42 | + if renderer == "ovrtx": |
| 43 | + steps = cursor.execute("SELECT start,end FROM NVTX_EVENTS WHERE text='ovrtx_step_execute' ORDER BY start").fetchall() |
| 44 | + rid_row = cursor.execute("SELECT id FROM StringIds WHERE value='RTX Rendering'").fetchone() |
| 45 | + if not rid_row: |
| 46 | + return None |
| 47 | + |
| 48 | + out = [] |
| 49 | + for s, e in cursor.execute("SELECT start,end FROM VULKAN_WORKLOAD WHERE textId=? ORDER BY start", (rid_row[0],)).fetchall(): |
| 50 | + for i, (ss, se) in enumerate(steps): |
| 51 | + if ss <= s <= se: |
| 52 | + if FRAME_PADDING < i + 1 <= (num_frames + FRAME_PADDING): |
| 53 | + out.append((e - s) / 1e6) |
| 54 | + break |
| 55 | + |
| 56 | + if renderer == "newton_renderer": |
| 57 | + ranges = cursor.execute(""" |
| 58 | + SELECT n.start, n.end FROM NVTX_EVENTS n |
| 59 | + JOIN StringIds s ON s.id=n.textId |
| 60 | + WHERE s.value LIKE '%NewtonWarpRenderer.render%' |
| 61 | + ORDER BY n.start |
| 62 | + """).fetchall() |
| 63 | + |
| 64 | + out = [] |
| 65 | + for i, (rs, re) in enumerate(ranges): |
| 66 | + next_rs = ranges[i + 1][0] if i + 1 < len(ranges) else float('inf') |
| 67 | + |
| 68 | + row = cursor.execute(""" |
| 69 | + SELECT MIN(k.start), MAX(k.end) FROM CUPTI_ACTIVITY_KIND_KERNEL k |
| 70 | + JOIN StringIds s ON s.id=k.demangledName |
| 71 | + WHERE k.start>=? AND k.start<? AND s.value LIKE '%render_megakernel%' |
| 72 | + """, (rs, next_rs)).fetchone() |
| 73 | + |
| 74 | + if row[0] is None: |
| 75 | + continue |
| 76 | + |
| 77 | + if FRAME_PADDING < i + 1 <= (num_frames + FRAME_PADDING): |
| 78 | + out.append((row[1] - row[0]) / 1e6) |
| 79 | + |
| 80 | + if out: |
| 81 | + return { |
| 82 | + "size": len(out), |
| 83 | + "median": statistics.median(out), |
| 84 | + "mean": statistics.mean(out), |
| 85 | + "min": min(out), |
| 86 | + "max": max(out), |
| 87 | + "stdev": statistics.stdev(out) if len(out) > 1 else 0 |
| 88 | + } |
| 89 | + return None |
| 90 | + |
| 91 | + |
| 92 | +def run_profile(profile: dict, num_frames: int, num_envs: int, resolution: int, save_image: bool, verbose: bool): |
| 93 | + warp_cache_path = os.path.abspath(os.path.join(OUTPUT_PATH, "warp-cache")) |
| 94 | + |
| 95 | + env = { |
| 96 | + "NVTX_PROFILE_PYTHON": "1", |
| 97 | + "NVTX_PROFILE_INCLUDE": "isaaclab,newton,warp,rsl_rl", |
| 98 | + "NEWTON_USE_CUDA_GRAPH": "0", |
| 99 | + "BENCHMARK_SAVE_IMAGE": "1" if save_image else "0", |
| 100 | + "BENCHMARK_RENDER_RESOLUTION": f"{resolution}", |
| 101 | + "WARP_CACHE_PATH": warp_cache_path, |
| 102 | + } |
| 103 | + |
| 104 | + if os.path.exists(warp_cache_path): |
| 105 | + shutil.rmtree(warp_cache_path) |
| 106 | + |
| 107 | + profile_name: str = profile["name"] |
| 108 | + preset: str = profile["preset"] |
| 109 | + renderer: str = preset.split(",")[0] |
| 110 | + trace: str = "nvtx" |
| 111 | + |
| 112 | + if renderer == "ovrtx_renderer": |
| 113 | + trace = "nvtx,vulkan,vulkan-annotations" |
| 114 | + env["LD_PRELOAD"] = os.path.join(site.getsitepackages()[0], "ovrtx/bin/plugins/omni.client.lib/libomniclient.so") |
| 115 | + env["CUDA_VISIBLE_DEVICES"] = "0" |
| 116 | + env["OMNI_KIT_ACCEPT_EULA"] = "YES" |
| 117 | + env["OVRTX_rtx_post_tonemap_op"] = "0" |
| 118 | + env["OVRTX_rtx_minimal_useMinimalPipeline"] = "1" if profile["settings"]["min-pipe"] else "0" |
| 119 | + env["OVRTX_app_profilerBackend"] = "nvtx" |
| 120 | + env["OVRTX_app_profileFromStart"] = "true" |
| 121 | + env["OVRTX_app_profilerMask"] = "1" |
| 122 | + |
| 123 | + |
| 124 | + if renderer == "newton_renderer": |
| 125 | + trace = "nvtx,cuda" |
| 126 | + env["NEWTON_BVH_SCENE"] = profile["settings"]["tlas"] |
| 127 | + env["NEWTON_BVH_GEOMETRY"] = profile["settings"]["blas"] |
| 128 | + |
| 129 | + os.makedirs(OUTPUT_PATH, exist_ok=True) |
| 130 | + profile_filename = os.path.join(OUTPUT_PATH, profile_name + ".nsys-rep") |
| 131 | + log_filename = os.path.join(OUTPUT_PATH, profile["name"] + ".log") |
| 132 | + |
| 133 | + cmd = [ |
| 134 | + "nsys", |
| 135 | + "profile", |
| 136 | + "--output", profile_filename, |
| 137 | + "--force-overwrite", "true", |
| 138 | + "--trace", trace, |
| 139 | + "--vulkan-gpu-workload", "individual", |
| 140 | + "--python-functions-trace", "scripts/benchmarks/nsys_trace.json", |
| 141 | + |
| 142 | + "./isaaclab.sh", "-p", "scripts/benchmarks/runtime.py", |
| 143 | + "--task", TASK_NAME, |
| 144 | + "--headless", |
| 145 | + "--enable_cameras", |
| 146 | + "--num_envs", f"{num_envs}", |
| 147 | + "--num_frames", f"{num_frames + FRAME_PADDING * 2}", |
| 148 | + "--output_path", OUTPUT_PATH, |
| 149 | + f"presets={preset}" |
| 150 | + ] |
| 151 | + |
| 152 | + with open(log_filename, "w") as file: |
| 153 | + process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=dict(os.environ) | env, text=True) |
| 154 | + |
| 155 | + for line in process.stdout: |
| 156 | + file.write(line) |
| 157 | + if verbose: |
| 158 | + sys.stdout.write(f"\x1b[90m{line}\x1b[0m") |
| 159 | + sys.stdout.flush() |
| 160 | + |
| 161 | + return_code = process.wait() |
| 162 | + if return_code != 0: |
| 163 | + print(f"Failed with exit code {process.returncode}, see {log_filename} for details.") |
| 164 | + return False |
| 165 | + |
| 166 | + subprocess.run([ "nsys", "stats", "--force-export", "true", "--report", "cuda_kern_exec_sum", "--format", "csv", profile_filename ], stdout=subprocess.PIPE) |
| 167 | + return parse_profile(profile_filename.replace(".nsys-rep", ".sqlite"), renderer, num_frames) |
| 168 | + |
| 169 | + |
| 170 | +parser = argparse.ArgumentParser("IsaacLab Benchmark: Sweep Franka Cabinet") |
| 171 | +parser.add_argument("--num-frames", type=int, default="20", help="Number of frames to render") |
| 172 | +parser.add_argument("--num-envs", type=int, default="1024", help="Number of environments to render") |
| 173 | +parser.add_argument("--resolution", type=int, default="256", help="Render resolution") |
| 174 | +parser.add_argument("-v", "--verbose", action="store_true", help="Verbose output") |
| 175 | +parser.add_argument("-s", "--save-image", action="store_true", help="Save image for debugging purposes") |
| 176 | +parser.add_argument("profile", nargs="*", help="Profiles to run") |
| 177 | + |
| 178 | +args = parser.parse_args() |
| 179 | + |
| 180 | +if not args.profile: |
| 181 | + print("Available profiles:") |
| 182 | + for profile in PROFILES: |
| 183 | + print(" " + profile["name"]) |
| 184 | + exit(0) |
| 185 | + |
| 186 | +profiles = set() |
| 187 | + |
| 188 | +for profile_name in args.profile: |
| 189 | + found_match = False |
| 190 | + for profile in PROFILES: |
| 191 | + if fnmatch.fnmatch(profile["name"], profile_name): |
| 192 | + profiles.add(profile["name"]) |
| 193 | + found_match = True |
| 194 | + |
| 195 | + if not found_match: |
| 196 | + print(f"Not profile found matching: {profile_name}") |
| 197 | + exit(-1) |
| 198 | + |
| 199 | +all_results = {} |
| 200 | +for profile_name in profiles: |
| 201 | + if profile := get_profile(profile_name): |
| 202 | + print(f"profile: {profile['name']}") |
| 203 | + print(f" preset: {profile['preset']}") |
| 204 | + for key, value in profile["settings"].items(): |
| 205 | + print(f" {key}: {value}") |
| 206 | + |
| 207 | + try: |
| 208 | + all_results[profile_name] = run_profile(profile, args.num_frames, args.num_envs, args.resolution, args.save_image, args.verbose) |
| 209 | + except KeyboardInterrupt: |
| 210 | + break |
| 211 | + |
| 212 | + if results := all_results[profile_name]: |
| 213 | + print(f" size: {results['size']}") |
| 214 | + for key, value in results.items(): |
| 215 | + if key != "size": |
| 216 | + print(f" {key}: {value:.2f}ms") |
| 217 | + print("") |
| 218 | + |
| 219 | +benchmark_faled = False |
| 220 | +print("") |
| 221 | +print("| PROFILE | SIZE | MEDIAN | MEAN | MIN | MAX | STDEV |") |
| 222 | +print("|------------------------------------------|------|--------------|--------------|--------------|--------------|--------------|") |
| 223 | +for profile_name, results in all_results.items(): |
| 224 | + if results: |
| 225 | + print(f"| {profile_name:<40} | {results['size']:>4} | {results['median']:>10.2f}ms | {results['mean']:>10.2f}ms | {results['min']:>10.2f}ms | {results['max']:>10.2f}ms | {results['stdev']:>10.2f}ms |") |
| 226 | + else: |
| 227 | + print(f"| {profile_name:<40} | FAILED |") |
| 228 | + benchmark_faled = True |
| 229 | +print("|------------------------------------------|------|--------------|--------------|--------------|--------------|--------------|") |
| 230 | +print("") |
| 231 | +if benchmark_faled: |
| 232 | + exit(-1) |
| 233 | +exit(0) |
0 commit comments