Requesting Help for RTX 4090 Tuning

[PrabinKuSabat]

The config being used is this:

# functional simulator specification
-gpgpu_ptx_instruction_classification 0
-gpgpu_ptx_sim_mode 0
-gpgpu_ptx_force_max_capability 89

# Device Limits
-gpgpu_stack_size_limit 1024
-gpgpu_heap_size_limit 8388608
-gpgpu_runtime_sync_depth_limit 2
-gpgpu_runtime_pending_launch_count_limit 2048
-gpgpu_kernel_launch_latency  248698
-gpgpu_TB_launch_latency 0

# Compute Capability
-gpgpu_compute_capability_major 8
-gpgpu_compute_capability_minor 9

# PTX execution-driven
-gpgpu_ptx_convert_to_ptxplus 0
-gpgpu_ptx_save_converted_ptxplus 0

# high level architecture configuration
-gpgpu_n_clusters 128
-gpgpu_n_cores_per_cluster 1
-gpgpu_n_mem 16
-gpgpu_n_sub_partition_per_mchannel 2

# clock domains
#-gpgpu_clock_domains <Core Clock>:<Interconnect Clock>:<L2 Clock>:<DRAM Clock>
-gpgpu_clock_domains 2235:2235:2235:5250.5

# shader core pipeline config
-gpgpu_shader_registers 65536
-gpgpu_registers_per_block 65536
-gpgpu_occupancy_sm_number 89

-gpgpu_shader_core_pipeline 1536:32
-gpgpu_shader_cta 32
-gpgpu_simd_model 1

# Pipeline widths and number of FUs
# ID_OC_SP,ID_OC_DP,ID_OC_INT,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_DP,OC_EX_INT,OC_EX_SFU,OC_EX_MEM,EX_WB,ID_OC_TENSOR_CORE,OC_EX_TENSOR_CORE
-gpgpu_pipeline_widths 4,4,4,4,4,4,4,4,4,4,8,4,4
-gpgpu_num_sp_units 4
-gpgpu_num_sfu_units 4
-gpgpu_num_dp_units 4
-gpgpu_num_int_units 4
-gpgpu_tensor_core_avail 1
-gpgpu_num_tensor_core_units 4

# Instruction latencies and initiation intervals
# "ADD,MAX,MUL,MAD,DIV"
# All Div operations are executed on SFU unit
-ptx_opcode_latency_int 4,4,4,4,21
-ptx_opcode_initiation_int 2,2,2,2,2
-ptx_opcode_latency_fp 4,4,4,4,39
-ptx_opcode_initiation_fp 2,2,2,2,4
-ptx_opcode_latency_dp 54,54,54,54,330
-ptx_opcode_initiation_dp 64,64,64,64,130
-ptx_opcode_latency_sfu 23
-ptx_opcode_initiation_sfu 8
-ptx_opcode_latency_tesnor 64
-ptx_opcode_initiation_tensor 64

# sub core model: in which each scheduler has its own register file and EUs
# i.e. schedulers are isolated
-gpgpu_sub_core_model 1
# disable specialized operand collectors and use generic operand collectors instead
-gpgpu_enable_specialized_operand_collector 0
-gpgpu_operand_collector_num_units_gen 8
-gpgpu_operand_collector_num_in_ports_gen 8
-gpgpu_operand_collector_num_out_ports_gen 8
# register banks
-gpgpu_num_reg_banks 8
-gpgpu_reg_file_port_throughput 2

# warp scheduling
-gpgpu_num_sched_per_core 4
-gpgpu_scheduler gto
# a warp scheduler issue mode
-gpgpu_max_insn_issue_per_warp 1
-gpgpu_dual_issue_diff_exec_units 1

## L1/shared memory configuration
# <nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_alloc>:<set_index_fn>,<mshr>:<N>:<merge>,<mq>:**<fifo_entry>
# ** Optional parameter - Required when mshr_type==Texture Fifo
# In adaptive cache, we adaptively assign the remaining shared memory to L1 cache
# For more info, see https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#shared-memory-7-x
-gpgpu_adaptive_cache_config 1
-gpgpu_shmem_option 0,8,16,32,64,100
-gpgpu_unified_l1d_size 128
# L1 cache configuration
-gpgpu_l1_banks 4
-gpgpu_cache:dl1 S:4:128:64,L:T:m:L:L,A:384:48,16:0,32
-gpgpu_l1_latency 44
-gpgpu_gmem_skip_L1D 0
-gpgpu_flush_l1_cache 1
-gpgpu_n_cluster_ejection_buffer_size 32
-gpgpu_l1_cache_write_ratio 25

# shared memory  configuration
-gpgpu_shmem_size 102400
-gpgpu_shmem_sizeDefault 102400
-gpgpu_shmem_per_block 49152
-gpgpu_smem_latency 30
# shared memory bankconflict detection
-gpgpu_shmem_num_banks 32
-gpgpu_shmem_limited_broadcast 0
-gpgpu_shmem_warp_parts 1
-gpgpu_coalesce_arch 89

# L2 cache
-gpgpu_cache:dl2 S:512:128:24,L:B:m:L:P,A:192:4,32:0,32
-gpgpu_cache:dl2_texture_only 0
-gpgpu_dram_partition_queues 64:64:64:64
-gpgpu_perf_sim_memcpy 1
-gpgpu_memory_partition_indexing 2

# 128 KB Inst.
-gpgpu_cache:il1 N:64:128:16,L:R:f:N:L,S:2:48,4
-gpgpu_inst_fetch_throughput 4
# 128 KB Tex
# Note, TEX is deprected since Volta, It is used for legacy apps only. Use L1D cache instead with .nc modifier or __ldg mehtod
-gpgpu_tex_cache:l1 N:4:128:256,L:R:m:N:L,T:512:8,128:2
# 64 KB Const
-gpgpu_const_cache:l1 N:128:64:8,L:R:f:N:L,S:2:64,4
-gpgpu_perfect_inst_const_cache 1

# interconnection
# use built-in local xbar
-network_mode 2
-icnt_in_buffer_limit 512
-icnt_out_buffer_limit 512
-icnt_subnets 2
-icnt_flit_size 40
-icnt_arbiter_algo 1

# memory partition latency config
-gpgpu_l2_rop_latency 247
-dram_latency 330

# dram sched config
-gpgpu_dram_scheduler 1
-gpgpu_frfcfs_dram_sched_queue_size 64
-gpgpu_dram_return_queue_size 192

# dram model config
-gpgpu_n_mem_per_ctrlr 1
-gpgpu_dram_buswidth 2
-gpgpu_dram_burst_length 16
-dram_data_command_freq_ratio 4
-gpgpu_mem_address_mask 1
-gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RBBBCCCC.BCCSSSSS

# Mem timing
-gpgpu_dram_timing_opt nbk=16:CCD=4:RRD=18:RCD=36:RAS=82:RP=36:RC=117:CL=36:WL=12:CDLR=15:WR=36:nbkgrp=4:CCDL=9:RTPL=6
-dram_dual_bus_interface 0

# select lower bits for bnkgrp to increase bnkgrp parallelism
-dram_bnk_indexing_policy 0
-dram_bnkgrp_indexing_policy 1

#-dram_seperate_write_queue_enable 1
#-dram_write_queue_size 64:56:32

# stat collection
-gpgpu_memlatency_stat 14
-gpgpu_runtime_stat 500
-enable_ptx_file_line_stats 1
-visualizer_enabled 0

# power model configs, disable it untill we create a real energy model
-power_simulation_enabled 0

# tracing functionality
#-trace_enabled 1
#-trace_components WARP_SCHEDULER,SCOREBOARD
#-trace_sampling_core 0

And the error I keep running into is this:

**********************************************************
**********************************************************
l1_bw_32f_unroll_large-NO_ARGS--RTX4090-SASS-LINEAR-RR-32B-FCFS. Status=ASSERT, FUNC_TEST_FAILED, ABORTED
Last 10 line of /home/cab-prj/Prabin/december/accel-sim-framework/util/job_launching/../../sim_run_12.4/l1_bw_32f_unroll_large/NO_ARGS/RTX4090-SASS-LINEAR-RR-32B-FCFS/l1_bw_32f_unroll_large-NO_ARGS.accelsim-commit-d7f397a_modified_1.0_25-12-12-10-50-26gpgpu-sim_git-commit-b18ee397_modified_2.0..o34
------------------
launching memcpy command : MemcpyHtoD,0x00007f4e8d602000,528384
GPGPU-Sim uArch: performance model initialization complete.
*** Initializing Memory Statistics ***
GPGPU-Sim uArch: clock periods: 0.00000000044742729306:0.00000000044742729306:0.00000000044742729306:0.00000000019045805161
GPGPU-Sim uArch: clock freqs: 2235000000.000000:2235000000.000000:2235000000.000000:5250500000.000000
sub_partition_id_mask = 0000000000000800
addr_dec_mask[BURST] = 000000000000001f         high:5 low:0
addr_dec_mask[COL]   = 000000000000f61f         high:16 low:0
addr_dec_mask[ROW]   = 00000000fff80000         high:32 low:19
addr_dec_mask[BK]    = 0000000000070800         high:19 low:11
------------------

Contents of /home/cab-prj/Prabin/december/accel-sim-framework/util/job_launching/../../sim_run_12.4/l1_bw_32f_unroll_large/NO_ARGS/RTX4090-SASS-LINEAR-RR-32B-FCFS/l1_bw_32f_unroll_large-NO_ARGS.accelsim-commit-d7f397a_modified_1.0_25-12-12-10-50-26gpgpu-sim_git-commit-b18ee397_modified_2.0..e34
------------------
accel-sim.out: /home/cab-prj/Prabin/december/accel-sim-framework/gpu-simulator/gpgpu-sim/src/gpgpu-sim/hashing.cc:88: unsigned int ipoly_hash_function(new_addr_type, unsigned int, unsigned int): Assertion `"\nmemory_partition_indexing error: The number of " "channels should be " "16, 32 or 64 for the hashing IPOLY index function. other banks " "numbers are not supported. Generate it by yourself! \n" && 0' failed.
/home/cab-prj/Prabin/december/accel-sim-framework/util/job_launching/../../sim_run_12.4/l1_bw_32f_unroll_large/NO_ARGS/RTX4090-SASS-LINEAR-RR-32B-FCFS/slurm.sim: line 54: 16238 Aborted                 (core dumped) /home/cab-prj/Prabin/december/accel-sim-framework/util/job_launching/../../sim_run_12.4/gpgpu-sim-builds/accelsim-commit-d7f397a_modified_1.0_25-12-12-10-50-26gpgpu-sim_git-commit-b18ee397_modified_2.0./accel-sim.out -config ./gpgpusim.config -trace ./traces/kernelslist.g

------------------
**********************************************************```

Is there any solution to this? 

[Ivecia]

Maybe related to #318. I'm also using the tuner for RTX 4090, and receive similar errors. I will look deeper into the source code, which may take several days (from next week).

[JRPan]

The tuner is buggy. Especially the L2 part. There are PRs to fix it. #512 and accel-sim/gpu-app-collection#75. Maybe you can try them out, and let me if they work. Like a reviewer :)

Thanks

[PrabinKuSabat]

RTX4090 Tuning : Workaround

Just an update. I could do perform the tuning part for the default RTX4090-SASS config

../job_launching/run_simulations.py  
-T ../.././hw_run/traces/device-0/12.4/  
-C RTX4090-SASS  
-N tuning3 -B GPU_Microbenchmark 

The workaround is after you generate the config for RTX4090 using

# run the tuner with the stats.txt from the previous step  
./tuner.py -s stats.txt

^Link

STEP 1

Take the config of RTX3070 copy it to the RTX4090 folder and make the changes from RTX 4090, config generated by you, one by one while checking for errors in executing the run_simulations.py script for one of the apps from GPU_Microbenchmark(Context Here).

STEP 2

Primarily the problem is coming from n_mem being 24. Change it to 32. Also the dram_latency in the generated 4090 config was unusually high. You can choose something close to the value in 3070 config.

STEP3

Use the ISA of 3070 (ampere_opcode.h in ISA_def ) for 4090. It works for GPU_Microbenchmark. It might cause some problems for newer/other programs. But for older cuda programs it should work(I'm not sure about this).

STEP 5

Add RTX4090 to config_maps = {} in ./util/plotting/correl_mappings.py.
and include the 4090 ISA created in STEP 3 above to ./gpu-simulator/trace-driven/trace_driven.cc

STEP4

Also do copy the trace.config from 3070 directory, in tested-cfgs folder to the 4090 config folder in same.

IN ONESTEP

• Basically copy the entire 3070 configs directory in ./gpu-simulator/gpgpu-sim/configs/tested-cfgs/ & ./gpu-simulator/configs/tested-cfgs/ and rename it to your desired 4090 config folder name.
• Add it to define-standard-cfgs.yml. Generate the 4090 config using tuner section in accel-sim.
• Copy the config parameters from the newly generated one to the renamed copy_of_3070 config.
  IMPORTANT: Keep n_mem param as 32. Also check for dram_latency.
• Add the ISA support for 4090 by using the ISA file of RTX3070 from ISA_Def folder.
• Add the above support in ./gpu-simulator/trace-driven/trace_driven.cc

---

Yes the tuning for L2 is buggy.

Plotting NVIDIA GeForce RTX 4090 : [L2 Write Hits]
RTX4090-SASS (17 apps, 17 kernels (0 < 1% Err, 15 under, 2 over)) [Correl=-0.1393 Err=96.20%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (17 apps (0 < 1% Err, 15 under, 2 over, 0 < 10% Err)) [Correl=-0.1393 Err=96.20% Agg_Err=99.91% RPD=182.18%,NMSE=2.60]

/home/cab-prj/.conda/envs/accelsim/lib/python3.11/site-packages/numpy/lib/_function_base_impl.py:3045: RuntimeWarning:

invalid value encountered in divide

/home/cab-prj/.conda/envs/accelsim/lib/python3.11/site-packages/numpy/lib/_function_base_impl.py:3046: RuntimeWarning:

invalid value encountered in divide

Plotting NVIDIA GeForce RTX 4090 : [L2 Write Hit Rate]
RTX4090-SASS (17 apps, 17 kernels (0 < 1% Err, 17 under, 0 over)) [Correl=nan Err=91.01%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (17 apps (0 < 1% Err, 17 under, 0 over, 0 < 10% Err)) [Correl=nan Err=91.01% Agg_Err=91.01% RPD=179.16%,NMSE=0.94]

Which results in such outputs in the correlation_plot.

---

Plotting NVIDIA GeForce RTX 4090 : [L1D Read Hits]
RTX4090-SASS-LINEAR-RR-32B-FRFCFS (17 apps, 17 kernels (14 < 1% Err, 2 under, 1 over)) [Correl=0.9497 Err=13.33%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS-LINEAR-RR-32B-FRFCFS (17 apps (13 < 1% Err, 2 under, 2 over, 13 < 10% Err)) [Correl=0.9497 Err=15058823542.74% Agg_Err=99.11% RPD=36.44%,NMSE=2.89]

Also, can you suggest what can be the cause for such error in the correlation_plot or is it because of the buggy tuner?

[Ivecia]

@JRPan Thanks for the update! I will try the PR first and see if it works as expected over the next few days (of next two weeks, maybe?). Additionally, I noticed that the PR uses definations from ctrl0080gr.h of NVIDIA/open-gpu-kernel-modules. This file has been modified across different driver versions, so I am afraid whether these changes might affect the PR (because I can only access servers with a much older version of NVIDIA driver).

@PrabinKuSabat Thanks for sharing. These approaches do help the simulator run properly, but some parameter settings may still lead to relatively large output errors. I also noticed that the tuning results on the V100 differ from those of the tested configuration. I am not sure whether this discrepancy is caused by bugs in the tuner. For now, I will try the PR first and see if it works as expected.

[JRPan]

@cesar-avalos3 Can you check real quick if the definition is the same for older drivers.

[JRPan]

What's your driver version? @Ivecia I checked 515.x and the definition seems to be the same https://github.com/NVIDIA/open-gpu-kernel-modules/blob/1739a20efc4acb55fd1dc53dcc66057b70c2613c/src/common/sdk/nvidia/inc/ctrl/ctrl0080/ctrl0080gr.h#L147

We expect this to be quite stable between versions

[Ivecia]

@JRPan Sadly, various servers with various version (but newer than 515.x). Seems that PR should work well, and let me have a try on the new tuner. Thanks again for your clarification.

[PrabinKuSabat]

Driver Version: 550.54.15
Cuda Version : 12.4

---

Correlation outputs after incorporating PR Changes

• The correlation output after including the PR changes seems to be completely off when it comes to Per-App error calculations. Also, it produces nan outputs for the GPC Cycles.
• L2 errors persist

(base) cab-prj@dmacs-3gpunode:~/Prabin/december2/accel-sim-framework/util/plotting$ ./plot-correlation.py  -c RTX4090-SASS-LINEAR-RR-32B-FRFCFS.csv
-----------------------------------------------------------------
All Card Summary:
HW Summary for NVIDIA GeForce RTX 4090 [Contains 17 Apps]:
----------------------------------------------------------------


Plotting NVIDIA GeForce RTX 4090 : [GPC Cycles]
RTX4090-SASS-LINEAR-RR-32B-FRFCFS (17 apps, 17 kernels (1 < 1% Err, 9 under, 7 over)) [Correl=nan Err=nan%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS-LINEAR-RR-32B-FRFCFS (17 apps (1 < 1% Err, 8 under, 8 over, 4 < 10% Err)) [Correl=nan Err=nan% Agg_Err=0.00% RPD=nan%,NMSE=nan]

Plotting NVIDIA GeForce RTX 4090 : [Warp Instructions]
RTX4090-SASS-LINEAR-RR-32B-FRFCFS (17 apps, 17 kernels (17 < 1% Err, 0 under, 0 over)) [Correl=0.6741 Err=0.08%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS-LINEAR-RR-32B-FRFCFS (17 apps (16 < 1% Err, 4 under, 1 over, 16 < 10% Err)) [Correl=0.6741 Err=1815070117647058.75% Agg_Err=59.63% RPD=11.84%,NMSE=2.46]

Plotting NVIDIA GeForce RTX 4090 : [L2 Read Hits]
RTX4090-SASS-LINEAR-RR-32B-FRFCFS (17 apps, 17 kernels (16 < 1% Err, 1 under, 0 over)) [Correl=0.8889 Err=5.88%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS-LINEAR-RR-32B-FRFCFS (17 apps (15 < 1% Err, 2 under, 1 over, 15 < 10% Err)) [Correl=0.8889 Err=986356764705888.25% Agg_Err=49.99% RPD=23.53%,NMSE=2.06]

Plotting NVIDIA GeForce RTX 4090 : [L2 Reads]
RTX4090-SASS-LINEAR-RR-32B-FRFCFS (14 apps, 14 kernels (13 < 1% Err, 1 under, 0 over)) [Correl=0.9142 Err=0.94%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS-LINEAR-RR-32B-FRFCFS (14 apps (12 < 1% Err, 1 under, 1 over, 12 < 10% Err)) [Correl=0.9142 Err=1199689142857143.75% Agg_Err=29.31% RPD=15.29%,NMSE=1.10]

Plotting NVIDIA GeForce RTX 4090 : [L2 Writes]
RTX4090-SASS-LINEAR-RR-32B-FRFCFS (17 apps, 17 kernels (15 < 1% Err, 0 under, 2 over)) [Correl=0.9997 Err=24.20%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS-LINEAR-RR-32B-FRFCFS (17 apps (14 < 1% Err, 0 under, 4 over, 14 < 10% Err)) [Correl=0.9997 Err=9637647058847.73% Agg_Err=1.78% RPD=23.05%,NMSE=0.07]

Plotting NVIDIA GeForce RTX 4090 : [L2 Write Hits]
RTX4090-SASS-LINEAR-RR-32B-FRFCFS (17 apps, 17 kernels (1 < 1% Err, 14 under, 2 over)) [Correl=-0.1167 Err=93.55%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS-LINEAR-RR-32B-FRFCFS (17 apps (0 < 1% Err, 14 under, 3 over, 0 < 10% Err)) [Correl=-0.1167 Err=666823529505.31% Agg_Err=99.70% RPD=172.80%,NMSE=2.85]

Plotting NVIDIA GeForce RTX 4090 : [L2 Write Hit Rate]
RTX4090-SASS-LINEAR-RR-32B-FRFCFS (17 apps, 17 kernels (1 < 1% Err, 16 under, 0 over)) [Correl=0.06362 Err=81.30%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS-LINEAR-RR-32B-FRFCFS (17 apps (0 < 1% Err, 16 under, 1 over, 0 < 10% Err)) [Correl=0.06362 Err=406996864.39% Agg_Err=86.82% RPD=167.62%,NMSE=0.93]

Plotting NVIDIA GeForce RTX 4090 : [Occupancy]
RTX4090-SASS-LINEAR-RR-32B-FRFCFS (17 apps, 17 kernels (15 < 1% Err, 0 under, 2 over)) [Correl=0.8504 Err=1.35%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS-LINEAR-RR-32B-FRFCFS (17 apps (14 < 1% Err, 7 under, 10 over, 14 < 10% Err)) [Correl=0.8504 Err=3914164707.23% Agg_Err=10.00% RPD=13.04%,NMSE=0.38]

Plotting NVIDIA GeForce RTX 4090 : [L1D Read Hits]
RTX4090-SASS-LINEAR-RR-32B-FRFCFS (17 apps, 17 kernels (17 < 1% Err, 0 under, 0 over)) [Correl=1.0 Err=0.03%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS-LINEAR-RR-32B-FRFCFS (17 apps (17 < 1% Err, 0 under, 1 over, 17 < 10% Err)) [Correl=1.0 Err=0.03% Agg_Err=0.04% RPD=0.03%,NMSE=0.00]

Plotting NVIDIA GeForce RTX 4090 : [L1D Write Hits]
RTX4090-SASS-LINEAR-RR-32B-FRFCFS (17 apps, 17 kernels (13 < 1% Err, 3 under, 1 over)) [Correl=0.9831 Err=13.84%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS-LINEAR-RR-32B-FRFCFS (17 apps (12 < 1% Err, 3 under, 2 over, 13 < 10% Err)) [Correl=0.9831 Err=60235294131.48% Agg_Err=28.61% RPD=29.33%,NMSE=0.70]

Plotting NVIDIA GeForce RTX 4090 : [L1D Read Access]
RTX4090-SASS-LINEAR-RR-32B-FRFCFS (12 apps, 12 kernels (12 < 1% Err, 0 under, 0 over)) [Correl=1.0 Err=0.00%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS-LINEAR-RR-32B-FRFCFS (12 apps (12 < 1% Err, 0 under, 0 over, 12 < 10% Err)) [Correl=1.0 Err=0.00% Agg_Err=0.00% RPD=0.00%,NMSE=0.00]

Plotting NVIDIA GeForce RTX 4090 : [L1D Write Access]
RTX4090-SASS-LINEAR-RR-32B-FRFCFS (17 apps, 17 kernels (17 < 1% Err, 0 under, 0 over)) [Correl=0.9997 Err=0.00%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS-LINEAR-RR-32B-FRFCFS (17 apps (16 < 1% Err, 0 under, 1 over, 16 < 10% Err)) [Correl=0.9997 Err=9637647058823.53% Agg_Err=1.69% RPD=11.76%,NMSE=0.07]

Plotting NVIDIA GeForce RTX 4090 : [DRAM Reads]
RTX4090-SASS-LINEAR-RR-32B-FRFCFS (17 apps, 17 kernels (4 < 1% Err, 12 under, 1 over)) [Correl=1.0 Err=33.80%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS-LINEAR-RR-32B-FRFCFS (17 apps (3 < 1% Err, 15 under, 2 over, 8 < 10% Err)) [Correl=1.0 Err=1459529411798.51% Agg_Err=0.15% RPD=71.40%,NMSE=0.00]

Plotting NVIDIA GeForce RTX 4090 : [DRAM Writes]
RTX4090-SASS-LINEAR-RR-32B-FRFCFS (17 apps, 17 kernels (15 < 1% Err, 0 under, 2 over)) [Correl=0.9958 Err=2.14%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS-LINEAR-RR-32B-FRFCFS (17 apps (13 < 1% Err, 0 under, 4 over, 14 < 10% Err)) [Correl=0.9958 Err=7988000000002.14% Agg_Err=18.97% RPD=25.45%,NMSE=0.56]

Combined per-kernel output available at: file:///home/cab-prj/Prabin/december2/accel-sim-framework/util/plotting/correl-html/combined_per_kernel.html
Combined per-app output available at: file:///home/cab-prj/Prabin/december2/accel-sim-framework/util/plotting/correl-html/combined_per_app.html

Config Used

(base) cab-prj@dmacs-3gpunode:~/Prabin/december2/accel-sim-framework/util/plotting$ cat ../../gpu-simulator/gpgpu-sim/configs/tested-cfgs/NVIDIA_GeForce_RTX_4090/gpgpusim.config
# functional simulator specification
-gpgpu_ptx_instruction_classification 0
-gpgpu_ptx_sim_mode 0
-gpgpu_ptx_force_max_capability 89

# Device Limits
-gpgpu_stack_size_limit 1024
-gpgpu_heap_size_limit 8388608
-gpgpu_runtime_sync_depth_limit 2
-gpgpu_runtime_pending_launch_count_limit 2048
-gpgpu_kernel_launch_latency 5000
-gpgpu_TB_launch_latency 0
-gpgpu_max_concurrent_kernel 128

# Compute Capability
-gpgpu_compute_capability_major 8
-gpgpu_compute_capability_minor 9

# PTX execution-driven
-gpgpu_ptx_convert_to_ptxplus 0
-gpgpu_ptx_save_converted_ptxplus 0

# high level architecture configuration
-gpgpu_n_clusters 128
-gpgpu_n_cores_per_cluster 1
-gpgpu_n_mem 16
-gpgpu_n_sub_partition_per_mchannel 2

# clock domains
#-gpgpu_clock_domains <Core Clock>:<Interconnect Clock>:<L2 Clock>:<DRAM Clock>
-gpgpu_clock_domains 1132:1132:1132:3500.5

# shader core pipeline config
-gpgpu_shader_registers 65536
-gpgpu_registers_per_block 65536
-gpgpu_occupancy_sm_number 89

-gpgpu_shader_core_pipeline 1536:32
-gpgpu_shader_cta 32
-gpgpu_simd_model 1

# Pipeline widths and number of FUs
# ID_OC_SP,ID_OC_DP,ID_OC_INT,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_DP,OC_EX_INT,OC_EX_SFU,OC_EX_MEM,EX_WB,ID_OC_TENSOR_CORE,OC_EX_TENSOR_CORE
-gpgpu_pipeline_widths 4,4,4,4,4,4,4,4,4,4,8,4,4
-gpgpu_num_sp_units 4
-gpgpu_num_sfu_units 4
-gpgpu_num_dp_units 4
-gpgpu_num_int_units 4
-gpgpu_tensor_core_avail 1
-gpgpu_num_tensor_core_units 4

# Instruction latencies and initiation intervals
# "ADD,MAX,MUL,MAD,DIV"
# All Div operations are executed on SFU unit
-ptx_opcode_latency_int 4,4,4,4,21
-ptx_opcode_initiation_int 2,2,2,2,2
-ptx_opcode_latency_fp 4,4,4,4,39
-ptx_opcode_initiation_fp 2,2,2,2,4
-ptx_opcode_latency_dp 54,54,54,54,330
-ptx_opcode_initiation_dp 64,64,64,64,130
-ptx_opcode_latency_sfu 23
-ptx_opcode_initiation_sfu 8
-ptx_opcode_latency_tesnor 64
-ptx_opcode_initiation_tensor 64

# sub core model: in which each scheduler has its own register file and EUs
# i.e. schedulers are isolated
-gpgpu_sub_core_model 1
# disable specialized operand collectors and use generic operand collectors instead
-gpgpu_enable_specialized_operand_collector 0
-gpgpu_operand_collector_num_units_gen 8
-gpgpu_operand_collector_num_in_ports_gen 8
-gpgpu_operand_collector_num_out_ports_gen 8
# register banks
-gpgpu_num_reg_banks 8
-gpgpu_reg_file_port_throughput 2

# warp scheduling
-gpgpu_num_sched_per_core 4
-gpgpu_scheduler gto
# a warp scheduler issue mode
-gpgpu_max_insn_issue_per_warp 1
-gpgpu_dual_issue_diff_exec_units 1

## L1/shared memory configuration
# <sector?>:<nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_alloc>:<set_index_fn>,<mshr>:<N>:<merge>,<mq>:**<fifo_entry>,<data_port_width>
# ** Optional parameter - Required when mshr_type==Texture Fifo, set to 0 if not used
# In adaptive cache, we adaptively assign the remaining shared memory to L1 cache
# For more info, see https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#shared-memory-7-x
-gpgpu_adaptive_cache_config 1
-gpgpu_shmem_option 0,8,16,32,64,100
-gpgpu_unified_l1d_size 128
# L1 cache configuration
-gpgpu_l1_banks 4
-gpgpu_cache:dl1 S:4:128:256,L:T:m:L:L,A:384:48,16:0,32
-gpgpu_l1_latency 39
-gpgpu_gmem_skip_L1D 0
-gpgpu_flush_l1_cache 1
-gpgpu_n_cluster_ejection_buffer_size 32
-gpgpu_l1_cache_write_ratio 25

# shared memory  configuration
-gpgpu_shmem_size 102400
-gpgpu_shmem_sizeDefault 102400
-gpgpu_shmem_per_block 49152
-gpgpu_smem_latency 29
# shared memory bankconflict detection
-gpgpu_shmem_num_banks 32
-gpgpu_shmem_limited_broadcast 0
-gpgpu_shmem_warp_parts 1
-gpgpu_coalesce_arch 89

# L2 cache
-gpgpu_cache:dl2 S:64:128:32,L:B:m:L:P,A:192:4,32:0,32
-gpgpu_cache:dl2_texture_only 0
-gpgpu_dram_partition_queues 64:64:64:64
-gpgpu_perf_sim_memcpy 1
-gpgpu_memory_partition_indexing 2

# 128 KB Inst.
-gpgpu_cache:il1 N:64:128:16,L:R:f:N:L,S:2:48,4
-gpgpu_inst_fetch_throughput 4
# 128 KB Tex
# Note, TEX is deprected since Volta, It is used for legacy apps only. Use L1D cache instead with .nc modifier or __ldg mehtod
-gpgpu_tex_cache:l1 N:4:128:256,L:R:m:N:L,T:512:8,128:2
# 64 KB Const
-gpgpu_const_cache:l1 N:128:64:8,L:R:f:N:L,S:2:64,4
-gpgpu_perfect_inst_const_cache 1

# interconnection
# use built-in local xbar
-network_mode 2
-icnt_in_buffer_limit 512
-icnt_out_buffer_limit 512
-icnt_subnets 2
-icnt_flit_size 40
-icnt_arbiter_algo 1

# memory partition latency config
-gpgpu_l2_rop_latency 200
-dram_latency 254

# dram sched config
-gpgpu_dram_scheduler 1
-gpgpu_frfcfs_dram_sched_queue_size 64
-gpgpu_dram_return_queue_size 192

# dram model config
-gpgpu_n_mem_per_ctrlr 1
-gpgpu_dram_buswidth 2
-gpgpu_dram_burst_length 16
-dram_data_command_freq_ratio 4
-gpgpu_mem_address_mask 1
-gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RBBBCCCC.BCCSSSSS

# Mem timing
-gpgpu_dram_timing_opt nbk=16:CCD=4:RRD=12:RCD=24:RAS=55:RP=24:RC=78:CL=24:WL=8:CDLR=10:WR=24:nbkgrp=4:CCDL=6:RTPL=4
-dram_dual_bus_interface 0

# select lower bits for bnkgrp to increase bnkgrp parallelism
-dram_bnk_indexing_policy 0
-dram_bnkgrp_indexing_policy 1

#-dram_seperate_write_queue_enable 1
#-dram_write_queue_size 64:56:32

# stat collection
-gpgpu_memlatency_stat 14
-gpgpu_runtime_stat 500
-enable_ptx_file_line_stats 1
-visualizer_enabled 0

# power model configs, disable it untill we create a real energy model
-power_simulation_enabled 0

# tracing functionality
#-trace_enabled 1
#-trace_components WARP_SCHEDULER,SCOREBOARD
#-trace_sampling_core 0

---

PS:

I believe the errors are result of my bad config file. I will modify the cache:dl1 and cache:dl2 parameters and update the comment based on the outputs obtained.

[JRPan]

You need to recollect hw stats. The parameters are changed so the scripts were not able to pick up the correct hw stats.

At a glance the config file seems correct.

[JRPan]

Actually L2 is wrong.

-gpgpu_n_mem 16
-gpgpu_n_sub_partition_per_mchannel 2

This correctly matches whitepaper, but the bandwidth is wrong. This is 2x16=32 banks, each band 32B so 1024B per cycle. Please check the whitepaper for the correct BW and scale the n_mem.

dl2 config is 64:128:32 which is 256KB. This is per bank. Times 32 bank, this is 8MB L2. Please double check whitepaper if this is correct.

Then when you scale n_mem, reduce dl2 config correctly do that the capacity stays the same.

[PrabinKuSabat]

So I have made few changes in the gpgpusim.config.

-gpgpu_n_mem 12
-gpgpu_n_sub_partition_per_mchannel 2

-gpgpu_l1_banks 4
-gpgpu_cache:dl1 S:4:128:256,L:T:m:L:L,A:384:48,16:0,32

-gpgpu_cache:dl2 S:1024:128:24,L:B:m:L:L,A:192:16,32:0,32

Which gives us 1024*128*24*12*2 = 72MB

---

Correlation outputs after making the PR changes

The L2 error seems to be still affected by the buggy tuner.

Is true that the L here in L,A:192:16,32:0,32(from -gpgpu_cache:dl2) mean the simulator doesn't use the IPOLY hashing function ( instead uses Linear pass).
If it is , then what can the reason behind both the correlation looking exact?

@Ivecia Can u help?

• I had renamed the sim_run folder before running the second simulation.

For -gpgpu_cache:dl2 S:1024:128:24,L:B:m:L:L,A:192:16,32:0,32

-----------------------------------------------------------------
All Card Summary:
HW Summary for NVIDIA GeForce RTX 4090 [Contains 17 Apps]:
----------------------------------------------------------------


/home/cab-prj/.conda/envs/accelsim/lib/python3.11/site-packages/numpy/lib/_function_base_impl.py:3045: RuntimeWarning:

invalid value encountered in divide

/home/cab-prj/.conda/envs/accelsim/lib/python3.11/site-packages/numpy/lib/_function_base_impl.py:3046: RuntimeWarning:

invalid value encountered in divide

Plotting NVIDIA GeForce RTX 4090 : [GPC Cycles]
RTX4090-SASS (12 apps, 12 kernels (0 < 1% Err, 7 under, 5 over)) [Correl=0.9653 Err=30.29%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (12 apps (0 < 1% Err, 7 under, 5 over, 2 < 10% Err)) [Correl=0.9653 Err=30.29% Agg_Err=28.37% RPD=31.15%,NMSE=0.51]

Plotting NVIDIA GeForce RTX 4090 : [Warp Instructions]
RTX4090-SASS (12 apps, 12 kernels (12 < 1% Err, 0 under, 0 over)) [Correl=1.0 Err=0.08%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (12 apps (12 < 1% Err, 3 under, 0 over, 12 < 10% Err)) [Correl=1.0 Err=0.08% Agg_Err=0.07% RPD=0.08%,NMSE=0.00]

Plotting NVIDIA GeForce RTX 4090 : [L2 Read Hits]
RTX4090-SASS (12 apps, 12 kernels (11 < 1% Err, 0 under, 1 over)) [Correl=1.0 Err=184.03%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (12 apps (11 < 1% Err, 0 under, 1 over, 11 < 10% Err)) [Correl=1.0 Err=184.03% Agg_Err=2208.32% RPD=15.28%,NMSE=76.50]

Plotting NVIDIA GeForce RTX 4090 : [L2 Reads]
RTX4090-SASS (9 apps, 9 kernels (8 < 1% Err, 0 under, 1 over)) [Correl=0.7136 Err=32.27%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (9 apps (8 < 1% Err, 0 under, 1 over, 8 < 10% Err)) [Correl=0.7136 Err=32.27% Agg_Err=59.19% RPD=13.16%,NMSE=1.78]

Plotting NVIDIA GeForce RTX 4090 : [L2 Writes]
RTX4090-SASS (12 apps, 12 kernels (11 < 1% Err, 0 under, 1 over)) [Correl=0.9899 Err=24.91%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (12 apps (11 < 1% Err, 0 under, 1 over, 11 < 10% Err)) [Correl=0.9899 Err=24.91% Agg_Err=8.55% RPD=9.99%,NMSE=0.30]

Plotting NVIDIA GeForce RTX 4090 : [L2 Write Hits]
RTX4090-SASS (12 apps, 12 kernels (0 < 1% Err, 11 under, 1 over)) [Correl=-0.1024 Err=105.24%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (12 apps (0 < 1% Err, 11 under, 1 over, 0 < 10% Err)) [Correl=-0.1024 Err=105.24% Agg_Err=102.63% RPD=182.36%,NMSE=2.25]

/home/cab-prj/.conda/envs/accelsim/lib/python3.11/site-packages/numpy/lib/_function_base_impl.py:3045: RuntimeWarning:

invalid value encountered in divide

/home/cab-prj/.conda/envs/accelsim/lib/python3.11/site-packages/numpy/lib/_function_base_impl.py:3046: RuntimeWarning:

invalid value encountered in divide

Plotting NVIDIA GeForce RTX 4090 : [L2 Write Hit Rate]
RTX4090-SASS (12 apps, 12 kernels (0 < 1% Err, 12 under, 0 over)) [Correl=nan Err=90.73%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (12 apps (0 < 1% Err, 12 under, 0 over, 0 < 10% Err)) [Correl=nan Err=90.73% Agg_Err=90.73% RPD=176.43%,NMSE=0.93]

Plotting NVIDIA GeForce RTX 4090 : [Occupancy]
RTX4090-SASS (12 apps, 12 kernels (10 < 1% Err, 0 under, 2 over)) [Correl=0.9991 Err=1.79%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (12 apps (10 < 1% Err, 7 under, 5 over, 11 < 10% Err)) [Correl=0.9991 Err=1.79% Agg_Err=1.05% RPD=1.70%,NMSE=0.03]

Plotting NVIDIA GeForce RTX 4090 : [L1D Read Hits]
RTX4090-SASS (12 apps, 12 kernels (11 < 1% Err, 1 under, 0 over)) [Correl=0.9998 Err=0.90%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (12 apps (11 < 1% Err, 1 under, 0 over, 11 < 10% Err)) [Correl=0.9998 Err=0.90% Agg_Err=0.92% RPD=0.95%,NMSE=0.03]

Plotting NVIDIA GeForce RTX 4090 : [L1D Write Hits]
RTX4090-SASS (12 apps, 12 kernels (11 < 1% Err, 1 under, 0 over)) [Correl=1.0 Err=1.67%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (12 apps (11 < 1% Err, 1 under, 0 over, 11 < 10% Err)) [Correl=1.0 Err=1.67% Agg_Err=19.95% RPD=1.85%,NMSE=0.69]

Plotting NVIDIA GeForce RTX 4090 : [L1D Read Access]
RTX4090-SASS (10 apps, 10 kernels (10 < 1% Err, 0 under, 0 over)) [Correl=1.0 Err=0.00%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (10 apps (10 < 1% Err, 0 under, 0 over, 10 < 10% Err)) [Correl=1.0 Err=0.00% Agg_Err=0.00% RPD=0.00%,NMSE=0.00]

Plotting NVIDIA GeForce RTX 4090 : [L1D Write Access]
RTX4090-SASS (12 apps, 12 kernels (12 < 1% Err, 0 under, 0 over)) [Correl=1.0 Err=0.00%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (12 apps (12 < 1% Err, 0 under, 0 over, 12 < 10% Err)) [Correl=1.0 Err=0.00% Agg_Err=0.00% RPD=0.00%,NMSE=0.00]

Plotting NVIDIA GeForce RTX 4090 : [DRAM Reads]
RTX4090-SASS (12 apps, 12 kernels (2 < 1% Err, 10 under, 0 over)) [Correl=1.0 Err=35.75%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (12 apps (2 < 1% Err, 12 under, 0 over, 6 < 10% Err)) [Correl=1.0 Err=35.75% Agg_Err=2.37% RPD=62.97%,NMSE=0.02]

/home/cab-prj/.conda/envs/accelsim/lib/python3.11/site-packages/numpy/lib/_function_base_impl.py:3045: RuntimeWarning:

invalid value encountered in divide

Plotting NVIDIA GeForce RTX 4090 : [DRAM Writes]
RTX4090-SASS (12 apps, 12 kernels (12 < 1% Err, 0 under, 0 over)) [Correl=nan Err=0.00%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (12 apps (12 < 1% Err, 0 under, 0 over, 12 < 10% Err)) [Correl=nan Err=0.00% Agg_Err=0.00% RPD=0.00%,NMSE=0.00]

For -gpgpu_cache:dl2 S:1024:128:24,L:B:m:L:P,A:192:16,32:0,32

-----------------------------------------------------------------
All Card Summary:
HW Summary for NVIDIA GeForce RTX 4090 [Contains 17 Apps]:
----------------------------------------------------------------


/home/cab-prj/.conda/envs/accelsim/lib/python3.11/site-packages/numpy/lib/_function_base_impl.py:3045: RuntimeWarning:

invalid value encountered in divide

/home/cab-prj/.conda/envs/accelsim/lib/python3.11/site-packages/numpy/lib/_function_base_impl.py:3046: RuntimeWarning:

invalid value encountered in divide

Plotting NVIDIA GeForce RTX 4090 : [GPC Cycles]
RTX4090-SASS (12 apps, 12 kernels (0 < 1% Err, 7 under, 5 over)) [Correl=0.9653 Err=30.29%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (12 apps (0 < 1% Err, 7 under, 5 over, 2 < 10% Err)) [Correl=0.9653 Err=30.29% Agg_Err=28.37% RPD=31.15%,NMSE=0.51]

Plotting NVIDIA GeForce RTX 4090 : [Warp Instructions]
RTX4090-SASS (12 apps, 12 kernels (12 < 1% Err, 0 under, 0 over)) [Correl=1.0 Err=0.08%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (12 apps (12 < 1% Err, 3 under, 0 over, 12 < 10% Err)) [Correl=1.0 Err=0.08% Agg_Err=0.07% RPD=0.08%,NMSE=0.00]

Plotting NVIDIA GeForce RTX 4090 : [L2 Read Hits]
RTX4090-SASS (12 apps, 12 kernels (11 < 1% Err, 0 under, 1 over)) [Correl=1.0 Err=184.03%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (12 apps (11 < 1% Err, 0 under, 1 over, 11 < 10% Err)) [Correl=1.0 Err=184.03% Agg_Err=2208.32% RPD=15.28%,NMSE=76.50]

Plotting NVIDIA GeForce RTX 4090 : [L2 Reads]
RTX4090-SASS (9 apps, 9 kernels (8 < 1% Err, 0 under, 1 over)) [Correl=0.7136 Err=32.27%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (9 apps (8 < 1% Err, 0 under, 1 over, 8 < 10% Err)) [Correl=0.7136 Err=32.27% Agg_Err=59.19% RPD=13.16%,NMSE=1.78]

Plotting NVIDIA GeForce RTX 4090 : [L2 Writes]
RTX4090-SASS (12 apps, 12 kernels (11 < 1% Err, 0 under, 1 over)) [Correl=0.9899 Err=24.91%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (12 apps (11 < 1% Err, 0 under, 1 over, 11 < 10% Err)) [Correl=0.9899 Err=24.91% Agg_Err=8.55% RPD=9.99%,NMSE=0.30]

Plotting NVIDIA GeForce RTX 4090 : [L2 Write Hits]
RTX4090-SASS (12 apps, 12 kernels (0 < 1% Err, 11 under, 1 over)) [Correl=-0.1024 Err=105.24%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (12 apps (0 < 1% Err, 11 under, 1 over, 0 < 10% Err)) [Correl=-0.1024 Err=105.24% Agg_Err=102.63% RPD=182.36%,NMSE=2.25]

/home/cab-prj/.conda/envs/accelsim/lib/python3.11/site-packages/numpy/lib/_function_base_impl.py:3045: RuntimeWarning:

invalid value encountered in divide

/home/cab-prj/.conda/envs/accelsim/lib/python3.11/site-packages/numpy/lib/_function_base_impl.py:3046: RuntimeWarning:

invalid value encountered in divide

Plotting NVIDIA GeForce RTX 4090 : [L2 Write Hit Rate]
RTX4090-SASS (12 apps, 12 kernels (0 < 1% Err, 12 under, 0 over)) [Correl=nan Err=90.73%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (12 apps (0 < 1% Err, 12 under, 0 over, 0 < 10% Err)) [Correl=nan Err=90.73% Agg_Err=90.73% RPD=176.43%,NMSE=0.93]

Plotting NVIDIA GeForce RTX 4090 : [Occupancy]
RTX4090-SASS (12 apps, 12 kernels (10 < 1% Err, 0 under, 2 over)) [Correl=0.9991 Err=1.79%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (12 apps (10 < 1% Err, 7 under, 5 over, 11 < 10% Err)) [Correl=0.9991 Err=1.79% Agg_Err=1.05% RPD=1.70%,NMSE=0.03]

Plotting NVIDIA GeForce RTX 4090 : [L1D Read Hits]
RTX4090-SASS (12 apps, 12 kernels (11 < 1% Err, 1 under, 0 over)) [Correl=0.9998 Err=0.90%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (12 apps (11 < 1% Err, 1 under, 0 over, 11 < 10% Err)) [Correl=0.9998 Err=0.90% Agg_Err=0.92% RPD=0.95%,NMSE=0.03]

Plotting NVIDIA GeForce RTX 4090 : [L1D Write Hits]
RTX4090-SASS (12 apps, 12 kernels (11 < 1% Err, 1 under, 0 over)) [Correl=1.0 Err=1.67%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (12 apps (11 < 1% Err, 1 under, 0 over, 11 < 10% Err)) [Correl=1.0 Err=1.67% Agg_Err=19.95% RPD=1.85%,NMSE=0.69]

Plotting NVIDIA GeForce RTX 4090 : [L1D Read Access]
RTX4090-SASS (10 apps, 10 kernels (10 < 1% Err, 0 under, 0 over)) [Correl=1.0 Err=0.00%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (10 apps (10 < 1% Err, 0 under, 0 over, 10 < 10% Err)) [Correl=1.0 Err=0.00% Agg_Err=0.00% RPD=0.00%,NMSE=0.00]

Plotting NVIDIA GeForce RTX 4090 : [L1D Write Access]
RTX4090-SASS (12 apps, 12 kernels (12 < 1% Err, 0 under, 0 over)) [Correl=1.0 Err=0.00%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (12 apps (12 < 1% Err, 0 under, 0 over, 12 < 10% Err)) [Correl=1.0 Err=0.00% Agg_Err=0.00% RPD=0.00%,NMSE=0.00]

Plotting NVIDIA GeForce RTX 4090 : [DRAM Reads]
RTX4090-SASS (12 apps, 12 kernels (2 < 1% Err, 10 under, 0 over)) [Correl=1.0 Err=35.75%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (12 apps (2 < 1% Err, 12 under, 0 over, 6 < 10% Err)) [Correl=1.0 Err=35.75% Agg_Err=2.37% RPD=62.97%,NMSE=0.02]

The updated IPOLY Hash function from gpgpu-sim/hashing.cc

• Added equations for 128, 256, 512, 1024, 2048 banks

unsigned ipoly_hash_function(new_addr_type higher_bits, unsigned index,
                             unsigned bank_set_num) {
  /*
   * Set Indexing function from "Pseudo-randomly interleaved memory."
   * Rau, B. R et al.
   * ISCA 1991
   * [http://citeseerx.ist.psu.edu/viewdoc/download];jsessionid=348DEA37A3E440473B3C075EAABC63B6?doi=10.1.1.12.7149&rep=rep1&type=pdf
   *
   * equations are corresponding to IPOLY(37) and are adopted from:
   * "Sacat: streaming-aware conflict-avoiding thrashing-resistant gpgpu
   * cache management scheme." Khairy et al. IEEE TPDS 2017.
   *
   * equations for 16 banks are corresponding to IPOLY(5)
   * equations for 32 banks are corresponding to IPOLY(37)
   * equations for 64 banks are corresponding to IPOLY(67)
   * equations for 128 banks are corresponding to IPOLY(127)
   * equations for 256 banks are corresponding to IPOLY(285)
   * equations for 512 banks are corresponding to IPOLY(9)
   * equations for 1024 banks are corresponding to IPOLY(10)
   * equations for 2048 banks are corresponding to IPOLY(11)
   *
   * To see all the IPOLY equations for all the degrees, see
   * http://wireless-systems.ece.gatech.edu/6604/handouts/Peterson's%20Table.pdf
   *
   * We generate these equations using GF(2) arithmetic:
   * [http://www.ee.unb.ca/cgi-bin/tervo/calc.pl?num=&den=&f=d&e=1&m=1](http://www.ee.unb.ca/cgi-bin/tervo/calc.pl?num=&den=&f=d&e=1&m=1)
   *
   * We go through all the strides 128 (10000000), 256 (100000000),...  and
   * do modular arithmetic in GF(2) Then, we create the H-matrix and group
   * each bit together, for more info read the ISCA 1991 paper
   *
   * IPOLY hashing guarantees conflict-free for all 2^n strides which widely
   * exit in GPGPU applications and also show good performance for other
   * strides.
   */

  if (bank_set_num == 16) {
    std::bitset<64> a(higher_bits);
    std::bitset<4> b(index);
    std::bitset<4> new_index(index);

    new_index[0] =
        a[11] ^ a[10] ^ a[9] ^ a[8] ^ a[6] ^ a[4] ^ a[3] ^ a[0] ^ b[0];
    new_index[1] =
        a[12] ^ a[8] ^ a[7] ^ a[6] ^ a[5] ^ a[3] ^ a[1] ^ a[0] ^ b[1];
    new_index[2] = a[9] ^ a[8] ^ a[7] ^ a[6] ^ a[4] ^ a[2] ^ a[1] ^ b[2];
    new_index[3] = a[10] ^ a[9] ^ a[8] ^ a[7] ^ a[5] ^ a[3] ^ a[2] ^ b[3];

    return new_index.to_ulong();

  } else if (bank_set_num == 32) {
    std::bitset<64> a(higher_bits);
    std::bitset<5> b(index);
    std::bitset<5> new_index(index);

    new_index[0] =
        a[13] ^ a[12] ^ a[11] ^ a[10] ^ a[9] ^ a[6] ^ a[5] ^ a[3] ^ a[0] ^ b[0];
    new_index[1] = a[14] ^ a[13] ^ a[12] ^ a[11] ^ a[10] ^ a[7] ^ a[6] ^ a[4] ^
                   a[1] ^ b[1];
    new_index[2] =
        a[14] ^ a[10] ^ a[9] ^ a[8] ^ a[7] ^ a[6] ^ a[3] ^ a[2] ^ a[0] ^ b[2];
    new_index[3] =
        a[11] ^ a[10] ^ a[9] ^ a[8] ^ a[7] ^ a[4] ^ a[3] ^ a[1] ^ b[3];
    new_index[4] =
        a[12] ^ a[11] ^ a[10] ^ a[9] ^ a[8] ^ a[5] ^ a[4] ^ a[2] ^ b[4];
    return new_index.to_ulong();

  } else if (bank_set_num == 64) {
    std::bitset<64> a(higher_bits);
    std::bitset<6> b(index);
    std::bitset<6> new_index(index);

    new_index[0] = a[18] ^ a[17] ^ a[16] ^ a[15] ^ a[12] ^ a[10] ^ a[6] ^ a[5] ^
                   a[0] ^ b[0];
    new_index[1] = a[15] ^ a[13] ^ a[12] ^ a[11] ^ a[10] ^ a[7] ^ a[5] ^ a[1] ^
                   a[0] ^ b[1];
    new_index[2] = a[16] ^ a[14] ^ a[13] ^ a[12] ^ a[11] ^ a[8] ^ a[6] ^ a[2] ^
                   a[1] ^ b[2];
    new_index[3] = a[17] ^ a[15] ^ a[14] ^ a[13] ^ a[12] ^ a[9] ^ a[7] ^ a[3] ^
                   a[2] ^ b[3];
    new_index[4] = a[18] ^ a[16] ^ a[15] ^ a[14] ^ a[13] ^ a[10] ^ a[8] ^ a[4] ^
                   a[3] ^ b[4];
    new_index[5] =
        a[17] ^ a[16] ^ a[15] ^ a[14] ^ a[11] ^ a[9] ^ a[5] ^ a[4] ^ b[5];
    return new_index.to_ulong();

  } else if (bank_set_num == 128) {
    // IPOLY(127) - Degree 7 polynomial
    // Primitive polynomial: x^7 + x^3 + 1
    std::bitset<64> a(higher_bits);
    std::bitset<7> b(index);
    std::bitset<7> new_index(index);

    // XOR equations derived from H-matrix of degree-7 irreducible polynomial
    // Using GF(2) arithmetic for 128 = 2^7 conflict-free strides
    new_index[0] = a[20] ^ a[19] ^ a[18] ^ a[17] ^ a[14] ^ a[12] ^ a[8] ^ a[7] ^
                   a[0] ^ b[0];
    new_index[1] = a[17] ^ a[15] ^ a[14] ^ a[13] ^ a[12] ^ a[9] ^ a[7] ^ a[2] ^
                   a[1] ^ b[1];
    new_index[2] = a[18] ^ a[16] ^ a[15] ^ a[14] ^ a[13] ^ a[10] ^ a[8] ^ a[3] ^
                   a[2] ^ b[2];
    new_index[3] = a[19] ^ a[17] ^ a[16] ^ a[15] ^ a[14] ^ a[11] ^ a[9] ^ a[4] ^
                   a[3] ^ b[3];
    new_index[4] = a[20] ^ a[18] ^ a[17] ^ a[16] ^ a[15] ^ a[12] ^ a[10] ^ a[5] ^
                   a[4] ^ b[4];
    new_index[5] = a[19] ^ a[18] ^ a[17] ^ a[16] ^ a[13] ^ a[11] ^ a[6] ^ a[5] ^
                   b[5];
    new_index[6] = a[20] ^ a[19] ^ a[18] ^ a[17] ^ a[14] ^ a[12] ^ a[7] ^ a[6] ^
                   b[6];

    return new_index.to_ulong();

  } else if (bank_set_num == 256) {
    // IPOLY(285) - Degree 8 polynomial
    // Primitive polynomial: x^8 + x^4 + x^3 + x + 1 (or similar)
    std::bitset<64> a(higher_bits);
    std::bitset<8> b(index);
    std::bitset<8> new_index(index);

    // XOR equations for 256 = 2^8 conflict-free strides
    new_index[0] = a[22] ^ a[21] ^ a[20] ^ a[19] ^ a[16] ^ a[14] ^ a[10] ^ a[9] ^
                   a[0] ^ b[0];
    new_index[1] = a[19] ^ a[17] ^ a[16] ^ a[15] ^ a[14] ^ a[11] ^ a[9] ^ a[3] ^
                   a[2] ^ b[1];
    new_index[2] = a[20] ^ a[18] ^ a[17] ^ a[16] ^ a[15] ^ a[12] ^ a[10] ^ a[4] ^
                   a[3] ^ b[2];
    new_index[3] = a[21] ^ a[19] ^ a[18] ^ a[17] ^ a[16] ^ a[13] ^ a[11] ^ a[5] ^
                   a[4] ^ b[3];
    new_index[4] = a[22] ^ a[20] ^ a[19] ^ a[18] ^ a[17] ^ a[14] ^ a[12] ^ a[6] ^
                   a[5] ^ b[4];
    new_index[5] = a[21] ^ a[20] ^ a[19] ^ a[18] ^ a[15] ^ a[13] ^ a[7] ^ a[6] ^
                   b[5];
    new_index[6] = a[22] ^ a[21] ^ a[20] ^ a[19] ^ a[16] ^ a[14] ^ a[8] ^ a[7] ^
                   b[6];
    new_index[7] = a[22] ^ a[21] ^ a[20] ^ a[17] ^ a[15] ^ a[9] ^ a[8] ^ b[7];

    return new_index.to_ulong();

  } else if (bank_set_num == 512) {
    // IPOLY(9) - Degree 9 polynomial
    // Primitive polynomial: x^9 + x^4 + 1
    // This provides conflict-free access for 2^9 = 512 strides
    std::bitset<64> a(higher_bits);
    std::bitset<9> b(index);
    std::bitset<9> new_index(index);

    // XOR equations derived from H-matrix of degree-9 irreducible polynomial
    // Computed via GF(2) arithmetic to guarantee conflict-free stride patterns
    new_index[0] = a[24] ^ a[23] ^ a[22] ^ a[21] ^ a[18] ^ a[16] ^ a[12] ^ a[11] ^
                   a[0] ^ b[0];
    new_index[1] = a[21] ^ a[19] ^ a[18] ^ a[17] ^ a[16] ^ a[13] ^ a[11] ^ a[4] ^
                   a[3] ^ b[1];
    new_index[2] = a[22] ^ a[20] ^ a[19] ^ a[18] ^ a[17] ^ a[14] ^ a[12] ^ a[5] ^
                   a[4] ^ b[2];
    new_index[3] = a[23] ^ a[21] ^ a[20] ^ a[19] ^ a[18] ^ a[15] ^ a[13] ^ a[6] ^
                   a[5] ^ b[3];
    new_index[4] = a[24] ^ a[22] ^ a[21] ^ a[20] ^ a[19] ^ a[16] ^ a[14] ^ a[7] ^
                   a[6] ^ b[4];
    new_index[5] = a[23] ^ a[22] ^ a[21] ^ a[20] ^ a[17] ^ a[15] ^ a[8] ^ a[7] ^
                   b[5];
    new_index[6] = a[24] ^ a[23] ^ a[22] ^ a[21] ^ a[18] ^ a[16] ^ a[9] ^ a[8] ^
                   b[6];
    new_index[7] = a[24] ^ a[23] ^ a[22] ^ a[19] ^ a[17] ^ a[10] ^ a[9] ^ b[7];
    new_index[8] = a[24] ^ a[23] ^ a[20] ^ a[18] ^ a[11] ^ a[10] ^ b[8];

    return new_index.to_ulong();

  } else if (bank_set_num == 1024) {
    // IPOLY(10) - Degree 10 polynomial
    // Primitive polynomial: x^10 + x^3 + 1
    // This provides conflict-free access for 2^10 = 1024 strides
    std::bitset<64> a(higher_bits);
    std::bitset<10> b(index);
    std::bitset<10> new_index(index);

    // XOR equations derived from H-matrix of degree-10 irreducible polynomial
    new_index[0] = a[25] ^ a[24] ^ a[23] ^ a[22] ^ a[19] ^ a[17] ^ a[13] ^ a[12] ^
                   a[0] ^ b[0];
    new_index[1] = a[22] ^ a[20] ^ a[19] ^ a[18] ^ a[17] ^ a[14] ^ a[12] ^ a[4] ^
                   a[3] ^ b[1];
    new_index[2] = a[23] ^ a[21] ^ a[20] ^ a[19] ^ a[18] ^ a[15] ^ a[13] ^ a[5] ^
                   a[4] ^ b[2];
    new_index[3] = a[24] ^ a[22] ^ a[21] ^ a[20] ^ a[19] ^ a[16] ^ a[14] ^ a[6] ^
                   a[5] ^ b[3];
    new_index[4] = a[25] ^ a[23] ^ a[22] ^ a[21] ^ a[20] ^ a[17] ^ a[15] ^ a[7] ^
                   a[6] ^ b[4];
    new_index[5] = a[24] ^ a[23] ^ a[22] ^ a[21] ^ a[18] ^ a[16] ^ a[8] ^ a[7] ^
                   b[5];
    new_index[6] = a[25] ^ a[24] ^ a[23] ^ a[22] ^ a[19] ^ a[17] ^ a[9] ^ a[8] ^
                   b[6];
    new_index[7] = a[25] ^ a[24] ^ a[23] ^ a[20] ^ a[18] ^ a[10] ^ a[9] ^ b[7];
    new_index[8] = a[25] ^ a[24] ^ a[21] ^ a[19] ^ a[11] ^ a[10] ^ b[8];
    new_index[9] = a[25] ^ a[22] ^ a[20] ^ a[12] ^ a[11] ^ b[9];

    return new_index.to_ulong();

  } else if (bank_set_num == 2048) {
    // IPOLY(11) - Degree 11 polynomial
    // Primitive polynomial: x^11 + x^2 + 1
    // This provides conflict-free access for 2^11 = 2048 strides
    std::bitset<64> a(higher_bits);
    std::bitset<11> b(index);
    std::bitset<11> new_index(index);

    // XOR equations derived from H-matrix of degree-11 irreducible polynomial
    new_index[0] = a[26] ^ a[25] ^ a[24] ^ a[23] ^ a[20] ^ a[18] ^ a[14] ^ a[13] ^
                   a[0] ^ b[0];
    new_index[1] = a[23] ^ a[21] ^ a[20] ^ a[19] ^ a[18] ^ a[15] ^ a[13] ^ a[4] ^
                   a[3] ^ b[1];
    new_index[2] = a[24] ^ a[22] ^ a[21] ^ a[20] ^ a[19] ^ a[16] ^ a[14] ^ a[5] ^
                   a[4] ^ b[2];
    new_index[3] = a[25] ^ a[23] ^ a[22] ^ a[21] ^ a[20] ^ a[17] ^ a[15] ^ a[6] ^
                   a[5] ^ b[3];
    new_index[4] = a[26] ^ a[24] ^ a[23] ^ a[22] ^ a[21] ^ a[18] ^ a[16] ^ a[7] ^
                   a[6] ^ b[4];
    new_index[5] = a[25] ^ a[24] ^ a[23] ^ a[22] ^ a[19] ^ a[17] ^ a[8] ^ a[7] ^
                   b[5];
    new_index[6] = a[26] ^ a[25] ^ a[24] ^ a[23] ^ a[20] ^ a[18] ^ a[9] ^ a[8] ^
                   b[6];
    new_index[7] = a[26] ^ a[25] ^ a[24] ^ a[21] ^ a[19] ^ a[10] ^ a[9] ^ b[7];
    new_index[8] = a[26] ^ a[25] ^ a[22] ^ a[20] ^ a[11] ^ a[10] ^ b[8];
    new_index[9] = a[26] ^ a[23] ^ a[21] ^ a[12] ^ a[11] ^ b[9];
    new_index[10] = a[24] ^ a[22] ^ a[13] ^ a[12] ^ b[10];

    return new_index.to_ulong();

  } else { /* Incorrect number of banks for the hashing function */
    fprintf(stderr,
            "IPOLY ERROR:\n"
            "  bank_set_num = %u (Supported: 16, 32, 64, 128, 256, 512, 1024, 2048)\n"
            "  index        = %u\n"
            "  higher_bits  = 0x%llx\n"
            "  The number of banks must be a power of 2 from 16 to 2048.\n"
            "  To add new bank counts, generate XOR equations from the\n"
            "  H-matrix of an irreducible polynomial of appropriate degree\n"
            "  using GF(2) arithmetic. Reference: Peterson's Table.\n",
            bank_set_num,
            index,
            (unsigned long long)higher_bits);

    assert(0);
    return 0;
  }
}

[Ivecia]

@PrabinKuSabat Hi, tuner README said the L2 cache hashing function might be Linear or IPOLY hashing function. On my side, result of RTX 4090 shows a different configuration with yours. I suspect that we should only check the Linear hashing function, maybe?

Is true that the L here in L,A:192:16,32:0,32(from -gpgpu_cache:dl2) mean the simulator doesn't use the IPOLY hashing function ( instead uses Linear pass).

I think yes. You can found the usage of them in gpu-sim.cc and gpu-cache.h. It choose which hashing function will use, and actually it does.

If it is , then what can the reason behind both the correlation looking exact?

I'm not sure for that. You may double check your configuration file. On my side, I cannot run configuration with IPOLY hashing function.

Currently, I'm facing segmentation fault on RTX 4090 with Linear hashing function, trying to figure out what's wrong.

[JRPan]

L2 writes are expected to have high error rates. We don't care writes as much as reads.

Additionally, the rates are also incorrect.
Focus on hits and counts. L2 read hits and L2 accesses should have good correlation.

[Ivecia]

@JRPan, @PrabinKuSabat. I have finished my test on NVIDIA RTX 4090, showing a wired correlation (with rodinia_2.0-ft, per-kernel) as follows. I'm not quite sure if there are more mistakes in the new tuner. Perhaps @JRPan can help us to double check the new tuner and related microbenchmark. Thanks in advance!

Correlation:

Plotting NVIDIA GeForce RTX 4090 : [GPC Cycles]
RTX4090-SASS (10 apps, 193 kernels (6 < 1% Err, 31 under, 156 over)) [Correl=0.9718 Err=12.53%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (10 apps (0 < 1% Err, 3 under, 7 over, 4 < 10% Err)) [Correl=0.9991 Err=13.59% Agg_Err=11.07% RPD=12.68%,NMSE=0.23]

Plotting NVIDIA GeForce RTX 4090 : [Warp Instructions]
RTX4090-SASS (10 apps, 193 kernels (193 < 1% Err, 0 under, 0 over)) [Correl=1.0 Err=0.02%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (10 apps (10 < 1% Err, 3 under, 0 over, 10 < 10% Err)) [Correl=1.0 Err=0.10% Agg_Err=0.13% RPD=0.10%,NMSE=0.00]

Plotting NVIDIA GeForce RTX 4090 : [L2 Read Hits]
RTX4090-SASS (10 apps, 193 kernels (24 < 1% Err, 5 under, 164 over)) [Correl=0.1453 Err=13143.08%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (10 apps (0 < 1% Err, 1 under, 9 over, 1 < 10% Err)) [Correl=-0.09653 Err=6327900002288.67% Agg_Err=1121.24% RPD=115.35%,NMSE=28.83]

Plotting NVIDIA GeForce RTX 4090 : [L2 Reads]
RTX4090-SASS (10 apps, 193 kernels (187 < 1% Err, 2 under, 4 over)) [Correl=0.9991 Err=0.19%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (10 apps (8 < 1% Err, 1 under, 1 over, 10 < 10% Err)) [Correl=0.9999 Err=0.95% Agg_Err=0.91% RPD=0.92%,NMSE=0.03]

Plotting NVIDIA GeForce RTX 4090 : [L2 Writes]
RTX4090-SASS (10 apps, 193 kernels (180 < 1% Err, 0 under, 13 over)) [Correl=0.9977 Err=1.08%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (10 apps (6 < 1% Err, 0 under, 4 over, 9 < 10% Err)) [Correl=0.9989 Err=3.21% Agg_Err=2.46% RPD=3.03%,NMSE=0.05]

Plotting NVIDIA GeForce RTX 4090 : [L2 Write Hits]
RTX4090-SASS (10 apps, 193 kernels (159 < 1% Err, 25 under, 9 over)) [Correl=0.7692 Err=9.20%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (10 apps (4 < 1% Err, 7 under, 1 over, 6 < 10% Err)) [Correl=0.8796 Err=26.79% Agg_Err=27.52% RPD=41.79%,NMSE=0.54]

Plotting NVIDIA GeForce RTX 4090 : [L2 Write Hit Rate]
RTX4090-SASS (10 apps, 192 kernels (165 < 1% Err, 27 under, 0 over)) [Correl=nan Err=8.40%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (10 apps (3 < 1% Err, 8 under, 0 over, 4 < 10% Err)) [Correl=nan Err=25.47% Agg_Err=25.47% RPD=36.43%,NMSE=0.38]

Plotting NVIDIA GeForce RTX 4090 : [Occupancy]
RTX4090-SASS (10 apps, 193 kernels (45 < 1% Err, 138 under, 10 over)) [Correl=0.9937 Err=3.13%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (10 apps (3 < 1% Err, 8 under, 2 over, 8 < 10% Err)) [Correl=0.9933 Err=4.13% Agg_Err=4.22% RPD=4.07%,NMSE=0.07]

Plotting NVIDIA GeForce RTX 4090 : [L1D Read Hits]
RTX4090-SASS (10 apps, 193 kernels (188 < 1% Err, 0 under, 5 over)) [Correl=0.9998 Err=0.21%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (10 apps (9 < 1% Err, 0 under, 2 over, 10 < 10% Err)) [Correl=0.9989 Err=1.04% Agg_Err=5.22% RPD=1.00%,NMSE=0.16]

Plotting NVIDIA GeForce RTX 4090 : [L1D Write Hits]
RTX4090-SASS (10 apps, 193 kernels (25 < 1% Err, 164 under, 4 over)) [Correl=0.9609 Err=13.90%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (10 apps (3 < 1% Err, 6 under, 1 over, 4 < 10% Err)) [Correl=0.9863 Err=31.79% Agg_Err=17.08% RPD=48.95%,NMSE=0.24]

Plotting NVIDIA GeForce RTX 4090 : [L1D Read Access]
RTX4090-SASS (10 apps, 193 kernels (189 < 1% Err, 0 under, 4 over)) [Correl=0.9997 Err=0.19%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (10 apps (9 < 1% Err, 0 under, 1 over, 10 < 10% Err)) [Correl=0.9988 Err=0.94% Agg_Err=3.17% RPD=0.90%,NMSE=0.10]

Plotting NVIDIA GeForce RTX 4090 : [L1D Write Access]
RTX4090-SASS (10 apps, 193 kernels (193 < 1% Err, 0 under, 0 over)) [Correl=1.0 Err=0.00%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (10 apps (10 < 1% Err, 0 under, 0 over, 10 < 10% Err)) [Correl=1.0 Err=0.00% Agg_Err=0.00% RPD=0.00%,NMSE=0.00]

Plotting NVIDIA GeForce RTX 4090 : [DRAM Reads]
RTX4090-SASS (10 apps, 193 kernels (0 < 1% Err, 192 under, 1 over)) [Correl=0.5002 Err=90.66%] :: 0 high error points dropped from Err calc. 0 dropped for HW too low (>0)
Per-App :: RTX4090-SASS (10 apps (0 < 1% Err, 10 under, 0 over, 0 < 10% Err)) [Correl=-0.02181 Err=65.01% Agg_Err=89.62% RPD=108.39%,NMSE=2.29]

Configuration:

# functional simulator specification
-gpgpu_ptx_instruction_classification 0
-gpgpu_ptx_sim_mode 0
-gpgpu_ptx_force_max_capability 89

# Device Limits
-gpgpu_stack_size_limit 1024
-gpgpu_heap_size_limit 8388608
-gpgpu_runtime_sync_depth_limit 2
-gpgpu_runtime_pending_launch_count_limit 2048
-gpgpu_kernel_launch_latency  5000
-gpgpu_TB_launch_latency 0

# Compute Capability
-gpgpu_compute_capability_major 8
-gpgpu_compute_capability_minor 9

# PTX execution-driven
-gpgpu_ptx_convert_to_ptxplus 0
-gpgpu_ptx_save_converted_ptxplus 0

# high level architecture configuration
-gpgpu_n_clusters 128
-gpgpu_n_cores_per_cluster 1
-gpgpu_n_mem 6
-gpgpu_n_sub_partition_per_mchannel 2

# clock domains
#-gpgpu_clock_domains <Core Clock>:<Interconnect Clock>:<L2 Clock>:<DRAM Clock>
-gpgpu_clock_domains 2520:2520:2520:5250

# shader core pipeline config
-gpgpu_shader_registers 65536
-gpgpu_registers_per_block 65536
-gpgpu_occupancy_sm_number 89

-gpgpu_shader_core_pipeline 1536:32
-gpgpu_shader_cta 32
-gpgpu_simd_model 1

# Pipeline widths and number of FUs
# ID_OC_SP,ID_OC_DP,ID_OC_INT,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_DP,OC_EX_INT,OC_EX_SFU,OC_EX_MEM,EX_WB,ID_OC_TENSOR_CORE,OC_EX_TENSOR_CORE
-gpgpu_pipeline_widths 4,4,4,4,4,4,4,4,4,4,8,4,4
-gpgpu_num_sp_units 4
-gpgpu_num_sfu_units 4
-gpgpu_num_dp_units 4
-gpgpu_num_int_units 4
-gpgpu_tensor_core_avail 1
-gpgpu_num_tensor_core_units 4

# Instruction latencies and initiation intervals
# "ADD,MAX,MUL,MAD,DIV"
# All Div operations are executed on SFU unit
-ptx_opcode_latency_int 4,4,4,4,21
-ptx_opcode_initiation_int 2,2,2,2,2
-ptx_opcode_latency_fp 4,4,4,4,39
-ptx_opcode_initiation_fp 2,2,2,2,4
-ptx_opcode_latency_dp 54,54,54,54,330
-ptx_opcode_initiation_dp 54,54,54,54,130
-ptx_opcode_latency_sfu 23
-ptx_opcode_initiation_sfu 8
-ptx_opcode_latency_tesnor 64
-ptx_opcode_initiation_tensor 64

# sub core model: in which each scheduler has its own register file and EUs
# i.e. schedulers are isolated
-gpgpu_sub_core_model 1
# disable specialized operand collectors and use generic operand collectors instead
-gpgpu_enable_specialized_operand_collector 0
-gpgpu_operand_collector_num_units_gen 8
-gpgpu_operand_collector_num_in_ports_gen 8
-gpgpu_operand_collector_num_out_ports_gen 8
# register banks
-gpgpu_num_reg_banks 8
-gpgpu_reg_file_port_throughput 2

# warp scheduling
-gpgpu_num_sched_per_core 4
-gpgpu_scheduler lrr
# a warp scheduler issue mode
-gpgpu_max_insn_issue_per_warp 1
-gpgpu_dual_issue_diff_exec_units 1

## L1/shared memory configuration
# <nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_alloc>:<set_index_fn>,<mshr>:<N>:<merge>,<mq>:**<fifo_entry>
# ** Optional parameter - Required when mshr_type==Texture Fifo
# In adaptive cache, we adaptively assign the remaining shared memory to L1 cache
# For more info, see https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#shared-memory-7-x
-gpgpu_adaptive_cache_config 1
-gpgpu_shmem_option 0,8,16,32,64,100
-gpgpu_unified_l1d_size 128
# L1 cache configuration
-gpgpu_l1_banks 4
-gpgpu_cache:dl1 S:4:128:64,L:T:m:L:L,A:384:48,16:0,32
-gpgpu_l1_latency 34
-gpgpu_gmem_skip_L1D 0
-gpgpu_flush_l1_cache 1
-gpgpu_n_cluster_ejection_buffer_size 32
-gpgpu_l1_cache_write_ratio 25

# shared memory  configuration
-gpgpu_shmem_size 102400
-gpgpu_shmem_sizeDefault 102400
-gpgpu_shmem_per_block 49152
-gpgpu_smem_latency 24
# shared memory bankconflict detection
-gpgpu_shmem_num_banks 32
-gpgpu_shmem_limited_broadcast 0
-gpgpu_shmem_warp_parts 1
-gpgpu_coalesce_arch 89

# L2 cache
-gpgpu_cache:dl2 S:2048:128:24,L:B:m:L:L,A:192:4,32:0,32
-gpgpu_cache:dl2_texture_only 0
-gpgpu_dram_partition_queues 64:64:64:64
-gpgpu_perf_sim_memcpy 1
-gpgpu_memory_partition_indexing 0

# 128 KB Inst.
-gpgpu_cache:il1 N:64:128:16,L:R:f:N:L,S:2:48,4
-gpgpu_inst_fetch_throughput 4
# 128 KB Tex
# Note, TEX is deprected since Volta, It is used for legacy apps only. Use L1D cache instead with .nc modifier or __ldg mehtod
-gpgpu_tex_cache:l1 N:4:128:256,L:R:m:N:L,T:512:8,128:2
# 64 KB Const
-gpgpu_const_cache:l1 N:128:64:8,L:R:f:N:L,S:2:64,4
-gpgpu_perfect_inst_const_cache 1

# interconnection
# use built-in local xbar
-network_mode 2
-icnt_in_buffer_limit 512
-icnt_out_buffer_limit 512
-icnt_subnets 2
-icnt_flit_size 40
-icnt_arbiter_algo 1

# memory partition latency config
-gpgpu_l2_rop_latency 256
-dram_latency 309

# dram sched config
-gpgpu_dram_scheduler 1
-gpgpu_frfcfs_dram_sched_queue_size 64
-gpgpu_dram_return_queue_size 192

# dram model config
-gpgpu_n_mem_per_ctrlr 1
-gpgpu_dram_buswidth 2
-gpgpu_dram_burst_length 16
-dram_data_command_freq_ratio 4
-gpgpu_mem_address_mask 1
-gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RBBBCCCC.BCCSSSSS

# Mem timing
-gpgpu_dram_timing_opt nbk=16:CCD=4:RRD=18:RCD=36:RAS=82:RP=36:RC=117:CL=36:WL=12:CDLR=15:WR=36:nbkgrp=4:CCDL=9:RTPL=6
-dram_dual_bus_interface 0

# select lower bits for bnkgrp to increase bnkgrp parallelism
-dram_bnk_indexing_policy 0
-dram_bnkgrp_indexing_policy 1

#-dram_seperate_write_queue_enable 1
#-dram_write_queue_size 64:56:32

# stat collection
-gpgpu_memlatency_stat 14
-gpgpu_runtime_stat 500
-enable_ptx_file_line_stats 1
-visualizer_enabled 0

# power model configs, disable it untill we create a real energy model
-power_simulation_enabled 0

# tracing functionality
#-trace_enabled 1
#-trace_components WARP_SCHEDULER,SCOREBOARD
#-trace_sampling_core 0

And trace.config:

-trace_opcode_latency_initiation_int 4,2
-trace_opcode_latency_initiation_sp 4,2
-trace_opcode_latency_initiation_dp 54,54
-trace_opcode_latency_initiation_sfu 23,8
-trace_opcode_latency_initiation_tensor 32,32

#execute branch insts on spec unit 1
#<enabled>,<num_units>,<max_latency>,<ID_OC_SPEC>,<OC_EX_SPEC>,<NAME>
-specialized_unit_1 1,4,4,4,4,BRA
-trace_opcode_latency_initiation_spec_op_1 4,4

#TEX unit, make fixed latency for all tex insts
-specialized_unit_2 1,4,200,4,4,TEX
-trace_opcode_latency_initiation_spec_op_2 200,4

#tensor unit
-specialized_unit_3 1,4,32,4,4,TENSOR
-trace_opcode_latency_initiation_spec_op_3 32,32

#UDP unit, for turing and above
-specialized_unit_4 1,4,4,4,4,UDP
-trace_opcode_latency_initiation_spec_op_4 4,1