Relocation of network buffers

[mrjimenez]

Hi,

I am working with an infineon xmc45_relax_kit in an application with network enabled, and I would like to have the network buffers relocated to DSRAM2 because I am running out of RAM, as can be seen in the linker output:

[288/288] Linking C executable zephyr/zephyr.elf
Memory region         Used Size  Region Size  %age Used
           FLASH:      259152 B       512 KB     49.43%
             RAM:       64020 B        64 KB     97.69%
          PSRAM1:          0 GB        64 KB      0.00%
          DSRAM2:          0 GB        32 KB      0.00%
        IDT_LIST:          0 GB        32 KB      0.00%

I added the following to prj.conf:

CONFIG_CODE_DATA_RELOCATION=y
CONFIG_CODE_DATA_RELOCATION_SRAM=y

And tried adding the following to CMakeLists.txt:

if (BOARD STREQUAL "xmc45_relax_kit")
	zephyr_code_relocate(FILES ${ZEPHYR_BASE}/subsys/net/ip/net_pkt.c LOCATION DSRAM2_DATA)
endif()

also tried:

	zephyr_code_relocate(LIBRARY subsys__net__ip LOCATION DSRAM2_DATA_BSS_NOINIT)

But I always get:

[283/288] Linking C executable zephyr/zephyr_pre0.elf
FAILED: zephyr/zephyr_pre0.elf zephyr/zephyr_pre0.map /home/myself/zephyrproject/workspace/apps/f1_network/build_xmc45/zephyr/zephyr_pre0.map 
: && ccache /home/myself/zephyr-sdk-0.17.4/arm-zephyr-eabi/bin/arm-zephyr-eabi-gcc  -gdwarf-4 zephyr/CMakeFiles/zephyr_pre0.dir/misc/empty_file.c.obj -o zephyr/zephyr_pre0.elf  zephyr/CMakeFiles/offsets.dir/./arch/arm/core/offsets/offsets.c.obj  -T  zephyr/linker_zephyr_pre0.cmd  -Wl,-Map,/home/myself/zephyrproject/workspace/apps/f1_network/build_xmc45/zephyr/zephyr_pre0.map  -Wl,--whole-archive  app/libapp.a  zephyr/libzephyr.a  zephyr/arch/common/libarch__common.a  zephyr/arch/arch/arm/core/libarch__arm__core.a  zephyr/arch/arch/arm/core/cortex_m/libarch__arm__core__cortex_m.a  zephyr/arch/arch/arm/core/mpu/libarch__arm__core__mpu.a  zephyr/lib/libc/picolibc/liblib__libc__picolibc.a  zephyr/lib/libc/common/liblib__libc__common.a  zephyr/lib/posix/options/liblib__posix__options.a  zephyr/lib/net_buf/liblib__net_buf.a  zephyr/lib/os/zvfs/liblib__os__zvfs.a  zephyr/subsys/modbus/libsubsys__modbus.a  zephyr/subsys/random/libsubsys__random.a  zephyr/subsys/net/libsubsys__net.a  zephyr/subsys/net/l2/virtual/libsubsys__net__l2__virtual.a  zephyr/subsys/net/l2/ethernet/libsubsys__net__l2__ethernet.a  zephyr/subsys/net/ip/libsubsys__net__ip.a  zephyr/subsys/net/lib/config/libsubsys__net__lib__config.a  zephyr/drivers/interrupt_controller/libdrivers__interrupt_controller.a  zephyr/drivers/console/libdrivers__console.a  zephyr/drivers/ethernet/libdrivers__ethernet.a  zephyr/drivers/mdio/libdrivers__mdio.a  zephyr/drivers/net/libdrivers__net.a  zephyr/drivers/pinctrl/libdrivers__pinctrl.a  zephyr/drivers/serial/libdrivers__serial.a  zephyr/drivers/timer/libdrivers__timer.a  modules/hal_infineon/libmodules_hal_infineon.a  -Wl,--no-whole-archive  zephyr/kernel/libkernel.a  -L/home/myself/zephyrproject/workspace/apps/f1_network/build_xmc45/zephyr  zephyr/libcode_relocation_source_lib.a  zephyr/arch/common/libisr_tables.a  -mcpu=cortex-m4  -mthumb  -mabi=aapcs  -mfp16-format=ieee  -mtp=soft  -fuse-ld=bfd  -Wl,--gc-sections  -Wl,--build-id=none  -Wl,--sort-common=descending  -Wl,--sort-section=alignment  -Wl,-u,_OffsetAbsSyms  -Wl,-u,_ConfigAbsSyms  -nostdlib  -static  -Wl,-X  -Wl,-N  -Wl,--orphan-handling=warn  -Wl,-no-pie  -specs=picolibc.specs  -DPICOLIBC_DOUBLE_PRINTF_SCANF -L"/home/myself/zephyr-sdk-0.17.4/arm-zephyr-eabi/bin/../lib/gcc/arm-zephyr-eabi/12.2.0/thumb/v7e-m/nofp" -lc -lgcc && cd /home/myself/zephyrproject/workspace/apps/f1_network/build_xmc45/zephyr && /usr/bin/cmake -E true
/home/myself/zephyr-sdk-0.17.4/arm-zephyr-eabi/bin/../lib/gcc/arm-zephyr-eabi/12.2.0/../../../../arm-zephyr-eabi/bin/ld.bfd: zephyr/arch/arch/arm/core/mpu/libarch__arm__core__mpu.a(arm_core_mpu.c.obj):(.rodata.static_regions+0xc): undefined reference to `__ram_text_reloc_start'
/home/myself/zephyr-sdk-0.17.4/arm-zephyr-eabi/bin/../lib/gcc/arm-zephyr-eabi/12.2.0/../../../../arm-zephyr-eabi/bin/ld.bfd: zephyr/arch/arch/arm/core/mpu/libarch__arm__core__mpu.a(arm_core_mpu.c.obj):(.rodata.static_regions+0x10): undefined reference to `__ram_text_reloc_size'
collect2: error: ld returned 1 exit status
ninja: build stopped: subcommand failed.
FATAL ERROR: command exited with status 1: /usr/bin/cmake --build /home/myself/zephyrproject/workspace/apps/f1_network/build_xmc45

Any ideas on what I am missing? If I am doing it wrong, what would be the right way to do it?

Best regards,
Marcelo.

[mrjimenez]

Hi again,

Just fixed the issue by removing CONFIG_CODE_DATA_RELOCATION_SRAM from prj.conf. I surely misunderstood something here.

[288/288] Linking C executable zephyr/zephyr.elf
Memory region         Used Size  Region Size  %age Used
           FLASH:      255212 B       512 KB     48.68%
             RAM:       54780 B        64 KB     83.59%
          PSRAM1:          0 GB        64 KB      0.00%
          DSRAM2:        8628 B        32 KB     26.33%
        IDT_LIST:          0 GB        32 KB      0.00%

[mrjimenez]

Continuing on the topic, even though I can now use DSRAM2, network buffers are still being alocated in RAM.

These are from zephyr.map:

Memory Configuration

Name             Origin             Length             Attributes
FLASH            0x0000000008000000 0x0000000000080000 xr
RAM              0x0000000020000000 0x0000000000010000 xw
PSRAM1           0x0000000010000000 0x0000000000010000 rw
DSRAM2           0x0000000030000000 0x0000000000008000 rw
IDT_LIST         0x00000000ffff7fff 0x0000000000008000 xw
*default*        0x0000000000000000 0xffffffffffffffff
...

net_buf_pool_area
                0x0000000020000448       0x58 load address 0x000000000803e088
                0x0000000020000448                _net_buf_pool_list_start = .
 *(SORT_BY_NAME(SORT_BY_ALIGNMENT(._net_buf_pool.static.*)))
 ._net_buf_pool.static.rx_bufs_
                0x0000000020000448       0x2c zephyr/subsys/net/ip/libsubsys__net__ip.a(net_pkt.c.obj)
 ._net_buf_pool.static.tx_bufs_
                0x0000000020000474       0x2c zephyr/subsys/net/ip/libsubsys__net__ip.a(net_pkt.c.obj)
                0x00000000200004a0                _net_buf_pool_list_end = .

...

 .noinit."WEST_TOPDIR/zephyr/subsys/net/ip/net_pkt.c".k_mem_slab_buf_tx_pkts
                0x000000002000c890      0x8b8 zephyr/subsys/net/ip/libsubsys__net__ip.a(net_pkt.c.obj)
                0x000000002000c890                _k_mem_slab_buf_tx_pkts
 .noinit."WEST_TOPDIR/zephyr/subsys/net/ip/net_pkt.c".k_mem_slab_buf_rx_pkts
                0x000000002000d148     0x1c20 zephyr/subsys/net/ip/libsubsys__net__ip.a(net_pkt.c.obj)
                0x000000002000d148                _k_mem_slab_buf_rx_pkts

And I found this in linker.cmd:

 .dsram2_noinit_reloc (NOLOAD) :
        {
                . = ALIGN(4);
                KEEP(*eth_xmc4xxx.c.obj(.noinit."WEST_TOPDIR/zephyr/drivers/ethernet/eth_xmc4xxx.c".k_mem_slab_buf_tx_frame_slab))
                KEEP(*net_mgmt.c.obj(.noinit."WEST_TOPDIR/zephyr/subsys/net/ip/net_mgmt.c".0))
                KEEP(*net_mgmt.c.obj(.noinit."WEST_TOPDIR/zephyr/subsys/net/ip/net_mgmt.c".0))
                KEEP(*net_mgmt.c.obj(.noinit."WEST_TOPDIR/zephyr/subsys/net/ip/net_mgmt.c".1))
                KEEP(*net_mgmt.c.obj(.noinit."WEST_TOPDIR/zephyr/subsys/net/ip/net_mgmt.c".1))
                KEEP(*net_pkt.c.obj(.noinit."WEST_TOPDIR/zephyr/subsys/net/ip/net_pkt.c".0))
                KEEP(*net_pkt.c.obj(.noinit."WEST_TOPDIR/zephyr/subsys/net/ip/net_pkt.c".1))
                KEEP(*net_pkt.c.obj(.noinit."WEST_TOPDIR/zephyr/subsys/net/ip/net_pkt.c".2))
                KEEP(*net_pkt.c.obj(.noinit."WEST_TOPDIR/zephyr/subsys/net/ip/net_pkt.c".3))
                KEEP(*net_pkt.c.obj(.noinit."WEST_TOPDIR/zephyr/subsys/net/ip/net_pkt.c".k_mem_slab_buf_rx_pkts))
                KEEP(*net_pkt.c.obj(.noinit."WEST_TOPDIR/zephyr/subsys/net/ip/net_pkt.c".k_mem_slab_buf_tx_pkts))
                KEEP(*net_tc.c.obj(.noinit."WEST_TOPDIR/zephyr/subsys/net/ip/net_tc.c".0))
                KEEP(*net_tc.c.obj(.noinit."WEST_TOPDIR/zephyr/subsys/net/ip/net_tc.c".1))
                KEEP(*sockets_service.c.obj(.noinit."WEST_TOPDIR/zephyr/subsys/net/lib/sockets/sockets_service.c".0))
                KEEP(*tcp.c.obj(.noinit."WEST_TOPDIR/zephyr/subsys/net/ip/tcp.c".0))
                KEEP(*tcp.c.obj(.noinit."WEST_TOPDIR/zephyr/subsys/net/ip/tcp.c".k_mem_slab_buf_tcp_conns_slab))
                . = ALIGN(4);
 } > DSRAM2 AT > DSRAM2
        __dsram2_noinit_reloc_end = .;
        __dsram2_noinit_reloc_start = ADDR(.dsram2_noinit_reloc);
        __dsram2_noinit_reloc_size = __dsram2_noinit_reloc_end - __dsram2_noinit_reloc_start;

Which suggests that this behavior is on purpose. My questions:

1. Where do these KEEP directives come from? I can see __noinit's and __net_buf_align's in the code, is there any relation?
2. Why do the network buffers have to stick to the RAM region? Is there a way to change this behavior?

[jukkar]

The net_pkt slabs are allocated by this macro

zephyr/include/zephyr/net/net_pkt.h

Line 1583 in 6c83b9b

#define NET_PKT_SLAB_DEFINE(name, count) \

and the net_buf pool is allocated by this macro

zephyr/include/zephyr/net/net_pkt.h

Line 1607 in 6c83b9b

#define NET_PKT_DATA_POOL_DEFINE(name, count) \

AFAIK the network packet/buffer data could be allocated also to some other place than the regular ram.

[ernestguillem]

In our case, an STM32H573ZI that we wanted to move the network buffers, we wrote an extra linkerscript as follows:

02_move_net_noinit_to_sram3.ld

/* Move networking noinit pools (net_pkt/tcp) to SRAM3 */
SECTION_DATA_PROLOGUE(.sram3_net_noinit, (NOLOAD),)
{
  KEEP(*net_pkt.c.obj(.noinit*))
  KEEP(*tcp.c.obj(.noinit*))
} GROUP_DATA_LINK_IN(SRAM3, SRAM3)

Then in our main CMakeLists.txt we added the following:

zephyr_linker_sources(RAM_SECTIONS linker/02_move_net_noinit_to_sram3.ld)

In our ram-report we see that those buffers moved to SRAM3 addresses

│   │   ├── net_pkt.c                                                                   39972  21.79%  -
        │   │   │   ├── _k_mem_slab_buf_rx_pkts                                                  2304   1.26%  0x2005d698 .sram3_net_noinit
        │   │   │   ├── _k_mem_slab_buf_tx_pkts                                                  1152   0.63%  0x2005d218 .sram3_net_noinit
        │   │   │   ├── _net_buf_rx_bufs                                                         2688   1.47%  0x2005c798 .sram3_net_noinit
        │   │   │   ├── _net_buf_tx_bufs                                                          896   0.49%  0x20056418 .sram3_net_noinit
        │   │   │   ├── log_dynamic_net_pkt                                                         4   0.00%  0x20001c84 log_dynamic_area
        │   │   │   ├── net_buf_data_rx_bufs                                                    24576  13.40%  0x20056798 .sram3_net_noinit
        │   │   │   ├── net_buf_data_tx_bufs                                                     8192   4.47%  0x20054418 .sram3_net_noinit
        │   │   │   ├── rx_bufs                                                                    48   0.03%  0x20002034 net_buf_pool_area
        │   │   │   ├── rx_pkts                                                                    32   0.02%  0x20001d2c k_mem_slab_area
        │   │   │   ├── tx_bufs                                                                    48   0.03%  0x20002064 net_buf_pool_area
        │   │   │   └── tx_pkts                                                                    32   0.02%  0x20001d6c k_mem_slab_area
        │   │   ├── net_tc.c                                                                     1796   0.98%  -
        │   │   │   ├── log_dynamic_net_tc                                                          4   0.00%  0x20001ca0 log_dynamic_area
        │   │   │   ├── rx_classes                                                                288   0.16%  0x200042e0 bss
        │   │   │   └── rx_stack                                                                 1504   0.82%  0x20017d30 noinit
        │   │   ├── route.c                                                                       816   0.44%  -
        │   │   │   ├── active_route_lifetime_timers                                                8   0.00%  0x200076e0 bss
        │   │   │   ├── log_dynamic_net_route                                                       4   0.00%  0x20001c88 log_dynamic_area
        │   │   │   ├── net_nbr_routes                                                             12   0.01%  0x20001488 datas
        │   │   │   ├── net_route_entries_pool                                                    544   0.30%  0x20001494 datas
        │   │   │   ├── net_route_nexthop_pool                                                    192   0.10%  0x200016b4 datas
        │   │   │   ├── route_lifetime_timer                                                       48   0.03%  0x20004460 bss
        │   │   │   └── routes                                                                      8   0.00%  0x200076e8 bss
        │   │   ├── tcp.c                                                                       12264   6.69%  -
        │   │   │   ├── _k_mem_slab_buf_tcp_conns_slab                                          10880   5.93%  0x2005e398 .sram3_net_noinit
        │   │   │   ├── log_dynamic_net_tcp                                                         4   0.00%  0x20001ca4 log_dynamic_area
        │   │   │   ├── tcp_conns                                                                   8   0.00%  0x200076f8 bss
        │   │   │   ├── tcp_conns_slab                                                             32   0.02%  0x20001d4c k_mem_slab_area
        │   │   │   ├── tcp_lock                                                                   24   0.01%  0x20001ed8 k_mutex_area
        │   │   │   ├── tcp_max_timeout_ms                                                          4   0.00%  0x20007700 bss
        │   │   │   ├── tcp_recv_cb                                                                 4   0.00%  0x200076f0 bss
        │   │   │   ├── tcp_send_cb                                                                 4   0.00%  0x200076f4 bss
        │   │   │   ├── tcp_work_q                                                                280   0.15%  0x20004490 bss
        │   │   │   └── work_q_stack                                                             1024   0.56%  0x2005df98 .sram3_net_noinit

[mrjimenez]

So, as I suspected, the zephyr_code_relocate() is not enough to make that particular relocation to work. The extra link script is the missing piece, thanks @ernestguillem !

It would be nice if the zephyr_code_relocate() worked in this case without an explicit linker script, those are not things one ususally deals with, and the right way to do it is not well documented, though I might be wrong. Maybe it is possible to add some changes to the script to perform the relocation, or a message telling us that it did not relocate it?

[ernestguillem]

Honestly I think it might be a bug. As per https://docs.zephyrproject.org/latest/kernel/code-relocation.html I would expect that if we use something like:
zephyr_code_relocate(FILES src/file1.c LOCATION SRAM3_DATA_BSS_NOINIT)
That all noinit data in file1.c would move to the SRAM3 location, however it didn't and we had to create several linker scripts to move large buffers such as JLink RTT's, network buffers, etc.
But I might be wrong 😅

[mrjimenez]

Yes, I agree, I think it is a bug, too.

I have submitted a PR with a fix. See #100260

No custom scripts required, just

CONFIG_CODE_DATA_RELOCATION=y

and

zephyr_code_relocate(LIBRARY drivers__ethernet LOCATION DSRAM2_BSS)
zephyr_code_relocate(LIBRARY subsys__net__ip LOCATION DSRAM2_NOINIT)

@ernestguillem If you still have the project at hand, maybe you could test it. I have tested here in an Infineon XMC4500 relax kit, and worked fine.

[ernestguillem]

Yes it does. It was a bit tricky since the project doesn't compile on main but does compile fine in 4.3.0, 4.2.1 and 4.2.0.
I've removed my linker script and applied your fix to 4.3.0 and then compiled with and without:
CONFIG_CODE_DATA_RELOCATION=y

zephyr_code_relocate(LIBRARY drivers__ethernet LOCATION SRAM3_DATA_BSS_NOINIT)
zephyr_code_relocate(LIBRARY subsys__net__ip LOCATION SRAM3_DATA_BSS_NOINIT)

And here's the diff:

26c26
<         │   │   │   ├── _k_fifo_buf_event_msgq                                                    440   0.24%  0x20016280 noinit
---
>         │   │   │   ├── _k_fifo_buf_event_msgq                                                    440   0.24%  0x20055ce0 .sram3_noinit_reloc
31c31
<         │   │   │   ├── mgmt_stack                                                                800   0.43%  0x20012718 noinit
---
>         │   │   │   ├── mgmt_stack                                                                800   0.43%  0x200559c0 .sram3_noinit_reloc
38,41c38,41
<         │   │   │   ├── _k_mem_slab_buf_rx_pkts                                                  2304   1.25%  0x2001f6b8 noinit
<         │   │   │   ├── _k_mem_slab_buf_tx_pkts                                                  1152   0.62%  0x2001f238 noinit
<         │   │   │   ├── _net_buf_rx_bufs                                                         2688   1.45%  0x2001e7b8 noinit
<         │   │   │   ├── _net_buf_tx_bufs                                                          896   0.48%  0x20018438 noinit
---
>         │   │   │   ├── _k_mem_slab_buf_rx_pkts                                                  2304   1.25%  0x2005ec98 .sram3_noinit_reloc
>         │   │   │   ├── _k_mem_slab_buf_tx_pkts                                                  1152   0.62%  0x2005f598 .sram3_noinit_reloc
>         │   │   │   ├── _net_buf_rx_bufs                                                         2688   1.45%  0x20055e98 .sram3_noinit_reloc
>         │   │   │   ├── _net_buf_tx_bufs                                                          896   0.48%  0x2005c918 .sram3_noinit_reloc
43,44c43,44
<         │   │   │   ├── net_buf_data_rx_bufs                                                    24576  13.29%  0x200187b8 noinit
<         │   │   │   ├── net_buf_data_tx_bufs                                                     8192   4.43%  0x20016438 noinit
---
>         │   │   │   ├── net_buf_data_rx_bufs                                                    24576  13.29%  0x20056918 .sram3_noinit_reloc
>         │   │   │   ├── net_buf_data_tx_bufs                                                     8192   4.43%  0x2005cc98 .sram3_noinit_reloc
54c54
<         │   │   │   └── rx_stack                                                                 1504   0.81%  0x20012a38 noinit
---
>         │   │   │   └── rx_stack                                                                 1504   0.81%  0x2005fa18 .sram3_noinit_reloc
64c64
<         │   │   │   ├── _k_mem_slab_buf_tcp_conns_slab                                          11136   6.02%  0x2001ffb8 noinit
---
>         │   │   │   ├── _k_mem_slab_buf_tcp_conns_slab                                          11136   6.02%  0x200603f8 .sram3_noinit_reloc
73c73
<         │   │   │   └── work_q_stack                                                             1024   0.55%  0x20013018 noinit
---
>         │   │   │   └── work_q_stack                                                             1024   0.55%  0x2005fff8 .sram3_noinit_reloc

Before BSS and DATA would move just fine but noinit would remain in SRAM1. With your fix they move just fine to SRAM3 on the STM32H5. Good job @mrjimenez !