[ICRDMA] Do not use RoCEv1 gid index if present. Improve rdma diagnostic info. EXT-1667 (#29694)

One port can have multiple indexes for same GID. The app must use correct index (RoCEv1 or RoCEv2).
Unfortunately we can't check index type (no support in the ibdrv wrapper) so right now we use some hack to pick RoCEv2 index.

This pr also adds more diagnostic info.

Author: dcherednik

ydb/library/actors/interconnect/interconnect_handshake.cpp

@@ -1447,6 +1447,9 @@ namespace NActors {
                     } else {
                         sb << "Unable to complete rdma READ work request due to cq runtime error";
                     }
+                    if (Rdma.Qp) {
+                        sb << " qp: " << Rdma.Qp;
+                    }
                     LOG_LOG_IC_X(NActorsServices::INTERCONNECT, "ICRDMA", NLog::PRI_ERROR, sb.c_str());
                     rdmaReadAck.SetErr(sb);
                 }

ydb/library/actors/interconnect/interconnect_mon.cpp

@@ -78,6 +78,22 @@ namespace NInterconnect {
                 }
             }
 
+            TString XdcFlagsToString(const ui8 xdcFlags) {
+                TString res;
+                if (xdcFlags & TInterconnectProxyTCP::TProxyStats::XDCFlags::MSG_ZERO_COPY_SEND) {
+                    res.append("MSG_ZC_SEND|");
+                }
+                if (xdcFlags & TInterconnectProxyTCP::TProxyStats::XDCFlags::RDMA_READ) {
+                    res.append("RDMA_READ|");
+                }
+                if (res.empty()) {
+                    res.append("_");
+                } else {
+                    res.pop_back();
+                }
+                return res;
+            }
+
             TString GenerateHtml() {
                 TStringStream str;
                 HTML(str) {
@@ -131,7 +147,7 @@ namespace NInterconnect {
                                     TABLED() { str << kv.second.TotalOutputQueueSize; }
                                     TABLED() { str << (kv.second.Connected ? "yes" : "<strong>no</strong>"); }
                                     TABLED() { str << (kv.second.ExternalDataChannel ? "yes" : "no")
-                                        << " (" << (kv.second.XDCFlags & TInterconnectProxyTCP::TProxyStats::XDCFlags::MSG_ZERO_COPY_SEND ? "MSG_ZC_SEND" : "_")  << ")"; }
+                                        << " (" << XdcFlagsToString(kv.second.XDCFlags) << ")"; }
                                     TABLED() { str << kv.second.Host; }
                                     TABLED() { str << kv.second.Port; }
                                     TABLED() {

ydb/library/actors/interconnect/interconnect_tcp_proxy.h

@@ -53,6 +53,7 @@ namespace NActors {
             enum XDCFlags {
                 NONE = 0,
                 MSG_ZERO_COPY_SEND = 1,
+                RDMA_READ = 1 << 1,
             };
             ui8 XDCFlags;
         };

ydb/library/actors/interconnect/interconnect_tcp_session.cpp

@@ -68,16 +68,25 @@ namespace NActors {
         SendUpdateToWhiteboard();
     }
 
-    std::optional<ui8> TInterconnectSessionTCP::GetXDCFlags() const {
+    std::optional<ui8> TInterconnectSessionTCP::GetXDCFlags() const noexcept {
+        std::optional<ui8> flags;
+        using NInterconnect::NRdma::TQueuePair;
         if (XdcSocket) {
+            flags = TInterconnectProxyTCP::TProxyStats::NONE;
             if (ZcProcessor.ZcStateIsOk()) {
-                return TInterconnectProxyTCP::TProxyStats::MSG_ZERO_COPY_SEND;
-            } else {
-                return TInterconnectProxyTCP::TProxyStats::NONE;
+                *flags |= TInterconnectProxyTCP::TProxyStats::MSG_ZERO_COPY_SEND;
+            }
+            if (RdmaQp) {
+                TQueuePair::TQpState res = RdmaQp->GetState(false);
+                TQueuePair::TQpS* qpState = std::get_if<TQueuePair::TQpS>(&res);
+                if (qpState) {
+                    if (TQueuePair::IsRtsState(*qpState)) {
+                        *flags |= TInterconnectProxyTCP::TProxyStats::RDMA_READ;
+                    }
+                }
             }
-        } else {
-            return {};
         }
+        return flags;
     }
 
     void TInterconnectSessionTCP::CloseInputSession() {

ydb/library/actors/interconnect/interconnect_tcp_session.h

@@ -466,7 +466,7 @@ namespace NActors {
             return ReceiveContext->ClockSkew_us;
         }
 
-        std::optional<ui8> GetXDCFlags() const;
+        std::optional<ui8> GetXDCFlags() const noexcept;
 
     private:
         friend class TInterconnectProxyTCP;

ydb/library/actors/interconnect/rdma/link_manager.cpp

@@ -29,14 +29,14 @@
 
 template <>
 struct std::less<ibv_gid> {
-    std::size_t operator()(const ibv_gid& a, const ibv_gid& b) const {
+    bool operator()(const ibv_gid& a, const ibv_gid& b) const noexcept {
         return std::tie(a.global.subnet_prefix, a.global.interface_id) <
                std::tie(b.global.subnet_prefix, b.global.interface_id);
     }
 };
 template <>
 struct std::equal_to<ibv_gid> {
-    bool operator()(const ibv_gid& a, const ibv_gid& b) const {
+    bool operator()(const ibv_gid& a, const ibv_gid& b) const noexcept {
         return a.global.interface_id == b.global.interface_id
             && a.global.subnet_prefix == b.global.subnet_prefix;
     }
@@ -118,10 +118,19 @@ static class TRdmaLinkManager {
                 continue;
             }
 
+            /*
+             * TODO: Add ibv_query_gid_table support into arcadia ibv wrapper (contrib/libs/ibdrv) 
+             * This extension allows to get type of GID index to skip unsupported RoCEv1 
+             */
+
             for (uint8_t portNum = 1; portNum <= devAttrs.phys_port_cnt; portNum++) {
                 ibv_port_attr portAttrs;
                 err = ibv_query_port(ctx, portNum, &portAttrs);
                 if (err == 0) {
+                    if (portAttrs.state != IBV_PORT_ACTIVE) {
+                        continue; //Skip non active ports
+                    }
+
                     for (int gidIndex = 0; gidIndex < portAttrs.gid_tbl_len; gidIndex++ ) {
                         auto ctx = TRdmaCtx::Create(deviceCtx, portNum, gidIndex);
                         if (!ctx) {
@@ -135,6 +144,10 @@ static class TRdmaLinkManager {
         }
         std::sort(CtxMap.begin(), CtxMap.end(),
             [](const auto& a, const auto& b) {
+                if (std::equal_to<ibv_gid>()(a.first, b.first)) {
+                    // Hack: Most implementations have RoCEv2 after RoCEv1, but we prefer RoCEv2 gid index 
+                    return a.second->GetGidIndex() > b.second->GetGidIndex();
+                }
                 return std::less<ibv_gid>()(a.first, b.first);
             });
 

ydb/library/actors/interconnect/rdma/rdma.cpp

@@ -290,6 +290,9 @@ void TQueuePair::Output(IOutputStream& os) const noexcept {
     } else {
         os << attr.qp_state;
     }
+    if (Ctx) {
+        os << ", ctx: " << *Ctx;
+    }
 }
 
 TQueuePair::TQpState TQueuePair::GetState(bool forseUpdate) const noexcept {
@@ -324,7 +327,7 @@ size_t TQueuePair::GetDeviceIndex() const noexcept {
     return Ctx->GetDeviceIndex();
 }
 
-bool TQueuePair::IsRtsState(TQpS state) {
+bool TQueuePair::IsRtsState(TQpS state) noexcept {
      enum ibv_qp_state qpState = static_cast<enum ibv_qp_state>(state.State);
      return qpState == IBV_QPS_RTS;
 }

ydb/library/actors/interconnect/rdma/rdma.h

@@ -94,7 +94,7 @@ class TQueuePair: public NNonCopyable::TMoveOnly {
     struct TQpS {
         int State;
     };
-    static bool IsRtsState(TQpS state);
+    static bool IsRtsState(TQpS state) noexcept;
     struct TQpErr {
         int Err;
     };

ydb/library/actors/interconnect/ut/interconnect_ut.cpp

@@ -225,7 +225,7 @@ Y_UNIT_TEST_SUITE(Interconnect) {
         {
             auto s = GetRdmaQpStatus(cluster, 1, 2);
             auto tokens = SplitString(s, ",");
-            UNIT_ASSERT(tokens.size() == 2);
+            UNIT_ASSERT(tokens.size() > 2);
             UNIT_ASSERT(tokens[1] == "QPS_RTS");
         }