24#include <infiniband/verbs.h>
28#include <sys/select.h>
53 #include <boost/python.hpp>
54namespace bp = boost::python;
67 uint32_t rxQueueDepth) {
68 return std::make_shared<rpr::Server>(
69 deviceName, ibPort, gidIndex,
maxPayload, rxQueueDepth);
77rpr::Server::Server(
const std::string& deviceName,
81 uint32_t rxQueueDepth)
82 :
rpr::
Core(deviceName, ibPort, gidIndex, maxPayload),
85 cq_(nullptr), qp_(nullptr), mr_(nullptr), comp_channel_(nullptr),
88 numBufs_(rxQueueDepth),
101 memset(hostGid_, 0, 16);
102 memset(fpgaGid_, 0, 16);
103 wakeFd_[0] = wakeFd_[1] = -1;
127 uint64_t slabSize64 =
static_cast<uint64_t
>(numBufs_) * bufSize_;
128 if (slabSize64 > UINT32_MAX)
130 "RX slab too large: %u * %u = %" PRIu64
" exceeds 4 GiB",
131 numBufs_, bufSize_, slabSize64));
132 slabSize_ =
static_cast<uint32_t
>(slabSize64);
134 void* slabPtr =
nullptr;
135 if (posix_memalign(&slabPtr, 4096, slabSize_) != 0 || !slabPtr)
137 "Failed to allocate RX slab (%u bytes)",
139 slab_ =
static_cast<uint8_t*
>(slabPtr);
140 memset(slab_, 0, slabSize_);
145 mr_ = ibv_reg_mr(
pd_, slab_, slabSize_,
146 IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE);
149 "ibv_reg_mr failed"));
151 mrAddr_ =
reinterpret_cast<uint64_t
>(slab_);
154 log_->info(
"MR: addr=0x%016" PRIx64
" rkey=0x%08x size=%u",
155 mrAddr_, mrRkey_, slabSize_);
165 comp_channel_ = ibv_create_comp_channel(
ctx_);
168 "ibv_create_comp_channel failed"));
170 int flags = fcntl(comp_channel_->fd, F_GETFL);
172 fcntl(comp_channel_->fd, F_SETFL, flags | O_NONBLOCK) < 0)
174 "failed to set comp channel fd non-blocking"));
177 if (pipe(wakeFd_) != 0)
179 "wake self-pipe creation failed"));
180 for (
int i = 0; i < 2; ++i) {
181 int flags = fcntl(wakeFd_[i], F_GETFL);
182 if (flags >= 0) fcntl(wakeFd_[i], F_SETFL, flags | O_NONBLOCK);
189 cq_ = ibv_create_cq(
ctx_,
190 static_cast<int>(numBufs_),
191 this, comp_channel_, 0);
194 "ibv_create_cq failed"));
199 struct ibv_qp_init_attr qpAttr;
200 memset(&qpAttr, 0,
sizeof(qpAttr));
201 qpAttr.qp_type = IBV_QPT_RC;
202 qpAttr.sq_sig_all = 0;
203 qpAttr.send_cq = cq_;
204 qpAttr.recv_cq = cq_;
205 qpAttr.cap.max_recv_wr = numBufs_;
206 qpAttr.cap.max_send_wr = 1;
207 qpAttr.cap.max_recv_sge = 1;
208 qpAttr.cap.max_send_sge = 1;
210 qp_ = ibv_create_qp(
pd_, &qpAttr);
213 "ibv_create_qp (RC) failed"));
215 hostQpn_ = qp_->qp_num;
221 struct ibv_qp_attr attr;
222 memset(&attr, 0,
sizeof(attr));
223 attr.qp_state = IBV_QPS_INIT;
226 attr.qp_access_flags = IBV_ACCESS_REMOTE_WRITE |
227 IBV_ACCESS_REMOTE_READ |
228 IBV_ACCESS_LOCAL_WRITE;
230 if (ibv_modify_qp(qp_, &attr,
234 IBV_QP_ACCESS_FLAGS))
236 "QP RESET→INIT failed"));
259 "ibv_query_gid failed"));
261 for (
int i = 0; i < 16; ++i) {
262 if (gid.raw[i] != 0) {
269 "GID query returned all-zero for "
270 "gidIndex=%u on device '%s' "
271 "(likely out-of-range; rdma_rxe "
272 "does not validate gidIndex and "
273 "returns an empty GID for unused "
277 memcpy(hostGid_, gid.raw, 16);
284 std::mt19937 psnRng(std::random_device {}());
285 hostRqPsn_ = psnRng() & 0xFFFFFF;
286 hostSqPsn_ = psnRng() & 0xFFFFFF;
288 log_->info(
"RC QP ready: qpn=0x%06x rqPsn=0x%06x sqPsn=0x%06x",
289 hostQpn_, hostRqPsn_, hostSqPsn_);
318void rpr::Server::cleanupResources() {
319 std::lock_guard<std::mutex> lock(resourcesMtx_);
330 ibv_destroy_comp_channel(comp_channel_);
331 comp_channel_ =
nullptr;
339 for (
int i = 0; i < 2; ++i) {
340 if (wakeFd_[i] >= 0) {
350void rpr::Server::setFpgaGid(
const std::string& gidBytes) {
351 if (gidBytes.size() != 16)
353 "GID must be 16 bytes, got %zu",
355 memcpy(fpgaGid_, gidBytes.c_str(), 16);
356 log_->info(
"FPGA GID stored");
362void rpr::Server::completeConnection(uint32_t fpgaQpn, uint32_t fpgaRqPsn,
363 uint32_t pmtu, uint32_t minRnrTimer) {
377 "instance has been stopped or did not finish construction"));
384 if (thread_ !=
nullptr)
386 "rocev2::Server::completeConnection",
387 "completeConnection already invoked; destroy and recreate "
388 "Server to re-establish the RC connection"));
394 if (pmtu < 1 || pmtu > 5)
396 "rocev2::Server::completeConnection",
397 "pmtu must be in the range 1..5 "
398 "(1=256 2=512 3=1024 4=2048 5=4096); got %u", pmtu));
400 log_->info(
"completeConnection: fpgaQpn=0x%06x fpgaRqPsn=0x%06x minRnrTimer=%u",
401 fpgaQpn, fpgaRqPsn, minRnrTimer);
405 for (uint32_t i = 0; i < numBufs_; ++i) postRecvWr(i);
406 log_->info(
"Pre-posted %u recv WRs", numBufs_);
411 memcpy(dgid.raw, fpgaGid_, 16);
413 struct ibv_qp_attr attr;
414 memset(&attr, 0,
sizeof(attr));
415 attr.qp_state = IBV_QPS_RTR;
416 attr.path_mtu =
static_cast<ibv_mtu
>(pmtu);
417 attr.dest_qp_num = fpgaQpn;
418 attr.rq_psn = fpgaRqPsn;
419 attr.max_dest_rd_atomic = 16;
420 attr.min_rnr_timer = minRnrTimer;
421 attr.ah_attr.is_global = 1;
422 attr.ah_attr.grh.dgid = dgid;
423 attr.ah_attr.grh.sgid_index = gidIndex_;
424 attr.ah_attr.grh.hop_limit = 64;
425 attr.ah_attr.port_num = ibPort_;
428 if (ibv_modify_qp(qp_, &attr,
434 IBV_QP_MAX_DEST_RD_ATOMIC |
435 IBV_QP_MIN_RNR_TIMER))
437 "QP INIT→RTR failed"));
440 log_->info(
"QP → RTR (minRnrTimer=%u)", minRnrTimer);
444 struct ibv_qp_attr attr;
445 memset(&attr, 0,
sizeof(attr));
446 attr.qp_state = IBV_QPS_RTS;
447 attr.sq_psn = hostSqPsn_;
451 attr.max_rd_atomic = 16;
453 if (ibv_modify_qp(qp_, &attr,
459 IBV_QP_MAX_QP_RD_ATOMIC))
461 "QP RTR→RTS failed"));
464 log_->info(
"QP → RTS — ready to receive RDMA SENDs");
467 std::shared_ptr<int> scopePtr = std::make_shared<int>(0);
468 threadEn_.store(
true);
469 thread_ =
new std::thread(&rpr::Server::runThread,
this,
470 std::weak_ptr<int>(scopePtr));
473 pthread_setname_np(thread_->native_handle(),
"RoCEv2Server");
480std::string rpr::Server::getGid()
const {
481 std::ostringstream oss;
482 for (
int i = 0; i < 16; i += 2) {
484 oss << std::hex << std::setfill(
'0')
485 << std::setw(2) <<
static_cast<int>(hostGid_[i])
486 << std::setw(2) <<
static_cast<int>(hostGid_[i+1]);
495void rpr::Server::postRecvWr(uint32_t slot) {
496 std::lock_guard<std::mutex> lock(resourcesMtx_);
503 if (slot >= numBufs_)
505 "slot=%u out of range (numBufs=%u)",
508 if (!qp_ || !mr_ || !slab_)
510 "resources are not available for slot=%u",
513 uint8_t* bufStart = slab_ + (
static_cast<uint64_t
>(slot) * bufSize_);
516 memset(&sge, 0,
sizeof(sge));
517 sge.addr =
reinterpret_cast<uint64_t
>(bufStart);
518 sge.length = bufSize_;
519 sge.lkey = mr_->lkey;
521 struct ibv_recv_wr wr;
522 memset(&wr, 0,
sizeof(wr));
523 wr.wr_id =
static_cast<uint64_t
>(slot);
528 struct ibv_recv_wr* bad =
nullptr;
529 if (ibv_post_recv(qp_, &wr, &bad))
531 "ibv_post_recv failed (slot=%u)", slot));
543void rpr::Server::retBuffer(uint8_t* data, uint32_t meta, uint32_t rawSize) {
549 uint32_t slot = meta & 0x00FFFFFF;
551 log_->debug(
"retBuffer: re-posting slot=%u", slot);
580void rpr::Server::processCompletion(
struct ibv_wc& wc) {
581 uint32_t slot =
static_cast<uint32_t
>(wc.wr_id);
585 if (slot >= numBufs_) {
586 log_->warning(
"CQ returned out-of-range wr_id=%" PRIu64
587 " (numBufs=%u); discarding",
588 static_cast<uint64_t
>(wc.wr_id), numBufs_);
592 if (wc.status != IBV_WC_SUCCESS) {
593 log_->warning(
"CQ error: %s (slot=%u)", ibv_wc_status_str(wc.status), slot);
595 if (wc.status == IBV_WC_WR_FLUSH_ERR) {
598 log_->error(
"QP in ERROR state (WR_FLUSH_ERR); exiting CQ poll thread");
599 threadEn_.store(
false);
609 if (wc.opcode != IBV_WC_RECV || !(wc.wc_flags & IBV_WC_WITH_IMM)) {
610 log_->warning(
"Unexpected opcode %d / wc_flags 0x%x (slot=%u), re-posting",
611 wc.opcode, wc.wc_flags, slot);
620 uint32_t imm = ntohl(wc.imm_data);
621 uint8_t channel =
static_cast<uint8_t
>(imm & 0xFF);
622 uint32_t dataSlot = (imm >> 8) & 0x00FFFFFF;
623 uint32_t payloadLen = wc.byte_len;
625 if (payloadLen == 0 || payloadLen > bufSize_) {
626 log_->warning(
"Bad payload len=%u (slot=%u), re-posting", payloadLen, slot);
634 uint8_t* slotPtr = slab_ + (
static_cast<uint64_t
>(slot) * bufSize_);
640 buff->setPayload(payloadLen);
643 frame->appendBuffer(buff);
644 frame->setChannel(channel);
645 frame->setFirstUser(
SsiSof);
646 frame->setLastUser(0);
648 log_->debug(
"RX wrId=%u dataSlot=%u channel=%u len=%u",
649 slot, dataSlot, channel, payloadLen);
653 frameCount_.fetch_add(1, std::memory_order_relaxed);
654 byteCount_.fetch_add(payloadLen, std::memory_order_relaxed);
668void rpr::Server::runThread(std::weak_ptr<int> lockPtr) {
669 while (!lockPtr.expired())
continue;
672 log_->info(
"RoCEv2 receive thread started (event-driven)");
678 constexpr int kPollBatch = 16;
679 struct ibv_wc wcArr[kPollBatch];
681 const int cqFd = comp_channel_->fd;
682 const int wakeFd = wakeFd_[0];
683 const int maxFd = (cqFd > wakeFd ? cqFd : wakeFd) + 1;
704 constexpr uint32_t kSpinBudget = 100000;
705 uint32_t idleSpins = 0;
707 while (threadEn_.load()) {
708 int n = ibv_poll_cq(cq_, kPollBatch, wcArr);
710 log_->error(
"ibv_poll_cq error (%d); exiting CQ poll thread", n);
714 for (
int k = 0; k < n; ++k) processCompletion(wcArr[k]);
721 if (++idleSpins < kSpinBudget)
continue;
726 if (ibv_req_notify_cq(cq_, 0)) {
727 log_->error(
"ibv_req_notify_cq failed; exiting CQ poll thread");
730 n = ibv_poll_cq(cq_, kPollBatch, wcArr);
732 log_->error(
"ibv_poll_cq error (%d); exiting CQ poll thread", n);
736 for (
int k = 0; k < n; ++k) processCompletion(wcArr[k]);
745 FD_SET(wakeFd, &rfds);
746 int s = select(maxFd, &rfds,
nullptr,
nullptr,
nullptr);
748 if (errno == EINTR)
continue;
749 log_->error(
"select() failed (errno=%d); exiting CQ poll thread", errno);
752 if (FD_ISSET(wakeFd, &rfds)) {
753 uint8_t drainBuf[64];
754 while (read(wakeFd, drainBuf,
sizeof(drainBuf)) > 0) {}
757 if (FD_ISSET(cqFd, &rfds)) {
763 while (ibv_get_cq_event(comp_channel_, &evCq, &evCtx) == 0) ++events;
764 if (events) ibv_ack_cq_events(cq_, events);
768 log_->error(
"RoCEv2 receive thread exiting on ibverbs error: %s", e.
what());
769 }
catch (
const std::exception& e) {
770 log_->error(
"RoCEv2 receive thread exiting on exception: %s", e.what());
772 log_->error(
"RoCEv2 receive thread exiting on unknown exception");
775 threadEn_.store(
false);
776 log_->info(
"RoCEv2 receive thread stopped");
785 log_->warning(
"RoCEv2 Server::acceptFrame: TX not supported, dropping");
791void rpr::Server::stop() {
809 threadEn_.store(
false);
813 if (wakeFd_[1] >= 0) {
814 const uint8_t one = 1;
815 ssize_t wr = write(wakeFd_[1], &one, 1);
820 if (thread_->joinable()) thread_->join();
837rpr::Server::~Server() {
855void rpr::Server::setup_python() {
859 bp::bases<rpr::Core, ris::Master, ris::Slave>,
862 bp::init<std::string, uint8_t, uint8_t, uint32_t, uint32_t>(
863 (bp::arg(
"deviceName"),
864 bp::arg(
"ibPort") = 1,
865 bp::arg(
"gidIndex") = 0,
868 .def(
"create", &rpr::Server::create)
869 .staticmethod(
"create")
870 .def(
"setFpgaGid", &rpr::Server::setFpgaGid)
871 .def(
"completeConnection", &rpr::Server::completeConnection,
873 bp::arg(
"fpgaRqPsn"),
875 bp::arg(
"minRnrTimer") = 1))
876 .def(
"getQpn", &rpr::Server::getQpn)
877 .def(
"getGid", &rpr::Server::getGid)
878 .def(
"getRqPsn", &rpr::Server::getRqPsn)
879 .def(
"getSqPsn", &rpr::Server::getSqPsn)
880 .def(
"getMrAddr", &rpr::Server::getMrAddr)
881 .def(
"getMrRkey", &rpr::Server::getMrRkey)
882 .def(
"getFrameCount", &rpr::Server::getFrameCount)
883 .def(
"getByteCount", &rpr::Server::getByteCount)
884 .def(
"stop", &rpr::Server::stop);
886 bp::implicitly_convertible<rpr::ServerPtr, rpr::CorePtr>();
887 bp::implicitly_convertible<rpr::ServerPtr, ris::MasterPtr>();
888 bp::implicitly_convertible<rpr::ServerPtr, ris::SlavePtr>();
Generic Rogue exception type.
char const * what() const
Returns exception text for standard exception handling.
static GeneralError create(std::string src, const char *fmt,...)
Creates a formatted error instance.
RAII helper that releases the Python GIL for a scope.
static std::shared_ptr< rogue::Logging > create(const std::string &name, bool quiet=false)
Creates a logger instance.
struct ibv_context * ctx_
uint32_t maxPayload() const
std::shared_ptr< rogue::interfaces::stream::Buffer > BufferPtr
Shared pointer alias for Buffer.
std::shared_ptr< rogue::interfaces::stream::Frame > FramePtr
Shared pointer alias for Frame.
std::shared_ptr< rogue::protocols::rocev2::Server > ServerPtr
static const uint32_t DefaultMaxPayload
static const uint32_t DefaultRxQueueDepth
static const uint8_t SsiSof