rogue
Loading...
Searching...
No Matches
Server.cpp
Go to the documentation of this file.
1
17#include "rogue/Directives.h"
18
20
21#include <arpa/inet.h>
22#include <errno.h>
23#include <fcntl.h>
24#include <infiniband/verbs.h>
25#include <inttypes.h>
26#include <stdint.h>
27#include <string.h>
28#include <sys/select.h>
29#include <unistd.h>
30
31#include <chrono>
32#include <cstdlib>
33#include <iomanip>
34#include <memory>
35#include <random>
36#include <sstream>
37#include <string>
38#include <thread>
39#include <vector>
40
41#include "rogue/GeneralError.h"
42#include "rogue/GilRelease.h"
43#include "rogue/Logging.h"
48
51
52#ifndef NO_PYTHON
53 #include <boost/python.hpp>
54namespace bp = boost::python;
55#endif
56
57// SSI Start-of-Frame bit set on every received frame
58static const uint8_t SsiSof = 0x02;
59
60// ---------------------------------------------------------------------------
61// Factory
62// ---------------------------------------------------------------------------
63rpr::ServerPtr rpr::Server::create(const std::string& deviceName,
64 uint8_t ibPort,
65 uint8_t gidIndex,
66 uint32_t maxPayload,
67 uint32_t rxQueueDepth) {
68 return std::make_shared<rpr::Server>(
69 deviceName, ibPort, gidIndex, maxPayload, rxQueueDepth);
70}
71
72// ---------------------------------------------------------------------------
73// Constructor
74// ibverbs setup through QP INIT. Receive thread starts in
75// completeConnection() once the FPGA QPN is known.
76// ---------------------------------------------------------------------------
77rpr::Server::Server(const std::string& deviceName,
78 uint8_t ibPort,
79 uint8_t gidIndex,
80 uint32_t maxPayload,
81 uint32_t rxQueueDepth)
82 : rpr::Core(deviceName, ibPort, gidIndex, maxPayload),
83 ris::Master(),
84 ris::Slave(),
85 cq_(nullptr), qp_(nullptr), mr_(nullptr), comp_channel_(nullptr),
86 slab_(nullptr),
87 slabSize_(0),
88 numBufs_(rxQueueDepth),
89 bufSize_(0),
90 hostQpn_(0),
91 hostRqPsn_(0),
92 hostSqPsn_(0),
93 mrAddr_(0),
94 mrRkey_(0),
95 thread_(nullptr),
96 threadEn_(false),
97 frameCount_(0),
98 byteCount_(0) {
99
100 log_ = rogue::Logging::create("rocev2.Server");
101 memset(hostGid_, 0, 16);
102 memset(fpgaGid_, 0, 16);
103 wakeFd_[0] = wakeFd_[1] = -1;
104
105 // Core releases the GIL while opening the device and allocating the
106 // protection domain. Release it here for the remaining ibverbs bring-up
107 // (reg_mr, create CQ/QP, modify to INIT, query GID). On a throw,
108 // GilRelease's destructor re-acquires the GIL before the boost.python
109 // exception translator runs.
110 rogue::GilRelease noGil;
111
112 // The destructor does NOT run on a partially-constructed object, so any
113 // throw between here and the end of the body would leak slab_ / mr_ /
114 // cq_ / qp_. Wrap the body in try/catch and call cleanupResources() on
115 // the way out before rethrowing — same effect as a stack of unique_ptr
116 // wrappers but with a single cleanup path.
117 try {
118 // -------------------------------------------------------------------
119 // 1. Allocate slab
120 // RC QPs do not prepend a GRH so each slot is exactly maxPayload_.
121 // posix_memalign accepts any size (aligned_alloc requires the size
122 // to be a multiple of the alignment per C11; with the default
123 // 9000 * 256 slab that constraint does not hold).
124 // -------------------------------------------------------------------
125 bufSize_ = maxPayload_;
126
127 uint64_t slabSize64 = static_cast<uint64_t>(numBufs_) * bufSize_;
128 if (slabSize64 > UINT32_MAX)
129 throw(rogue::GeneralError::create("rocev2::Server::Server",
130 "RX slab too large: %u * %u = %" PRIu64 " exceeds 4 GiB",
131 numBufs_, bufSize_, slabSize64));
132 slabSize_ = static_cast<uint32_t>(slabSize64);
133
134 void* slabPtr = nullptr;
135 if (posix_memalign(&slabPtr, 4096, slabSize_) != 0 || !slabPtr)
136 throw(rogue::GeneralError::create("rocev2::Server::Server",
137 "Failed to allocate RX slab (%u bytes)",
138 slabSize_));
139 slab_ = static_cast<uint8_t*>(slabPtr);
140 memset(slab_, 0, slabSize_);
141
142 // -------------------------------------------------------------------
143 // 2. Register slab as a single MR
144 // -------------------------------------------------------------------
145 mr_ = ibv_reg_mr(pd_, slab_, slabSize_,
146 IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE);
147 if (!mr_)
148 throw(rogue::GeneralError::create("rocev2::Server::Server",
149 "ibv_reg_mr failed"));
150
151 mrAddr_ = reinterpret_cast<uint64_t>(slab_);
152 mrRkey_ = mr_->rkey;
153
154 log_->info("MR: addr=0x%016" PRIx64 " rkey=0x%08x size=%u",
155 mrAddr_, mrRkey_, slabSize_);
156
157 // -------------------------------------------------------------------
158 // 3a. Completion channel + self-pipe (event-driven CQ, no busy-poll)
159 // The CQ below is bound to comp_channel_ so each completion makes
160 // the channel fd readable; runThread() blocks in select() on that
161 // fd plus wakeFd_ (a self-pipe stop() writes to) for prompt,
162 // sleepless shutdown. Both fds are non-blocking so the drain loops
163 // terminate cleanly (ibv_get_cq_event returns EAGAIN when empty).
164 // -------------------------------------------------------------------
165 comp_channel_ = ibv_create_comp_channel(ctx_);
166 if (!comp_channel_)
167 throw(rogue::GeneralError::create("rocev2::Server::Server",
168 "ibv_create_comp_channel failed"));
169 {
170 int flags = fcntl(comp_channel_->fd, F_GETFL);
171 if (flags < 0 ||
172 fcntl(comp_channel_->fd, F_SETFL, flags | O_NONBLOCK) < 0)
173 throw(rogue::GeneralError::create("rocev2::Server::Server",
174 "failed to set comp channel fd non-blocking"));
175 }
176
177 if (pipe(wakeFd_) != 0)
178 throw(rogue::GeneralError::create("rocev2::Server::Server",
179 "wake self-pipe creation failed"));
180 for (int i = 0; i < 2; ++i) {
181 int flags = fcntl(wakeFd_[i], F_GETFL);
182 if (flags >= 0) fcntl(wakeFd_[i], F_SETFL, flags | O_NONBLOCK);
183 }
184
185 // -------------------------------------------------------------------
186 // 3. Create Completion Queue (bound to comp_channel_ for event-driven
187 // notification; cq_context = this)
188 // -------------------------------------------------------------------
189 cq_ = ibv_create_cq(ctx_,
190 static_cast<int>(numBufs_),
191 this, comp_channel_, 0);
192 if (!cq_)
193 throw(rogue::GeneralError::create("rocev2::Server::Server",
194 "ibv_create_cq failed"));
195
196 // -------------------------------------------------------------------
197 // 4. Create RC Queue Pair
198 // -------------------------------------------------------------------
199 struct ibv_qp_init_attr qpAttr;
200 memset(&qpAttr, 0, sizeof(qpAttr));
201 qpAttr.qp_type = IBV_QPT_RC;
202 qpAttr.sq_sig_all = 0;
203 qpAttr.send_cq = cq_;
204 qpAttr.recv_cq = cq_;
205 qpAttr.cap.max_recv_wr = numBufs_;
206 qpAttr.cap.max_send_wr = 1;
207 qpAttr.cap.max_recv_sge = 1;
208 qpAttr.cap.max_send_sge = 1;
209
210 qp_ = ibv_create_qp(pd_, &qpAttr);
211 if (!qp_)
212 throw(rogue::GeneralError::create("rocev2::Server::Server",
213 "ibv_create_qp (RC) failed"));
214
215 hostQpn_ = qp_->qp_num;
216
217 // -------------------------------------------------------------------
218 // 5. QP: RESET → INIT
219 // -------------------------------------------------------------------
220 {
221 struct ibv_qp_attr attr;
222 memset(&attr, 0, sizeof(attr));
223 attr.qp_state = IBV_QPS_INIT;
224 attr.pkey_index = 0;
225 attr.port_num = ibPort_;
226 attr.qp_access_flags = IBV_ACCESS_REMOTE_WRITE |
227 IBV_ACCESS_REMOTE_READ |
228 IBV_ACCESS_LOCAL_WRITE;
229
230 if (ibv_modify_qp(qp_, &attr,
231 IBV_QP_STATE |
232 IBV_QP_PKEY_INDEX |
233 IBV_QP_PORT |
234 IBV_QP_ACCESS_FLAGS))
235 throw(rogue::GeneralError::create("rocev2::Server::Server",
236 "QP RESET→INIT failed"));
237 }
238
239 // -------------------------------------------------------------------
240 // 6. Read host GID
241 //
242 // The rdma_rxe (Soft-RoCE) kernel driver does NOT validate
243 // gidIndex against the populated range of its GID table —
244 // ibv_query_gid returns rc=0 for any in-range index per the
245 // reported gid_tbl_len (1024 on RXE) and silently writes an
246 // all-zero GID for unpopulated slots. A zero GID passes the
247 // rc==0 check here but cripples the later QP→RTR transition
248 // (dest GID cannot resolve to a peer), producing a cryptic
249 // downstream failure whose trail back to the bad gidIndex is
250 // obscured. Validate the returned GID is non-zero so users
251 // get a clear, actionable error pinned to the gidIndex they
252 // actually passed — valid RoCE GIDs are never all-zero
253 // (IPv4-mapped carries the 0xFFFF marker, IB-native uses an
254 // fe80::/10 or assigned subnet prefix).
255 // -------------------------------------------------------------------
256 union ibv_gid gid;
257 if (ibv_query_gid(ctx_, ibPort_, gidIndex_, &gid))
258 throw(rogue::GeneralError::create("rocev2::Server::Server",
259 "ibv_query_gid failed"));
260 bool zeroGid = true;
261 for (int i = 0; i < 16; ++i) {
262 if (gid.raw[i] != 0) {
263 zeroGid = false;
264 break;
265 }
266 }
267 if (zeroGid)
268 throw(rogue::GeneralError::create("rocev2::Server::Server",
269 "GID query returned all-zero for "
270 "gidIndex=%u on device '%s' "
271 "(likely out-of-range; rdma_rxe "
272 "does not validate gidIndex and "
273 "returns an empty GID for unused "
274 "slots)",
275 gidIndex_,
276 deviceName_.c_str()));
277 memcpy(hostGid_, gid.raw, 16);
278
279 // -------------------------------------------------------------------
280 // 7. Random starting PSNs (seeded per construction to avoid
281 // deterministic sequences across process restarts; RC QP PSN is
282 // expected to be randomized to reduce stale/replay confusion).
283 // -------------------------------------------------------------------
284 std::mt19937 psnRng(std::random_device {}());
285 hostRqPsn_ = psnRng() & 0xFFFFFF;
286 hostSqPsn_ = psnRng() & 0xFFFFFF;
287
288 log_->info("RC QP ready: qpn=0x%06x rqPsn=0x%06x sqPsn=0x%06x",
289 hostQpn_, hostRqPsn_, hostSqPsn_);
290 } catch (...) {
291 // Failed construction: the destructor does NOT run on a partially
292 // constructed object, so release the ibverbs resources and free the slab
293 // here. No zero-copy Buffer exists yet, so freeing the slab is safe.
294 cleanupResources();
295 if (slab_) {
296 free(slab_);
297 slab_ = nullptr;
298 }
299 throw;
300 }
301}
302
303// ---------------------------------------------------------------------------
304// cleanupResources — release the EXTERNAL ibverbs resources owned by Server
305// (QP, CQ, comp-channel, MR registration, wake pipe), in reverse order of
306// allocation. Idempotent.
307//
308// The RX slab is deliberately NOT freed here: it is this Pool's buffer backing,
309// and per the base Pool convention (ris::Pool::~Pool) its lifetime is the Server
310// object's. Freeing it in stop() would be a use-after-free for any zero-copy
311// Buffer still held downstream (each points into slab_ and keeps this Server
312// alive via shared_ptr<Pool>). ~Server frees the slab; the failed-construction
313// path frees it explicitly (the destructor does not run on a partial object).
314//
315// Serialized with postRecvWr() so a deferred retBuffer() cannot re-post using
316// qp_/mr_ while stop() is destroying them.
317// ---------------------------------------------------------------------------
318void rpr::Server::cleanupResources() {
319 std::lock_guard<std::mutex> lock(resourcesMtx_);
320
321 if (qp_) {
322 ibv_destroy_qp(qp_);
323 qp_ = nullptr;
324 }
325 if (cq_) {
326 ibv_destroy_cq(cq_);
327 cq_ = nullptr;
328 }
329 if (comp_channel_) {
330 ibv_destroy_comp_channel(comp_channel_);
331 comp_channel_ = nullptr;
332 }
333 if (mr_) {
334 ibv_dereg_mr(mr_);
335 mr_ = nullptr;
336 }
337 // slab_ is intentionally NOT freed here — it is the Pool buffer backing (see
338 // the comment above); ~Server and the failed-construction path free it.
339 for (int i = 0; i < 2; ++i) {
340 if (wakeFd_[i] >= 0) {
341 close(wakeFd_[i]);
342 wakeFd_[i] = -1;
343 }
344 }
345}
346
347// ---------------------------------------------------------------------------
348// setFpgaGid
349// ---------------------------------------------------------------------------
350void rpr::Server::setFpgaGid(const std::string& gidBytes) {
351 if (gidBytes.size() != 16)
352 throw(rogue::GeneralError::create("rocev2::Server::setFpgaGid",
353 "GID must be 16 bytes, got %zu",
354 gidBytes.size()));
355 memcpy(fpgaGid_, gidBytes.c_str(), 16);
356 log_->info("FPGA GID stored");
357}
358
359// ---------------------------------------------------------------------------
360// completeConnection — finish handshake and start receive thread
361// ---------------------------------------------------------------------------
362void rpr::Server::completeConnection(uint32_t fpgaQpn, uint32_t fpgaRqPsn,
363 uint32_t pmtu, uint32_t minRnrTimer) {
364 // Release the GIL for the ibverbs bring-up (two ibv_modify_qp transitions and
365 // the recv-WR pre-post loop) — blocking syscalls run from Python with the GIL
366 // held. Mirrors the GIL handling in AxiStreamDma's construction/bring-up.
367 rogue::GilRelease noGil;
368
369 // Reject use after stop() before touching the QP. stop() deliberately keeps
370 // slab_ alive so retained zero-copy buffers stay valid, but it destroys the
371 // QP and deregisters the MR, so every ibverbs call below would operate on a
372 // released resource. Fail fast with a lifecycle error rather than relying on
373 // the pre-post loop to surface it as an out-of-resources failure, mirroring
374 // the entry guards in AxiStreamDma::acceptReq() / acceptFrame().
375 if (!qp_ || !mr_)
376 throw(rogue::GeneralError("rocev2::Server::completeConnection",
377 "instance has been stopped or did not finish construction"));
378
379 // Single-use: a second call would reassign thread_ and orphan the
380 // original std::thread. Real misuse would also be caught by
381 // ibv_modify_qp rejecting INIT→RTR when the QP is already in RTS,
382 // but guarding here gives a clearer error and prevents the raw
383 // pointer reassignment pattern entirely.
384 if (thread_ != nullptr)
386 "rocev2::Server::completeConnection",
387 "completeConnection already invoked; destroy and recreate "
388 "Server to re-establish the RC connection"));
389
390 // pmtu is blindly static_cast<ibv_mtu>() below; validate the range up
391 // front so a direct C++ caller (bypassing the Python wrapper which
392 // already validates) can't smuggle an undefined enum value into
393 // ibv_modify_qp. Mirrors RoCEv2Server.__init__'s 1..5 check.
394 if (pmtu < 1 || pmtu > 5)
396 "rocev2::Server::completeConnection",
397 "pmtu must be in the range 1..5 "
398 "(1=256 2=512 3=1024 4=2048 5=4096); got %u", pmtu));
399
400 log_->info("completeConnection: fpgaQpn=0x%06x fpgaRqPsn=0x%06x minRnrTimer=%u",
401 fpgaQpn, fpgaRqPsn, minRnrTimer);
402
403 // Pre-post all receive WRs BEFORE moving to RTR so no incoming
404 // RDMA WRITE-with-Immediate finds an empty RQ (which would cause RNR).
405 for (uint32_t i = 0; i < numBufs_; ++i) postRecvWr(i);
406 log_->info("Pre-posted %u recv WRs", numBufs_);
407
408 // QP: INIT → RTR
409 {
410 union ibv_gid dgid;
411 memcpy(dgid.raw, fpgaGid_, 16);
412
413 struct ibv_qp_attr attr;
414 memset(&attr, 0, sizeof(attr));
415 attr.qp_state = IBV_QPS_RTR;
416 attr.path_mtu = static_cast<ibv_mtu>(pmtu);
417 attr.dest_qp_num = fpgaQpn;
418 attr.rq_psn = fpgaRqPsn;
419 attr.max_dest_rd_atomic = 16;
420 attr.min_rnr_timer = minRnrTimer;
421 attr.ah_attr.is_global = 1;
422 attr.ah_attr.grh.dgid = dgid;
423 attr.ah_attr.grh.sgid_index = gidIndex_;
424 attr.ah_attr.grh.hop_limit = 64;
425 attr.ah_attr.port_num = ibPort_;
426 attr.ah_attr.sl = 0;
427
428 if (ibv_modify_qp(qp_, &attr,
429 IBV_QP_STATE |
430 IBV_QP_AV |
431 IBV_QP_PATH_MTU |
432 IBV_QP_DEST_QPN |
433 IBV_QP_RQ_PSN |
434 IBV_QP_MAX_DEST_RD_ATOMIC |
435 IBV_QP_MIN_RNR_TIMER))
436 throw(rogue::GeneralError::create("rocev2::Server::completeConnection",
437 "QP INIT→RTR failed"));
438 }
439
440 log_->info("QP → RTR (minRnrTimer=%u)", minRnrTimer);
441
442 // QP: RTR → RTS
443 {
444 struct ibv_qp_attr attr;
445 memset(&attr, 0, sizeof(attr));
446 attr.qp_state = IBV_QPS_RTS;
447 attr.sq_psn = hostSqPsn_;
448 attr.timeout = 14;
449 attr.retry_cnt = 3;
450 attr.rnr_retry = 7; // infinite (host SQ never transmits; cosmetic consistency)
451 attr.max_rd_atomic = 16;
452
453 if (ibv_modify_qp(qp_, &attr,
454 IBV_QP_STATE |
455 IBV_QP_SQ_PSN |
456 IBV_QP_TIMEOUT |
457 IBV_QP_RETRY_CNT |
458 IBV_QP_RNR_RETRY |
459 IBV_QP_MAX_QP_RD_ATOMIC))
460 throw(rogue::GeneralError::create("rocev2::Server::completeConnection",
461 "QP RTR→RTS failed"));
462 }
463
464 log_->info("QP → RTS — ready to receive RDMA SENDs");
465
466 // Start receive thread
467 std::shared_ptr<int> scopePtr = std::make_shared<int>(0);
468 threadEn_.store(true);
469 thread_ = new std::thread(&rpr::Server::runThread, this,
470 std::weak_ptr<int>(scopePtr));
471
472#ifndef __MACH__
473 pthread_setname_np(thread_->native_handle(), "RoCEv2Server");
474#endif
475}
476
477// ---------------------------------------------------------------------------
478// getGid
479// ---------------------------------------------------------------------------
480std::string rpr::Server::getGid() const {
481 std::ostringstream oss;
482 for (int i = 0; i < 16; i += 2) {
483 if (i) oss << ':';
484 oss << std::hex << std::setfill('0')
485 << std::setw(2) << static_cast<int>(hostGid_[i])
486 << std::setw(2) << static_cast<int>(hostGid_[i+1]);
487 }
488 return oss.str();
489}
490
491// ---------------------------------------------------------------------------
492// postRecvWr — post a receive WR for slot `slot`
493// wr_id == slot index so no lookup is needed on completion
494// ---------------------------------------------------------------------------
495void rpr::Server::postRecvWr(uint32_t slot) {
496 std::lock_guard<std::mutex> lock(resourcesMtx_);
497
498 // Defensive: slot indexes into slab_; a corrupted wr_id (from the CQ)
499 // or meta (from retBuffer) must not produce an out-of-bounds slab
500 // pointer. Normal control flow keeps slot < numBufs_ because we set
501 // wr_id = slot at post time and encode slot in meta at createBuffer
502 // time, but the check is cheap and catches any future regression.
503 if (slot >= numBufs_)
504 throw(rogue::GeneralError::create("rocev2::Server::postRecvWr",
505 "slot=%u out of range (numBufs=%u)",
506 slot, numBufs_));
507
508 if (!qp_ || !mr_ || !slab_)
509 throw(rogue::GeneralError::create("rocev2::Server::postRecvWr",
510 "resources are not available for slot=%u",
511 slot));
512
513 uint8_t* bufStart = slab_ + (static_cast<uint64_t>(slot) * bufSize_);
514
515 struct ibv_sge sge;
516 memset(&sge, 0, sizeof(sge));
517 sge.addr = reinterpret_cast<uint64_t>(bufStart);
518 sge.length = bufSize_;
519 sge.lkey = mr_->lkey;
520
521 struct ibv_recv_wr wr;
522 memset(&wr, 0, sizeof(wr));
523 wr.wr_id = static_cast<uint64_t>(slot);
524 wr.sg_list = &sge;
525 wr.num_sge = 1;
526 wr.next = nullptr;
527
528 struct ibv_recv_wr* bad = nullptr;
529 if (ibv_post_recv(qp_, &wr, &bad))
530 throw(rogue::GeneralError::create("rocev2::Server::postRecvWr",
531 "ibv_post_recv failed (slot=%u)", slot));
532}
533
534// ---------------------------------------------------------------------------
535// retBuffer — zero-copy hook
536//
537// Called by Buffer::~Buffer() when the last FramePtr holding this slot is
538// released by downstream. We re-post the slot to the QP so the FPGA can
539// write into it again.
540//
541// meta lower 24 bits = slot index (set in createBuffer() call in runThread)
542// ---------------------------------------------------------------------------
543void rpr::Server::retBuffer(uint8_t* data, uint32_t meta, uint32_t rawSize) {
544 // retBuffer runs from Buffer::~Buffer(), which can fire on a Python thread
545 // holding the GIL; release it around the ibv_post_recv re-post and decCounter
546 // (which locks). Matches ris::Pool::retBuffer() and AxiStreamDma::retBuffer().
547 rogue::GilRelease noGil;
548
549 uint32_t slot = meta & 0x00FFFFFF;
550
551 log_->debug("retBuffer: re-posting slot=%u", slot);
552
553 // Re-post only while the QP and MR are still registered. That condition is
554 // evaluated inside postRecvWr() under resourcesMtx_, together with the
555 // ibv_post_recv() call itself, so a return that overlaps stop() either
556 // completes its re-post before cleanupResources() destroys the QP or throws
557 // because the resources are already gone — it can never post to a QP that is
558 // being destroyed. Deliberately NOT gated on threadEn_ here: that read would
559 // be outside the mutex and is not what makes the re-post safe. Mirrors
560 // AxiStreamDma::retBuffer(), which likewise validates its descriptor inside
561 // the lock that stop() takes to close it.
562 try {
563 postRecvWr(slot);
564 } catch (...) {
565 // A failed re-post only forfeits one receive credit, and after stop()
566 // there is no consumer for it. Never propagate: retBuffer() runs from
567 // Buffer::~Buffer().
568 }
569
570 decCounter(rawSize);
571}
572
573// ---------------------------------------------------------------------------
574// processCompletion — handle one receive completion (zero-copy).
575// Decodes the immediate, wraps the slab slot as a rogue Buffer, and forwards it
576// downstream; retBuffer() re-posts the slot's credit when downstream releases
577// the buffer. On IBV_WC_WR_FLUSH_ERR (QP in ERROR) it clears threadEn_ so the
578// caller's drain/poll loop exits.
579// ---------------------------------------------------------------------------
580void rpr::Server::processCompletion(struct ibv_wc& wc) {
581 uint32_t slot = static_cast<uint32_t>(wc.wr_id);
582
583 // Defensive: every posted WR sets wr_id = slot with slot < numBufs_; a
584 // corrupted wr_id must not be dereferenced as a slab offset below.
585 if (slot >= numBufs_) {
586 log_->warning("CQ returned out-of-range wr_id=%" PRIu64
587 " (numBufs=%u); discarding",
588 static_cast<uint64_t>(wc.wr_id), numBufs_);
589 return;
590 }
591
592 if (wc.status != IBV_WC_SUCCESS) {
593 log_->warning("CQ error: %s (slot=%u)", ibv_wc_status_str(wc.status), slot);
594
595 if (wc.status == IBV_WC_WR_FLUSH_ERR) {
596 // QP transitioned to ERROR; remaining posted WRs flush with this
597 // status. Do NOT re-post — signal the poll loop to stop.
598 log_->error("QP in ERROR state (WR_FLUSH_ERR); exiting CQ poll thread");
599 threadEn_.store(false);
600 return;
601 }
602
603 postRecvWr(slot);
604 return;
605 }
606
607 // RDMA-SEND-with-immediate completes as IBV_WC_RECV with the IBV_WC_WITH_IMM
608 // flag set (RDMA-WRITE-with-immediate would have been IBV_WC_RECV_RDMA_WITH_IMM).
609 if (wc.opcode != IBV_WC_RECV || !(wc.wc_flags & IBV_WC_WITH_IMM)) {
610 log_->warning("Unexpected opcode %d / wc_flags 0x%x (slot=%u), re-posting",
611 wc.opcode, wc.wc_flags, slot);
612 postRecvWr(slot);
613 return;
614 }
615
616 // Decode immediate value: bits [7:0] = channel id; bits [31:8] = the
617 // free-running ring position the FPGA stamped (addrCount, informational).
618 // With RDMA-SEND (two-sided) the payload landed in the CONSUMED recv-WR's
619 // SGE buffer (= slot). RoCE carries imm_data in network byte order.
620 uint32_t imm = ntohl(wc.imm_data);
621 uint8_t channel = static_cast<uint8_t>(imm & 0xFF);
622 uint32_t dataSlot = (imm >> 8) & 0x00FFFFFF;
623 uint32_t payloadLen = wc.byte_len;
624
625 if (payloadLen == 0 || payloadLen > bufSize_) {
626 log_->warning("Bad payload len=%u (slot=%u), re-posting", payloadLen, slot);
627 postRecvWr(slot);
628 return;
629 }
630
631 // Zero-copy: the SEND landed the payload at slab_ + slot*bufSize_; wrap it
632 // as a rogue Buffer. The meta is the same slot so retBuffer() re-posts THAT
633 // credit when downstream releases the buffer.
634 uint8_t* slotPtr = slab_ + (static_cast<uint64_t>(slot) * bufSize_);
635
636 ris::BufferPtr buff = createBuffer(slotPtr,
637 slot & 0x00FFFFFF,
638 payloadLen,
639 bufSize_);
640 buff->setPayload(payloadLen);
641
642 ris::FramePtr frame = ris::Frame::create();
643 frame->appendBuffer(buff);
644 frame->setChannel(channel);
645 frame->setFirstUser(SsiSof);
646 frame->setLastUser(0);
647
648 log_->debug("RX wrId=%u dataSlot=%u channel=%u len=%u",
649 slot, dataSlot, channel, payloadLen);
650
651 sendFrame(frame);
652
653 frameCount_.fetch_add(1, std::memory_order_relaxed);
654 byteCount_.fetch_add(payloadLen, std::memory_order_relaxed);
655}
656
657// ---------------------------------------------------------------------------
658// runThread — event-driven CQ completion loop (zero-copy, no busy-poll)
659//
660// Blocks in select() on the completion-channel fd (raised when the CQ delivers
661// an event) and the self-pipe wakeFd_ (raised by stop()). On a CQ event it
662// drains the channel, RE-ARMS before draining the CQ — so a completion that
663// lands during the drain still raises a fresh event rather than being missed —
664// then processes every ready completion. There is NO sleep(): the kernel
665// parks the thread until a completion or shutdown actually occurs, so the
666// receive rate is not capped by a fixed poll-then-sleep cadence.
667// ---------------------------------------------------------------------------
668void rpr::Server::runThread(std::weak_ptr<int> lockPtr) {
669 while (!lockPtr.expired()) continue;
670
671 log_->logThreadId();
672 log_->info("RoCEv2 receive thread started (event-driven)");
673
674 rogue::GilRelease noGil; // Release GIL for the entire thread;
675 // sendFrame() re-acquires via ScopedGil when
676 // calling Python slaves.
677
678 constexpr int kPollBatch = 16;
679 struct ibv_wc wcArr[kPollBatch];
680
681 const int cqFd = comp_channel_->fd;
682 const int wakeFd = wakeFd_[0];
683 const int maxFd = (cqFd > wakeFd ? cqFd : wakeFd) + 1;
684
685 // The loop body is wrapped in try/catch so that an ibverbs failure
686 // (postRecvWr throws rogue::GeneralError on ibv_post_recv failure) shuts the
687 // thread down cleanly instead of escaping the thread entry point and
688 // triggering std::terminate.
689 try {
690 // Active-poll while completions are flowing — no sleep and no per-batch
691 // interrupt/event latency (ibv_poll_cq reads the CQ from memory). The
692 // RDMA-SEND flow control is closed-loop: the FPGA self-throttles to our
693 // recv-WR re-post rate, so any per-batch wait (a sleep OR a completion-
694 // event/interrupt that is subject to NIC coalescing) lengthens the
695 // credit-return latency, the FPGA delivers in bursts, the CQ empties
696 // between batches, and throughput collapses to a bursty half-rate
697 // equilibrium. Polling keeps the re-post latency low so the loop stays
698 // in the full-rate regime.
699 //
700 // Only when the CQ has stayed empty for a bounded spin budget (the
701 // source is genuinely idle, not a brief inter-batch gap) do we ARM the
702 // completion channel and block in select() — parking the thread with no
703 // CPU spin and no sleep until the next completion or a shutdown wake.
704 constexpr uint32_t kSpinBudget = 100000; // empty polls (~sub-us each) before parking
705 uint32_t idleSpins = 0;
706
707 while (threadEn_.load()) {
708 int n = ibv_poll_cq(cq_, kPollBatch, wcArr);
709 if (n < 0) {
710 log_->error("ibv_poll_cq error (%d); exiting CQ poll thread", n);
711 break;
712 }
713 if (n > 0) {
714 for (int k = 0; k < n; ++k) processCompletion(wcArr[k]);
715 idleSpins = 0;
716 continue;
717 }
718
719 // CQ empty: spin a bounded number of polls so a brief inter-batch
720 // gap does not pay the event-wake latency.
721 if (++idleSpins < kSpinBudget) continue;
722 idleSpins = 0;
723
724 // Sustained idle: arm the channel, then re-check once (a completion
725 // may have landed between the last poll and the arm) before parking.
726 if (ibv_req_notify_cq(cq_, 0)) {
727 log_->error("ibv_req_notify_cq failed; exiting CQ poll thread");
728 break;
729 }
730 n = ibv_poll_cq(cq_, kPollBatch, wcArr);
731 if (n < 0) {
732 log_->error("ibv_poll_cq error (%d); exiting CQ poll thread", n);
733 break;
734 }
735 if (n > 0) {
736 for (int k = 0; k < n; ++k) processCompletion(wcArr[k]);
737 continue; // armed; the pending event is consumed at the next park
738 }
739
740 // Truly idle and armed: park until a completion event or a shutdown
741 // wake. No timeout — the self-pipe guarantees prompt, sleepless exit.
742 fd_set rfds;
743 FD_ZERO(&rfds);
744 FD_SET(cqFd, &rfds);
745 FD_SET(wakeFd, &rfds);
746 int s = select(maxFd, &rfds, nullptr, nullptr, nullptr);
747 if (s < 0) {
748 if (errno == EINTR) continue;
749 log_->error("select() failed (errno=%d); exiting CQ poll thread", errno);
750 break;
751 }
752 if (FD_ISSET(wakeFd, &rfds)) {
753 uint8_t drainBuf[64];
754 while (read(wakeFd, drainBuf, sizeof(drainBuf)) > 0) {}
755 break;
756 }
757 if (FD_ISSET(cqFd, &rfds)) {
758 // Consume every queued CQ event (fd is non-blocking) and ack
759 // them as a batch so the channel fd clears, then resume polling.
760 struct ibv_cq* evCq;
761 void* evCtx;
762 int events = 0;
763 while (ibv_get_cq_event(comp_channel_, &evCq, &evCtx) == 0) ++events;
764 if (events) ibv_ack_cq_events(cq_, events);
765 }
766 }
767 } catch (const rogue::GeneralError& e) {
768 log_->error("RoCEv2 receive thread exiting on ibverbs error: %s", e.what());
769 } catch (const std::exception& e) {
770 log_->error("RoCEv2 receive thread exiting on exception: %s", e.what());
771 } catch (...) {
772 log_->error("RoCEv2 receive thread exiting on unknown exception");
773 }
774
775 threadEn_.store(false);
776 log_->info("RoCEv2 receive thread stopped");
777}
778
779// ---------------------------------------------------------------------------
780// acceptFrame — TX not supported. The parameter is required by the
781// stream::Slave contract but intentionally unused; name omitted to silence
782// -Wunused-parameter without a `(void)frame;` cast in the body.
783// ---------------------------------------------------------------------------
784void rpr::Server::acceptFrame(ris::FramePtr /*frame*/) {
785 log_->warning("RoCEv2 Server::acceptFrame: TX not supported, dropping");
786}
787
788// ---------------------------------------------------------------------------
789// stop / destructor
790// ---------------------------------------------------------------------------
791void rpr::Server::stop() {
792 // Callers must quiesce completeConnection() and other public ibverbs
793 // resource operations before shutdown. Deferred zero-copy buffer returns
794 // are the one operation allowed to overlap stop(); resourcesMtx_ serializes
795 // their receive-WR re-posts with cleanupResources() below.
796
797 // Release the GIL for the duration of teardown. stop() is driven from Python
798 // (RoCEv2Server._stop) with the GIL held, and the receive thread re-acquires
799 // the GIL via ScopedGil inside sendFrame() when a downstream slave is Python;
800 // joining that thread below while holding the GIL would deadlock. Matches
801 // AxiStreamDma::stop() / udp::Server::stop() / TcpCore::stop().
802 rogue::GilRelease noGil;
803
804 // Signal the thread to exit if it is still running. Always join /
805 // delete thread_ when it is non-null: runThread() may have already
806 // cleared threadEn_ itself (e.g. on IBV_WC_WR_FLUSH_ERR or a caught
807 // exception), which would otherwise leave thread_ joinable and cause
808 // ~std::thread to terminate the process.
809 threadEn_.store(false);
810
811 // Break the receive thread out of its blocking select() immediately by
812 // making wakeFd_[0] readable — no timeout/poll wait needed for shutdown.
813 if (wakeFd_[1] >= 0) {
814 const uint8_t one = 1;
815 ssize_t wr = write(wakeFd_[1], &one, 1);
816 (void)wr; // best-effort; the thread also re-checks threadEn_
817 }
818
819 if (thread_) {
820 if (thread_->joinable()) thread_->join();
821 delete thread_;
822 thread_ = nullptr;
823 }
824 // Release the external ibverbs resources now — QP/CQ/MR(dereg)/comp-channel
825 // and the wake pipe. The RX slab is NOT released here: it is this Pool's
826 // buffer backing and zero-copy Buffers handed downstream still point into
827 // it. The governing rule is the base Pool contract — ris::Pool::~Pool()
828 // frees its buffer memory at destruction, and every Buffer holds a
829 // shared_ptr to its Pool — so the slab's lifetime is the Server object's.
830 // It is freed in ~Server (below), after the last outstanding Buffer has
831 // released the Server. AxiStreamDma::stop() now follows the same split
832 // (see #1277): it closes its per-instance descriptor while leaving the
833 // shared zero-copy mapping alive until destruction.
834 cleanupResources();
835}
836
837rpr::Server::~Server() {
838 this->stop();
839
840 // Free the RX slab last, following the base Pool convention: ris::Pool::~Pool()
841 // frees its buffer backing (dataQ_) at destruction, never in a stop(). Every
842 // zero-copy Buffer created over slab_ holds a shared_ptr to this Server
843 // (Pool::createBuffer -> Buffer::source_), so the destructor cannot run until
844 // the last outstanding Buffer is gone — the slab is therefore safe to free
845 // here and is never freed while a downstream frame still references it.
846 if (slab_) {
847 free(slab_);
848 slab_ = nullptr;
849 }
850}
851
852// ---------------------------------------------------------------------------
853// Python bindings
854// ---------------------------------------------------------------------------
855void rpr::Server::setup_python() {
856#ifndef NO_PYTHON
857 bp::class_<rpr::Server,
859 bp::bases<rpr::Core, ris::Master, ris::Slave>,
860 boost::noncopyable>(
861 "Server",
862 bp::init<std::string, uint8_t, uint8_t, uint32_t, uint32_t>(
863 (bp::arg("deviceName"),
864 bp::arg("ibPort") = 1,
865 bp::arg("gidIndex") = 0,
866 bp::arg("maxPayload") = rpr::DefaultMaxPayload,
867 bp::arg("rxQueueDepth") = rpr::DefaultRxQueueDepth)))
868 .def("create", &rpr::Server::create)
869 .staticmethod("create")
870 .def("setFpgaGid", &rpr::Server::setFpgaGid)
871 .def("completeConnection", &rpr::Server::completeConnection,
872 (bp::arg("fpgaQpn"),
873 bp::arg("fpgaRqPsn"),
874 bp::arg("pmtu") = 5,
875 bp::arg("minRnrTimer") = 1))
876 .def("getQpn", &rpr::Server::getQpn)
877 .def("getGid", &rpr::Server::getGid)
878 .def("getRqPsn", &rpr::Server::getRqPsn)
879 .def("getSqPsn", &rpr::Server::getSqPsn)
880 .def("getMrAddr", &rpr::Server::getMrAddr)
881 .def("getMrRkey", &rpr::Server::getMrRkey)
882 .def("getFrameCount", &rpr::Server::getFrameCount)
883 .def("getByteCount", &rpr::Server::getByteCount)
884 .def("stop", &rpr::Server::stop);
885
886 bp::implicitly_convertible<rpr::ServerPtr, rpr::CorePtr>();
887 bp::implicitly_convertible<rpr::ServerPtr, ris::MasterPtr>();
888 bp::implicitly_convertible<rpr::ServerPtr, ris::SlavePtr>();
889#endif
890}
Generic Rogue exception type.
char const * what() const
Returns exception text for standard exception handling.
static GeneralError create(std::string src, const char *fmt,...)
Creates a formatted error instance.
RAII helper that releases the Python GIL for a scope.
Definition GilRelease.h:36
static std::shared_ptr< rogue::Logging > create(const std::string &name, bool quiet=false)
Creates a logger instance.
Definition Logging.cpp:95
struct ibv_pd * pd_
Definition Core.h:47
struct ibv_context * ctx_
Definition Core.h:46
uint32_t maxPayload() const
Definition Core.h:65
std::shared_ptr< rogue::interfaces::stream::Buffer > BufferPtr
Shared pointer alias for Buffer.
Definition Buffer.h:270
std::shared_ptr< rogue::interfaces::stream::Frame > FramePtr
Shared pointer alias for Frame.
Definition Frame.h:549
std::shared_ptr< rogue::protocols::rocev2::Server > ServerPtr
Definition Server.h:192
static const uint32_t DefaultMaxPayload
Definition Core.h:40
static const uint32_t DefaultRxQueueDepth
Definition Core.h:35
static const uint8_t SsiSof
Definition Server.cpp:58