rogue
Loading...
Searching...
No Matches
Server.h
Go to the documentation of this file.
1
18#ifndef __ROGUE_PROTOCOLS_ROCEV2_SERVER_H__
19#define __ROGUE_PROTOCOLS_ROCEV2_SERVER_H__
20#include "rogue/Directives.h"
21
22#include <infiniband/verbs.h>
23#include <stdint.h>
24
25#include <atomic>
26#include <memory>
27#include <mutex>
28#include <string>
29#include <thread>
30#include <vector>
31
37
38namespace rogue {
39namespace protocols {
40namespace rocev2 {
41
45 private:
46 // -----------------------------------------------------------------------
47 // ibverbs resources
48 // -----------------------------------------------------------------------
49 struct ibv_cq* cq_;
50 struct ibv_qp* qp_;
51 struct ibv_mr* mr_;
52 struct ibv_comp_channel* comp_channel_; // event-driven CQ notification channel
53
54 // Contiguous slab — subdivided into numBufs_ fixed-size slots.
55 // Registered with the HCA once; never copied from.
56 uint8_t* slab_;
57 uint32_t slabSize_;
58 uint32_t numBufs_;
59 uint32_t bufSize_; // == maxPayload_ (no GRH on RC)
60
61 // -----------------------------------------------------------------------
62 // Host-side RC parameters. All initialized to 0 in the member-init list
63 // so an early throw inside the ctor body cannot leave a reader of the
64 // getters (e.g. a getXxx() hooked into a LocalVariable's localGet in the
65 // failed-construction propagation path) observing indeterminate values.
66 // Real values are written inside the ctor body as each ibverbs stage
67 // completes successfully.
68 // -----------------------------------------------------------------------
69 uint32_t hostQpn_;
70 uint8_t hostGid_[16];
71 uint32_t hostRqPsn_;
72 uint32_t hostSqPsn_;
73 uint64_t mrAddr_;
74 uint32_t mrRkey_;
75
76 // FPGA GID — set by setFpgaGid() before completeConnection()
77 uint8_t fpgaGid_[16];
78
79 // -----------------------------------------------------------------------
80 // Receive thread
81 // -----------------------------------------------------------------------
82 std::thread* thread_;
83 std::atomic<bool> threadEn_;
84
85 // Self-pipe used to break the receive thread out of its blocking select()
86 // for prompt, sleepless shutdown. stop() writes one byte to wakeFd_[1];
87 // runThread() selects on wakeFd_[0]. Both ends are non-blocking.
88 int wakeFd_[2];
89
90 // Serializes ibverbs resource teardown against deferred buffer re-posts.
91 std::mutex resourcesMtx_;
92
93 // Intentional shadow: both Core and stream::Slave expose a `log_` member,
94 // so any unqualified `log_` from inside Server would otherwise be
95 // ambiguous. Declaring our own `log_` here consolidates both names onto
96 // Server's "rocev2.Server" logger (assigned in the Server ctor) and
97 // silences the multiple-inheritance ambiguity.
98 std::shared_ptr<rogue::Logging> log_;
99
100 // RX counters
101 std::atomic<uint64_t> frameCount_;
102 std::atomic<uint64_t> byteCount_;
103
104 void postRecvWr(uint32_t slot);
105 void runThread(std::weak_ptr<int> lockPtr);
106
107 // Process one receive completion (decode immediate, wrap zero-copy buffer,
108 // forward downstream). On IBV_WC_WR_FLUSH_ERR it clears threadEn_ so the
109 // poll/drain loop in runThread() exits.
110 void processCompletion(struct ibv_wc& wc);
111
112 // Releases the external ibverbs resources owned by Server (QP, CQ,
113 // comp-channel, MR registration, wake pipe) in reverse allocation order.
114 // Idempotent. Does NOT free the RX slab: that is this Pool's buffer backing
115 // and lives with the Server object (freed in ~Server, mirroring
116 // ris::Pool::~Pool), so a zero-copy Buffer still held downstream is never
117 // stranded. Resource teardown is serialized with postRecvWr() so deferred
118 // buffer returns cannot use a QP or MR while it is being destroyed. The
119 // failed-construction path frees the slab explicitly.
120 void cleanupResources();
121
122 protected:
123 // -----------------------------------------------------------------------
124 // Zero-copy hook — called by Buffer::~Buffer() when the last downstream
125 // reference to a frame is released. Re-posts the slab slot to the QP.
126 // The slot index is encoded in the lower 24 bits of meta. May run
127 // concurrently with stop(): postRecvWr() validates the QP and MR and posts
128 // the WR under resourcesMtx_, the same lock cleanupResources() holds while
129 // destroying them, so the re-post either lands before teardown or is
130 // skipped.
131 // -----------------------------------------------------------------------
132 void retBuffer(uint8_t* data, uint32_t meta, uint32_t rawSize) override;
133
134 public:
135 static std::shared_ptr<rogue::protocols::rocev2::Server> create(
136 const std::string& deviceName,
137 uint8_t ibPort,
138 uint8_t gidIndex,
139 uint32_t maxPayload = DefaultMaxPayload,
140 uint32_t rxQueueDepth = DefaultRxQueueDepth);
141
142 Server(const std::string& deviceName,
143 uint8_t ibPort,
144 uint8_t gidIndex,
145 uint32_t maxPayload,
146 uint32_t rxQueueDepth);
147
148 ~Server();
149
163 void stop();
164
165 void setFpgaGid(const std::string& gidBytes);
166
167 // Single-use bring-up: moves the QP INIT -> RTR -> RTS and starts the
168 // receive thread. Throws rogue::GeneralError if called after stop() has
169 // released the QP and MR.
170 //
171 // minRnrTimer: IB spec RNR timer code embedded in RNR NAK packets.
172 // 1=0.01ms 14=1ms 18=4ms 22=16ms 31=491ms
173 void completeConnection(uint32_t fpgaQpn,
174 uint32_t fpgaRqPsn,
175 uint32_t pmtu = 5,
176 uint32_t minRnrTimer = 1);
177
178 uint32_t getQpn() const { return hostQpn_; }
179 std::string getGid() const;
180 uint32_t getRqPsn() const { return hostRqPsn_; }
181 uint32_t getSqPsn() const { return hostSqPsn_; }
182 uint64_t getMrAddr() const { return mrAddr_; }
183 uint32_t getMrRkey() const { return mrRkey_; }
184 uint64_t getFrameCount() const { return frameCount_.load(); }
185 uint64_t getByteCount() const { return byteCount_.load(); }
186
188
189 static void setup_python();
190};
191
192typedef std::shared_ptr<rogue::protocols::rocev2::Server> ServerPtr;
193
194} // namespace rocev2
195} // namespace protocols
196} // namespace rogue
197
198#endif
Stream master endpoint.
Definition Master.h:65
static std::shared_ptr< rogue::interfaces::stream::Master > create()
Creates a stream master.
Definition Master.cpp:40
Stream slave endpoint and default frame pool.
Definition Slave.h:72
uint32_t maxPayload() const
Definition Core.h:65
uint64_t getFrameCount() const
Definition Server.h:184
void retBuffer(uint8_t *data, uint32_t meta, uint32_t rawSize) override
Definition Server.cpp:543
void setFpgaGid(const std::string &gidBytes)
Definition Server.cpp:350
uint64_t getMrAddr() const
Definition Server.h:182
uint32_t getMrRkey() const
Definition Server.h:183
uint32_t getRqPsn() const
Definition Server.h:180
uint32_t getSqPsn() const
Definition Server.h:181
std::string getGid() const
Definition Server.cpp:480
void completeConnection(uint32_t fpgaQpn, uint32_t fpgaRqPsn, uint32_t pmtu=5, uint32_t minRnrTimer=1)
Definition Server.cpp:362
void acceptFrame(rogue::interfaces::stream::FramePtr frame) override
Definition Server.cpp:784
void stop()
Stops the receive thread and releases external ibverbs resources.
Definition Server.cpp:791
uint64_t getByteCount() const
Definition Server.h:185
std::shared_ptr< rogue::interfaces::stream::Frame > FramePtr
Shared pointer alias for Frame.
Definition Frame.h:549
std::shared_ptr< rogue::protocols::rocev2::Server > ServerPtr
Definition Server.h:192
static const uint32_t DefaultMaxPayload
Definition Core.h:40
static const uint32_t DefaultRxQueueDepth
Definition Core.h:35