rogue
Loading...
Searching...
No Matches
TcpCore.cpp
Go to the documentation of this file.
1
17#include "rogue/Directives.h"
18
20
21#include <inttypes.h>
22#include <zmq.h>
23
24#include <cstring>
25#include <memory>
26#include <string>
27
28#include "rogue/GeneralError.h"
29#include "rogue/GilRelease.h"
30#include "rogue/Logging.h"
35
37
38#ifndef NO_PYTHON
39 #include <boost/python.hpp>
40namespace bp = boost::python;
41#endif
42
44ris::TcpCorePtr ris::TcpCore::create(const std::string& addr, uint16_t port, bool server) {
45 ris::TcpCorePtr r = std::make_shared<ris::TcpCore>(addr, port, server);
46 return (r);
47}
48
50ris::TcpCore::TcpCore(const std::string& addr, uint16_t port, bool server) {
51 int32_t opt;
52 std::string logstr;
53
54 this->server_ = server;
55
56 logstr = "stream.TcpCore.";
57 logstr.append(addr);
58 logstr.append(".");
59 if (server)
60 logstr.append("Server.");
61 else
62 logstr.append("Client.");
63 logstr.append(std::to_string(port));
64
65 this->bridgeLog_ = rogue::Logging::create(logstr);
66
67 this->rxPool_ = std::make_shared<ris::Pool>();
68
69 // Format address
70 this->pullAddr_ = "tcp://";
71 this->pullAddr_.append(addr);
72 this->pullAddr_.append(":");
73 this->pushAddr_ = this->pullAddr_;
74
75 this->zmqCtx_ = zmq_ctx_new();
76 this->zmqPull_ = zmq_socket(this->zmqCtx_, ZMQ_PULL);
77 this->zmqPush_ = zmq_socket(this->zmqCtx_, ZMQ_PUSH);
78
79 // Throws before threadEn_=true skip the dtor's stop(); free the context here.
80 try {
81 // Don't buffer when no connection
82 opt = 1;
83 if (zmq_setsockopt(this->zmqPush_, ZMQ_IMMEDIATE, &opt, sizeof(int32_t)) != 0)
84 throw(rogue::GeneralError("stream::TcpCore::TcpCore", "Failed to set socket immediate"));
85
86 opt = 0;
87 if (zmq_setsockopt(this->zmqPush_, ZMQ_LINGER, &opt, sizeof(int32_t)) != 0)
88 throw(rogue::GeneralError("stream::TcpCore::TcpCore", "Failed to set socket linger"));
89
90 if (zmq_setsockopt(this->zmqPull_, ZMQ_LINGER, &opt, sizeof(int32_t)) != 0)
91 throw(rogue::GeneralError("stream::TcpCore::TcpCore", "Failed to set socket linger"));
92
93 opt = 100;
94 if (zmq_setsockopt(this->zmqPull_, ZMQ_RCVTIMEO, &opt, sizeof(int32_t)) != 0)
95 throw(rogue::GeneralError("stream::TcpCore::TcpCore", "Failed to set socket receive timeout"));
96
97 // Server mode
98 if (server) {
99 this->pullAddr_.append(std::to_string(static_cast<int64_t>(port)));
100 this->pushAddr_.append(std::to_string(static_cast<int64_t>(port + 1)));
101
102 this->bridgeLog_->debug("Creating pull server port: %s", this->pullAddr_.c_str());
103
104 if (zmq_bind(this->zmqPull_, this->pullAddr_.c_str()) < 0)
105 throw(rogue::GeneralError::create("stream::TcpCore::TcpCore",
106 "Failed to bind server to port %" PRIu16
107 " at address %s, another process may be using this port",
108 port,
109 addr.c_str()));
110
111 this->bridgeLog_->debug("Creating push server port: %s", this->pushAddr_.c_str());
112
113 if (zmq_bind(this->zmqPush_, this->pushAddr_.c_str()) < 0)
114 throw(rogue::GeneralError::create("stream::TcpCore::TcpCore",
115 "Failed to bind server to port %" PRIu16
116 " at address %s, another process may be using this port",
117 port + 1,
118 addr.c_str()));
119
120 // Client mode
121 } else {
122 this->pullAddr_.append(std::to_string(static_cast<int64_t>(port + 1)));
123 this->pushAddr_.append(std::to_string(static_cast<int64_t>(port)));
124
125 this->bridgeLog_->debug("Creating pull client port: %s", this->pullAddr_.c_str());
126
127 if (zmq_connect(this->zmqPull_, this->pullAddr_.c_str()) < 0)
128 throw(rogue::GeneralError::create("stream::TcpCore::TcpCore",
129 "Failed to connect to remote port %" PRIu16 " at address %s",
130 port + 1,
131 addr.c_str()));
132
133 this->bridgeLog_->debug("Creating push client port: %s", this->pushAddr_.c_str());
134
135 if (zmq_connect(this->zmqPush_, this->pushAddr_.c_str()) < 0)
136 throw(rogue::GeneralError::create("stream::TcpCore::TcpCore",
137 "Failed to connect to remote port %" PRIu16 " at address %s",
138 port,
139 addr.c_str()));
140 }
141
142 this->bridgeLog_->debug("TCP stream bridge ready. pull=%s push=%s",
143 this->pullAddr_.c_str(),
144 this->pushAddr_.c_str());
145
146 threadEn_ = true;
147 this->thread_ = std::make_unique<std::thread>(&ris::TcpCore::runThread, this);
148
149 // Set a thread name
150#ifndef __MACH__
151 pthread_setname_np(thread_->native_handle(), "TcpCore");
152#endif
153 } catch (...) {
154 // ~std::thread on a joinable thread calls std::terminate(); join first.
155 // RCVTIMEO=100 was already set on zmqPull_ above so the worker exits
156 // within ~100 ms of threadEn_=false. Release the GIL around join() to
157 // match stop(): the worker may be blocked in sendFrame() acquiring the
158 // GIL to deliver into a Python slave, and join while holding the GIL
159 // would deadlock.
160 threadEn_ = false;
161 if (thread_) {
162 {
163 rogue::GilRelease noGil;
164 thread_->join();
165 }
166 thread_.reset();
167 }
168 if (zmqPull_ != nullptr) {
169 zmq_close(zmqPull_);
170 zmqPull_ = nullptr;
171 }
172 if (zmqPush_ != nullptr) {
173 zmq_close(zmqPush_);
174 zmqPush_ = nullptr;
175 }
176 if (zmqCtx_ != nullptr) {
177 zmq_ctx_destroy(zmqCtx_);
178 zmqCtx_ = nullptr;
179 }
180 throw;
181 }
182}
183
185ris::TcpCore::~TcpCore() {
186 this->stop();
187}
188
189// deprecated
190void ris::TcpCore::close() {
191 this->stop();
192}
193
194void ris::TcpCore::stop() {
195 if (threadEn_) {
196 rogue::GilRelease noGil;
197 threadEn_ = false;
198 thread_->join();
199 thread_.reset();
200 this->bridgeLog_->debug("Stopping TCP stream bridge. pull=%s push=%s",
201 this->pullAddr_.c_str(),
202 this->pushAddr_.c_str());
203 // Serialize with in-flight acceptFrame() callers before destroying sockets.
204 std::lock_guard<std::mutex> lock(bridgeMtx_);
205 if (zmqPull_ != nullptr) {
206 zmq_close(zmqPull_);
207 zmqPull_ = nullptr;
208 }
209 if (zmqPush_ != nullptr) {
210 zmq_close(zmqPush_);
211 zmqPush_ = nullptr;
212 }
213 if (zmqCtx_ != nullptr) {
214 zmq_ctx_destroy(zmqCtx_);
215 zmqCtx_ = nullptr;
216 }
217 }
218}
219
220bool ris::TcpCore::rebuildPushSocket() {
221 if (this->zmqPush_ != nullptr) {
222 zmq_close(this->zmqPush_);
223 this->zmqPush_ = nullptr;
224 }
225
226 this->zmqPush_ = zmq_socket(this->zmqCtx_, ZMQ_PUSH);
227 if (this->zmqPush_ == nullptr) {
228 bridgeLog_->error("Failed to create replacement push socket on %s: %s",
229 this->pushAddr_.c_str(),
230 zmq_strerror(zmq_errno()));
231 return false;
232 }
233
234 int32_t opt = 1;
235 if (zmq_setsockopt(this->zmqPush_, ZMQ_IMMEDIATE, &opt, sizeof(opt)) != 0) {
236 bridgeLog_->error("Failed to set ZMQ_IMMEDIATE on rebuilt push socket: %s",
237 zmq_strerror(zmq_errno()));
238 zmq_close(this->zmqPush_);
239 this->zmqPush_ = nullptr;
240 return false;
241 }
242
243 opt = 0;
244 if (zmq_setsockopt(this->zmqPush_, ZMQ_LINGER, &opt, sizeof(opt)) != 0) {
245 bridgeLog_->error("Failed to set ZMQ_LINGER on rebuilt push socket: %s",
246 zmq_strerror(zmq_errno()));
247 zmq_close(this->zmqPush_);
248 this->zmqPush_ = nullptr;
249 return false;
250 }
251
252 if (this->server_) {
253 if (zmq_bind(this->zmqPush_, this->pushAddr_.c_str()) < 0) {
254 bridgeLog_->error("Failed to rebind push socket to %s: %s",
255 this->pushAddr_.c_str(),
256 zmq_strerror(zmq_errno()));
257 zmq_close(this->zmqPush_);
258 this->zmqPush_ = nullptr;
259 return false;
260 }
261 } else {
262 if (zmq_connect(this->zmqPush_, this->pushAddr_.c_str()) < 0) {
263 bridgeLog_->error("Failed to reconnect push socket to %s: %s",
264 this->pushAddr_.c_str(),
265 zmq_strerror(zmq_errno()));
266 zmq_close(this->zmqPush_);
267 this->zmqPush_ = nullptr;
268 return false;
269 }
270 }
271
272 return true;
273}
274
276void ris::TcpCore::acceptFrame(ris::FramePtr frame) {
277 uint32_t x;
278 uint8_t* data;
279 uint16_t flags;
280 uint8_t chan;
281 uint8_t err;
282 zmq_msg_t msg[4];
283
284 rogue::GilRelease noGil;
285 ris::FrameLockPtr frLock = frame->lock();
286 std::lock_guard<std::mutex> lock(bridgeMtx_);
287
288 if (!threadEn_) {
289 bridgeLog_->debug("Dropping frame on stopped bridge %s (payload=%" PRIu32 ")",
290 this->pushAddr_.c_str(),
291 frame->getPayload());
292 return;
293 }
294
295 // Retry rebuild if a previous failure left zmqPush_ null.
296 if (this->zmqPush_ == nullptr) {
297 if (!rebuildPushSocket()) {
298 bridgeLog_->warning("Push socket unavailable on %s; dropping frame "
299 "(payload=%" PRIu32 ")",
300 this->pushAddr_.c_str(),
301 frame->getPayload());
302 return;
303 }
304 }
305
306 if (zmq_msg_init_size(&(msg[0]), 2) < 0) {
307 bridgeLog_->warning("Failed to init message header");
308 return;
309 }
310 if (zmq_msg_init_size(&(msg[1]), 1) < 0) {
311 bridgeLog_->warning("Failed to init message header");
312 zmq_msg_close(&(msg[0]));
313 return;
314 }
315 if (zmq_msg_init_size(&(msg[2]), 1) < 0) {
316 bridgeLog_->warning("Failed to init message header");
317 zmq_msg_close(&(msg[0]));
318 zmq_msg_close(&(msg[1]));
319 return;
320 }
321
322 if (zmq_msg_init_size(&(msg[3]), frame->getPayload()) < 0) {
323 bridgeLog_->warning("Failed to init message with size %" PRIu32, frame->getPayload());
324 zmq_msg_close(&(msg[0]));
325 zmq_msg_close(&(msg[1]));
326 zmq_msg_close(&(msg[2]));
327 return;
328 }
329
330 flags = frame->getFlags();
331 std::memcpy(zmq_msg_data(&(msg[0])), &flags, 2);
332
333 chan = frame->getChannel();
334 std::memcpy(zmq_msg_data(&(msg[1])), &chan, 1);
335
336 err = frame->getError();
337 std::memcpy(zmq_msg_data(&(msg[2])), &err, 1);
338
339 // Copy data
340 ris::FrameIterator iter = frame->begin();
341 data = reinterpret_cast<uint8_t*>(zmq_msg_data(&(msg[3])));
342 ris::fromFrame(iter, frame->getPayload(), data);
343
344 bool sendFailed = false;
345 for (x = 0; x < 4; x++) {
346 if (zmq_sendmsg(this->zmqPush_, &(msg[x]), (x == 3) ? 0 : ZMQ_SNDMORE) < 0) {
347 bridgeLog_->warning("Failed to push message part %" PRIu32 " (frame size %" PRIu32 ") on %s: %s",
348 x,
349 frame->getPayload(),
350 this->pushAddr_.c_str(),
351 zmq_strerror(zmq_errno()));
352 sendFailed = true;
353 zmq_msg_close(&(msg[x]));
354 for (uint32_t y = x + 1; y < 4; y++) zmq_msg_close(&(msg[y]));
355 break;
356 }
357 }
358
359 if (sendFailed) {
360 bridgeLog_->error("Multi-part frame failed mid-stream on part %" PRIu32
361 "; peer may have received a torso-only frame. "
362 "Resetting push socket to clear PUSH multipart FSM.",
363 x);
364
365 if (!rebuildPushSocket()) {
366 bridgeLog_->error("Unable to recover TcpCore push socket on %s; "
367 "next acceptFrame() will retry the rebuild",
368 this->pushAddr_.c_str());
369 }
370 return;
371 }
372
373 bridgeLog_->debug("Pushed TCP frame with size %" PRIu32 " on %s", frame->getPayload(), this->pushAddr_.c_str());
374}
375
377void ris::TcpCore::runThread() {
378 ris::FramePtr frame;
379 uint64_t more;
380 size_t moreSize;
381 uint8_t* data;
382 uint32_t size;
383 uint32_t msgCnt;
384 uint32_t x;
385 zmq_msg_t msg[4];
386 uint16_t flags;
387 uint8_t chan;
388 uint8_t err;
389
390 bridgeLog_->logThreadId();
391
392 while (threadEn_) {
393 for (x = 0; x < 4; x++) zmq_msg_init(&(msg[x]));
394 msgCnt = 0;
395 x = 0;
396
397 // Get message
398 do {
399 // Get the message
400 if (zmq_recvmsg(this->zmqPull_, &(msg[x]), 0) >= 0) {
401 if (x != 3) x++;
402 msgCnt++;
403
404 // Is there more data?
405 more = 0;
406 moreSize = 8;
407 zmq_getsockopt(this->zmqPull_, ZMQ_RCVMORE, &more, &moreSize);
408 } else {
409 more = 1;
410 }
411 } while (threadEn_ && more);
412
413 // Proper message received
414 if (threadEn_ && (msgCnt == 4)) {
415 // Check sizes
416 if ((zmq_msg_size(&(msg[0])) != 2) || (zmq_msg_size(&(msg[1])) != 1) || (zmq_msg_size(&(msg[2])) != 1)) {
417 bridgeLog_->warning(
418 "Bad message sizes. flags=%zu channel=%zu error=%zu",
419 zmq_msg_size(&(msg[0])),
420 zmq_msg_size(&(msg[1])),
421 zmq_msg_size(&(msg[2])));
422 for (x = 0; x < msgCnt; x++) zmq_msg_close(&(msg[x]));
423 continue; // while (1)
424 }
425
426 // Get fields
427 std::memcpy(&flags, zmq_msg_data(&(msg[0])), 2);
428 std::memcpy(&chan, zmq_msg_data(&(msg[1])), 1);
429 std::memcpy(&err, zmq_msg_data(&(msg[2])), 1);
430
431 // Get message info
432 data = reinterpret_cast<uint8_t*>(zmq_msg_data(&(msg[3])));
433 size = zmq_msg_size(&(msg[3]));
434
435 // Generate frame. Not reqLocalFrame(): the constructor starts this
436 // thread before a shared_ptr owns the bridge, and the pool on this
437 // object needs shared_from_this() to allocate.
438 frame = rxPool_->acceptReq(size, false);
439 frame->setPayload(size);
440
441 // Copy data
442 ris::FrameIterator iter = frame->begin();
443 ris::toFrame(iter, size, data);
444
445 // Set frame meta data and send
446 frame->setFlags(flags);
447 frame->setChannel(chan);
448 frame->setError(err);
449
450 bridgeLog_->debug("Pulled frame with size %" PRIu32, frame->getPayload());
451 sendFrame(frame);
452 }
453
454 for (x = 0; x < msgCnt; x++) zmq_msg_close(&(msg[x]));
455 }
456}
457
458void ris::TcpCore::setup_python() {
459#ifndef NO_PYTHON
460
461#pragma GCC diagnostic push
462#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
463 bp::class_<ris::TcpCore, ris::TcpCorePtr, bp::bases<ris::Master, ris::Slave>, boost::noncopyable>("TcpCore",
464 bp::no_init)
465 .def("close", &ris::TcpCore::close);
466#pragma GCC diagnostic pop
467
468 bp::implicitly_convertible<ris::TcpCorePtr, ris::MasterPtr>();
469 bp::implicitly_convertible<ris::TcpCorePtr, ris::SlavePtr>();
470#endif
471}
Generic Rogue exception type.
static GeneralError create(std::string src, const char *fmt,...)
Creates a formatted error instance.
RAII helper that releases the Python GIL for a scope.
Definition GilRelease.h:36
static std::shared_ptr< rogue::Logging > create(const std::string &name, bool quiet=false)
Creates a logger instance.
Definition Logging.cpp:95
Random-access byte iterator across a Frame payload.
static void fromFrame(rogue::interfaces::stream::FrameIterator &iter, uint32_t size, void *dst)
Copies bytes from a frame iterator to a destination pointer.
std::shared_ptr< rogue::interfaces::stream::TcpCore > TcpCorePtr
Shared pointer alias for TcpCore.
Definition TcpCore.h:147
std::shared_ptr< rogue::interfaces::stream::Frame > FramePtr
Shared pointer alias for Frame.
Definition Frame.h:549
static void toFrame(rogue::interfaces::stream::FrameIterator &iter, uint32_t size, void *src)
Copies bytes from a source pointer into a frame iterator.
std::shared_ptr< rogue::interfaces::stream::FrameLock > FrameLockPtr
Shared pointer alias for FrameLock.
Definition FrameLock.h:110