rogue
Loading...
Searching...
No Matches
TcpClient.cpp
Go to the documentation of this file.
1
17#include "rogue/Directives.h"
18
20
21#include <inttypes.h>
22#include <zmq.h>
23
24#include <cstring>
25#include <memory>
26#include <string>
27#include <chrono>
28
29#include "rogue/GeneralError.h"
30#include "rogue/GilRelease.h"
31#include "rogue/Logging.h"
32#include "rogue/ScopedGil.h"
36
38
39#ifndef NO_PYTHON
40 #include <boost/python.hpp>
41namespace bp = boost::python;
42#endif
43
44namespace {
45static constexpr double DefaultReadyTimeout = 10.0;
46static constexpr double DefaultReadyPeriod = 0.1;
47} // namespace
48
50rim::TcpClientPtr rim::TcpClient::create(std::string addr, uint16_t port, bool waitReady) {
51 rim::TcpClientPtr r = std::make_shared<rim::TcpClient>(addr, port, waitReady);
52 return (r);
53}
54
56rim::TcpClient::TcpClient(std::string addr, uint16_t port, bool waitReady) : rim::Slave(4, 0xFFFFFFFF) {
57 int32_t opt;
58 std::string logstr;
59
60 logstr = "memory.TcpClient.";
61 logstr.append(addr);
62 logstr.append(".");
63 logstr.append(std::to_string(port));
64
65 this->bridgeLog_ = rogue::Logging::create(logstr);
66 this->probeSeq_ = 0;
67 this->probeId_ = 0;
68 this->probeDone_ = false;
69 this->probeResult_.clear();
70 this->waitReadyOnStart_ = waitReady;
71
72 // Format address
73 this->respAddr_ = "tcp://";
74 this->respAddr_.append(addr);
75 this->respAddr_.append(":");
76 this->reqAddr_ = this->respAddr_;
77
78 this->zmqCtx_ = zmq_ctx_new();
79 this->zmqResp_ = zmq_socket(this->zmqCtx_, ZMQ_PULL);
80 this->zmqReq_ = zmq_socket(this->zmqCtx_, ZMQ_PUSH);
81
82 // Throws before threadEn_=true skip the dtor's stop(); free the context here.
83 try {
84 // Don't buffer when no connection
85 opt = 1;
86 if (zmq_setsockopt(this->zmqReq_, ZMQ_IMMEDIATE, &opt, sizeof(int32_t)) != 0)
87 throw(rogue::GeneralError("memory::TcpClient::TcpClient", "Failed to set socket immediate"));
88
89 this->respAddr_.append(std::to_string(static_cast<int64_t>(port + 1)));
90 this->reqAddr_.append(std::to_string(static_cast<int64_t>(port)));
91
92 this->bridgeLog_->debug("Creating response client port: %s", this->respAddr_.c_str());
93
94 opt = 0;
95 if (zmq_setsockopt(this->zmqResp_, ZMQ_LINGER, &opt, sizeof(int32_t)) != 0)
96 throw(rogue::GeneralError("memory::TcpClient::TcpClient", "Failed to set socket linger"));
97
98 if (zmq_setsockopt(this->zmqReq_, ZMQ_LINGER, &opt, sizeof(int32_t)) != 0)
99 throw(rogue::GeneralError("memory::TcpClient::TcpClient", "Failed to set socket linger"));
100
101 opt = 100;
102 if (zmq_setsockopt(this->zmqResp_, ZMQ_RCVTIMEO, &opt, sizeof(int32_t)) != 0)
103 throw(rogue::GeneralError("memory::TcpClient::TcpClient", "Failed to set socket receive timeout"));
104
105 if (zmq_connect(this->zmqResp_, this->respAddr_.c_str()) < 0)
106 throw(rogue::GeneralError::create("memory::TcpClient::TcpClient",
107 "Failed to connect to remote port %" PRIu16 " at address %s",
108 port + 1,
109 addr.c_str()));
110
111 this->bridgeLog_->debug("Creating request client port: %s", this->reqAddr_.c_str());
112
113 if (zmq_connect(this->zmqReq_, this->reqAddr_.c_str()) < 0)
114 throw(rogue::GeneralError::create("memory::TcpClient::TcpClient",
115 "Failed to connect to remote port %" PRIu16 " at address %s",
116 port,
117 addr.c_str()));
118
119 // Start rx thread
120 threadEn_ = true;
121 this->thread_ = std::make_unique<std::thread>(&rim::TcpClient::runThread, this);
122
123 // Set a thread name
124#ifndef __MACH__
125 pthread_setname_np(thread_->native_handle(), "TcpClient");
126#endif
127 } catch (...) {
128 // ~std::thread on a joinable thread calls std::terminate(); join first.
129 // RCVTIMEO=100 was already set on zmqResp_ above so the worker exits
130 // within ~100 ms of threadEn_=false. Release the GIL around join() to
131 // match stop(): the worker may be blocked acquiring the GIL to deliver
132 // a transaction into Python, and joining while holding the GIL would
133 // deadlock.
134 threadEn_ = false;
135 if (thread_) {
136 {
137 rogue::GilRelease noGil;
138 thread_->join();
139 }
140 thread_.reset();
141 }
142 if (zmqResp_ != nullptr) {
143 zmq_close(zmqResp_);
144 zmqResp_ = nullptr;
145 }
146 if (zmqReq_ != nullptr) {
147 zmq_close(zmqReq_);
148 zmqReq_ = nullptr;
149 }
150 if (zmqCtx_ != nullptr) {
151 zmq_ctx_destroy(zmqCtx_);
152 zmqCtx_ = nullptr;
153 }
154 throw;
155 }
156}
157
159rim::TcpClient::~TcpClient() {
160 this->stop();
161}
162
163// deprecated
164void rim::TcpClient::close() {
165 this->stop();
166}
167
168void rim::TcpClient::stop() {
169 if (threadEn_) {
170 rogue::GilRelease noGil;
171 threadEn_ = false;
172 thread_->join();
173 thread_.reset();
174 // Serialize with in-flight doTransaction()/waitReady() senders before
175 // destroying sockets.
176 std::lock_guard<std::mutex> lock(bridgeMtx_);
177 zmq_close(this->zmqResp_);
178 zmqResp_ = nullptr;
179 zmq_close(this->zmqReq_);
180 zmqReq_ = nullptr;
181 zmq_ctx_destroy(this->zmqCtx_);
182 zmqCtx_ = nullptr;
183 }
184}
185
186bool rim::TcpClient::waitReady(double timeout, double period) {
187 uint32_t id;
188 uint64_t addr = 0;
189 uint32_t size = 0;
190 uint32_t type = rim::TcpBridgeProbe;
191 zmq_msg_t msg[4];
192 auto deadline = std::chrono::steady_clock::now() + std::chrono::duration<double>(timeout);
193
194 // Retry the probe until the overall timeout expires. This verifies the
195 // full request/response bridge path, not just local socket setup.
196 while (std::chrono::steady_clock::now() < deadline) {
197 {
198 std::lock_guard<std::mutex> probeLock(probeMtx_);
199 // Track the current probe ID so the response thread can match the
200 // incoming bridge reply back to this wait operation.
201 probeId_ = ++probeSeq_;
202 probeDone_ = false;
203 probeResult_.clear();
204 id = probeId_;
205 }
206
207 // Build a minimal internal bridge-control request. The server handles
208 // this locally and responds with the normal six-part completion
209 // message, but without touching downstream memory.
210 zmq_msg_init_size(&(msg[0]), 4);
211 std::memcpy(zmq_msg_data(&(msg[0])), &id, 4);
212 zmq_msg_init_size(&(msg[1]), 8);
213 std::memcpy(zmq_msg_data(&(msg[1])), &addr, 8);
214 zmq_msg_init_size(&(msg[2]), 4);
215 std::memcpy(zmq_msg_data(&(msg[2])), &size, 4);
216 zmq_msg_init_size(&(msg[3]), 4);
217 std::memcpy(zmq_msg_data(&(msg[3])), &type, 4);
218
219 {
220 std::lock_guard<std::mutex> block(bridgeMtx_);
221 // Send through the same request socket and multipart framing used
222 // by normal memory transactions so this exercises the real bridge
223 // path end-to-end.
224 for (uint32_t x = 0; x < 4; ++x) {
225 if (zmq_sendmsg(this->zmqReq_, &(msg[x]), (x == 3 ? 0 : ZMQ_SNDMORE) | ZMQ_DONTWAIT) < 0) {
226 bridgeLog_->debug("Readiness probe send failed for port %s", this->reqAddr_.c_str());
227 break;
228 }
229 }
230 }
231
232 for (uint32_t x = 0; x < 4; ++x) zmq_msg_close(&(msg[x]));
233
234 std::unique_lock<std::mutex> probeLock(probeMtx_);
235 // The receive thread sets probeDone_/probeResult_ when it sees a probe
236 // response with the matching ID. We only wait for one retry period
237 // here so we can re-send if the server is not up yet.
238 probeCond_.wait_for(probeLock, std::chrono::duration<double>(period), [&]() { return probeDone_ && probeId_ == id; });
239
240 if (probeDone_ && probeId_ == id && probeResult_ == "OK") {
241 bridgeLog_->debug("Readiness probe succeeded for port %s", this->reqAddr_.c_str());
242 return true;
243 }
244 }
245
246 bridgeLog_->warning("Timed out waiting for bridge readiness on port %s", this->reqAddr_.c_str());
247 return false;
248}
249
250void rim::TcpClient::start() {
251 if (!waitReadyOnStart_) return;
252
253 if (!waitReady(DefaultReadyTimeout, DefaultReadyPeriod)) {
254 throw(rogue::GeneralError::create("memory::TcpClient::start",
255 "Timed out waiting for remote TcpServer readiness on %s",
256 this->reqAddr_.c_str()));
257 }
258}
259
261void rim::TcpClient::doTransaction(rim::TransactionPtr tran) {
262 uint32_t x;
263 uint32_t msgCnt;
264 zmq_msg_t msg[5];
265 uint32_t id;
266 uint64_t addr;
267 uint32_t size;
268 uint32_t type;
269
270 rogue::GilRelease noGil;
271 std::lock_guard<std::mutex> block(bridgeMtx_);
272 rim::TransactionLockPtr lock = tran->lock();
273
274 if (this->zmqReq_ == nullptr) {
275 tran->error("TcpClient stopped");
276 return;
277 }
278
279 // ID message
280 id = tran->id();
281 zmq_msg_init_size(&(msg[0]), 4);
282 std::memcpy(zmq_msg_data(&(msg[0])), &id, 4);
283
284 // Addr message
285 addr = tran->address();
286 zmq_msg_init_size(&(msg[1]), 8);
287 std::memcpy(zmq_msg_data(&(msg[1])), &addr, 8);
288
289 // Size message
290 size = tran->size();
291 zmq_msg_init_size(&(msg[2]), 4);
292 std::memcpy(zmq_msg_data(&(msg[2])), &size, 4);
293
294 // Type message
295 type = tran->type();
296 zmq_msg_init_size(&(msg[3]), 4);
297 std::memcpy(zmq_msg_data(&(msg[3])), &type, 4);
298
299 // Write transaction
300 if (type == rim::Write || type == rim::Post) {
301 msgCnt = 5;
302 zmq_msg_init_size(&(msg[4]), size);
303 std::memcpy(zmq_msg_data(&(msg[4])), tran->begin(), size);
304
305 // Read transaction
306 } else {
307 msgCnt = 4;
308 }
309
310 bridgeLog_->debug("Requested transaction id=%" PRIu32 ", addr=0x%" PRIx64 ", size=%" PRIu32 ", type=%" PRIu32
311 ", cnt=%" PRIu32 ", port: %s",
312 id,
313 addr,
314 size,
315 type,
316 msgCnt,
317 this->reqAddr_.c_str());
318
319 // Add transaction
320 if (type == rim::Post)
321 tran->done();
322 else
323 addTransaction(tran);
324
325 // Send message
326 for (x = 0; x < msgCnt; x++) {
327 if (zmq_sendmsg(this->zmqReq_, &(msg[x]), ((x == (msgCnt - 1) ? 0 : ZMQ_SNDMORE)) | ZMQ_DONTWAIT) < 0) {
328 bridgeLog_->warning("Failed to send transaction %" PRIu32 ", msg %" PRIu32 " on %s: %s",
329 id,
330 x,
331 this->reqAddr_.c_str(),
332 zmq_strerror(zmq_errno()));
333 }
334 }
335}
336
338void rim::TcpClient::runThread() {
340 bool err;
341 uint64_t more;
342 size_t moreSize;
343 uint32_t x;
344 uint32_t msgCnt;
345 zmq_msg_t msg[6];
346 uint32_t id;
347 uint64_t addr;
348 uint32_t size;
349 uint32_t type;
350 char result[1000];
351
352 bridgeLog_->logThreadId();
353
354 while (threadEn_) {
355 for (x = 0; x < 6; x++) zmq_msg_init(&(msg[x]));
356 msgCnt = 0;
357 x = 0;
358
359 // Get message
360 do {
361 // Get the message
362 if (zmq_recvmsg(this->zmqResp_, &(msg[x]), 0) >= 0) {
363 if (x != 5) x++;
364 msgCnt++;
365
366 // Is there more data?
367 more = 0;
368 moreSize = 8;
369 zmq_getsockopt(this->zmqResp_, ZMQ_RCVMORE, &more, &moreSize);
370 } else {
371 more = 1;
372 }
373 } while (threadEn_ && more);
374
375 // Proper message received
376 if (threadEn_ && (msgCnt == 6)) {
377 // Check sizes
378 if ((zmq_msg_size(&(msg[0])) != 4) || (zmq_msg_size(&(msg[1])) != 8) || (zmq_msg_size(&(msg[2])) != 4) ||
379 (zmq_msg_size(&(msg[3])) != 4) || (zmq_msg_size(&(msg[5])) > 999)) {
380 bridgeLog_->warning(
381 "Bad message sizes. id=%zu addr=%zu size=%zu type=%zu result=%zu",
382 zmq_msg_size(&(msg[0])),
383 zmq_msg_size(&(msg[1])),
384 zmq_msg_size(&(msg[2])),
385 zmq_msg_size(&(msg[3])),
386 zmq_msg_size(&(msg[5])));
387 for (x = 0; x < msgCnt; x++) zmq_msg_close(&(msg[x]));
388 continue; // while (1)
389 }
390
391 // Get return fields
392 std::memcpy(&id, zmq_msg_data(&(msg[0])), 4);
393 std::memcpy(&addr, zmq_msg_data(&(msg[1])), 8);
394 std::memcpy(&size, zmq_msg_data(&(msg[2])), 4);
395 std::memcpy(&type, zmq_msg_data(&(msg[3])), 4);
396
397 memset(result, 0, 1000);
398 std::strncpy(result, reinterpret_cast<char*>(zmq_msg_data(&(msg[5]))), zmq_msg_size(&(msg[5])));
399
400 if (type == rim::TcpBridgeProbe) {
401 std::lock_guard<std::mutex> probeLock(probeMtx_);
402 if (id == probeId_) {
403 probeResult_ = result;
404 probeDone_ = true;
405 probeCond_.notify_all();
406 }
407 for (x = 0; x < msgCnt; x++) zmq_msg_close(&(msg[x]));
408 continue; // while (1)
409 }
410
411 // Find Transaction
412 if ((tran = getTransaction(id)) == NULL) {
413 bridgeLog_->warning("Failed to find transaction id=%" PRIu32, id);
414 for (x = 0; x < msgCnt; x++) zmq_msg_close(&(msg[x]));
415 continue; // while (1)
416 }
417
418 // Lock transaction
419 rim::TransactionLockPtr lock = tran->lock();
420
421 // No early continue past this point: every path must fall through to
422 // the GIL-protected tran release at the bottom of the loop.
423 if (tran->expired()) {
424 bridgeLog_->warning("Dropping late response for expired transaction. Id=%" PRIu32, id);
425 } else if ((addr != tran->address()) || (size != tran->size()) || (type != tran->type())) {
426 bridgeLog_->warning("Transaction data mismatch. Id=%" PRIu32, id);
427 tran->error("Transaction data mismatch in TcpClient");
428 } else if ((type != rim::Write) && (zmq_msg_size(&(msg[4])) != size)) {
429 bridgeLog_->warning("Transaction size mismatch. Id=%" PRIu32, id);
430 tran->error("Received transaction response did not match header size");
431 } else {
432 // Copy data if read
433 if (type != rim::Write) std::memcpy(tran->begin(), zmq_msg_data(&(msg[4])), size);
434 if (strcmp(result, "OK") != 0)
435 tran->errorStr(std::string(result));
436 else
437 tran->done();
438 bridgeLog_->debug("Response for transaction id=%" PRIu32 ", addr=0x%" PRIx64 ", size=%" PRIu32
439 ", type=%" PRIu32 ", cnt=%" PRIu32 ", port: %s, Result: (%s)",
440 id,
441 addr,
442 size,
443 type,
444 msgCnt,
445 this->respAddr_.c_str(),
446 result);
447 }
448 }
449 for (x = 0; x < msgCnt; x++) zmq_msg_close(&(msg[x]));
450
451 // Release here under the GIL instead of at the next getTransaction()
452 // overwrite or thread exit. A Python Slave or Hub that forwards a
453 // transaction back into C++ via super()._doTransaction() re-enters through
454 // the from-python shared_ptr converter, which attaches a Boost.Python
455 // deleter that calls Py_DECREF without the GIL. bp::no_init does not
456 // prevent this: it only suppresses __init__, not the converter.
457#ifndef NO_PYTHON
458 if (tran && Py_IsInitialized()) {
460 tran.reset();
461 } else {
462 tran.reset();
463 }
464#else
465 tran.reset();
466#endif
467 }
468}
469
470void rim::TcpClient::setup_python() {
471#ifndef NO_PYTHON
472
473#pragma GCC diagnostic push
474#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
475 bp::class_<rim::TcpClient, rim::TcpClientPtr, bp::bases<rim::Slave>, boost::noncopyable>(
476 "TcpClient",
477 bp::init<std::string, uint16_t, bp::optional<bool> >())
478 .def("close", &rim::TcpClient::close)
479 .def("waitReady", &rim::TcpClient::waitReady)
480 .def("_start", &rim::TcpClient::start)
481 .def("_stop", &rim::TcpClient::stop);
482#pragma GCC diagnostic pop
483
484 bp::implicitly_convertible<rim::TcpClientPtr, rim::SlavePtr>();
485#endif
486}
Generic Rogue exception type.
static GeneralError create(std::string src, const char *fmt,...)
Creates a formatted error instance.
RAII helper that releases the Python GIL for a scope.
Definition GilRelease.h:36
static std::shared_ptr< rogue::Logging > create(const std::string &name, bool quiet=false)
Creates a logger instance.
Definition Logging.cpp:95
RAII helper that acquires the Python GIL for a scope.
Definition ScopedGil.h:35
Memory slave device.
Definition Slave.h:54
bool waitReady(double timeout, double period)
Wait for the remote TcpServer path to respond to a bridge probe.
std::shared_ptr< rogue::interfaces::memory::TransactionLock > TransactionLockPtr
Shared pointer alias for TransactionLock.
static const uint32_t TcpBridgeProbe
Internal TCP bridge readiness probe transaction type.
Definition Constants.h:67
static const uint32_t Write
Memory write transaction type.
Definition Constants.h:43
std::shared_ptr< rogue::interfaces::memory::Transaction > TransactionPtr
Shared pointer alias for Transaction.
static const uint32_t Post
Memory posted write transaction type.
Definition Constants.h:50
std::shared_ptr< rogue::interfaces::memory::TcpClient > TcpClientPtr
Shared pointer alias for TcpClient.
Definition TcpClient.h:189