rogue
Loading...
Searching...
No Matches
AxiStreamDma.cpp
Go to the documentation of this file.
1
17#include "rogue/Directives.h"
18
20
21#include <inttypes.h>
22#include <poll.h>
23
24#include <chrono>
25#include <cstdio>
26#include <cstdlib>
27#include <map>
28#include <memory>
29#include <mutex>
30#include <string>
31#include <thread>
32
33#include "rogue/GeneralError.h"
34#include "rogue/GilRelease.h"
35#include "rogue/Helpers.h"
40
41namespace rha = rogue::hardware::axi;
43
44#ifndef NO_PYTHON
45 #include <boost/python.hpp>
46namespace bp = boost::python;
47#endif
48
49std::map<std::string, std::shared_ptr<rha::AxiStreamDmaShared> > rha::AxiStreamDma::sharedBuffers_;
50
51// Protect sharedBuffers_ against concurrent AxiStreamDma construction.
52static std::mutex sharedBuffersMtx;
53
54rha::AxiStreamDmaShared::AxiStreamDmaShared(std::string path) {
55 this->fd = -1;
56 this->path = path;
57 // Start at 0; openShared() is the single point that increments openCount
58 // for every caller (including the !zCopyEn early-return path), so a freshly
59 // constructed record must contribute zero to the refcount. Initializing to
60 // 1 here previously double-counted entries seeded by zeroCopyDisable() and
61 // was the root cause of openCount drifting negative on close.
62 this->openCount = 0;
63 this->rawBuff = NULL;
64 this->bCount = 0;
65 this->bSize = 0;
66 this->zCopyEn = true;
67}
68
70rha::AxiStreamDmaSharedPtr rha::AxiStreamDma::openShared(std::string path, rogue::LoggingPtr log) {
72 std::lock_guard<std::mutex> lock(sharedBuffersMtx);
73 std::map<std::string, rha::AxiStreamDmaSharedPtr>::iterator it;
75
76 // Entry already exists
77 if ((it = sharedBuffers_.find(path)) != sharedBuffers_.end()) {
78 ret = it->second;
79 log->debug("Reusing existing shared file descriptor for %s", path.c_str());
80
81 // Already opened by an earlier caller; just count the new user.
82 if (ret->fd != -1) {
83 ret->openCount++;
84 log->debug("Shared file descriptor already opened for %s", path.c_str());
85 return ret;
86 }
87
88 // Entry was seeded by zeroCopyDisable() (or fully closed and is being
89 // reopened with zCopyEn=false): no kernel fd will be held, but every
90 // AxiStreamDma sharing this path is still a refcount user. Without
91 // this branch, closeShared() would decrement an entry that never got
92 // counted, driving openCount negative.
93 if (!ret->zCopyEn) {
94 ret->openCount++;
95 log->debug("Zero copy is disabled. Not Mapping Buffers for %s", path.c_str());
96 return ret;
97 }
98
99 // Fall through to the open+map path; increment after success so a
100 // throw before insertion cannot leak openCount.
101
102 // Create new record
103 } else {
104 ret = std::make_shared<rha::AxiStreamDmaShared>(path);
105 log->debug("Opening new shared file descriptor for %s", path.c_str());
106 }
107
108 // We need to open device and create shared buffers
109 if ((ret->fd = ::open(path.c_str(), O_RDWR)) < 0)
110 throw(rogue::GeneralError::create("AxiStreamDma::openShared", "Failed to open device file: %s", path.c_str()));
111
112 // Check driver version ( ApiVersion 0x05 (or less) is the 32-bit address version)
113 //
114 // closeShared() leaves entries in sharedBuffers_ after openCount drops to
115 // 0 (only fd/bCount/bSize/rawBuff are reset), so an entry can be re-entered
116 // here via the "fd == -1, zCopyEn == true" fall-through. If we close the
117 // fd on error but leave ret->fd != -1, the persistent map entry will still
118 // hold the (now closed) descriptor, and the next openShared() caller would
119 // hit the "ret->fd != -1" fast-path and reuse a stale fd. Reset the
120 // shared record fd (ret->fd) to -1 (and clear any partial mapping state)
121 // on every error path before throwing. Note: ret->fd here is the shared
122 // AxiStreamDmaShared record field, distinct from the AxiStreamDma::fd_
123 // per-instance member.
124 if (dmaGetApiVersion(ret->fd) < 0x06) {
125 ::close(ret->fd);
126 ret->fd = -1;
127 throw(rogue::GeneralError("AxiStreamDma::openShared",
128 R"(Bad kernel driver version detected. Please re-compile kernel driver.
129 Note that aes-stream-driver (v5.15.2 or earlier) and rogue (v5.11.1 or earlier) are compatible with the 32-bit address API.
130 To use later versions (64-bit address API),, you will need to upgrade both rogue and aes-stream-driver at the same time to:
131 \t\taes-stream-driver = v5.16.0 (or later)
132 \t\trogue = v5.13.0 (or later))"));
133 }
134
135 // Check for mismatch in the rogue/loaded_driver API versions
136 if (dmaCheckVersion(ret->fd) < 0) {
137 ::close(ret->fd);
138 ret->fd = -1;
140 "AxiStreamDma::openShared",
141 "Rogue DmaDriver.h API Version (DMA_VERSION) does not match the aes-stream-driver API version"));
142 }
143
144 // Result may be that rawBuff = NULL
145 // Should this just be a warning?
146 ret->rawBuff = dmaMapDma(ret->fd, &(ret->bCount), &(ret->bSize));
147 if (ret->rawBuff == NULL) {
148 // dmaMapDma may have written to bCount/bSize even on failure; query
149 // the buffer count one more time for the user-facing error message,
150 // then scrub the shared record so a future openShared() on a
151 // re-entered entry starts clean.
152 uint32_t mapSize = dmaGetBuffCount(ret->fd) + 8192;
153
154 ::close(ret->fd);
155 ret->fd = -1;
156 ret->bCount = 0;
157 ret->bSize = 0;
158 ret->rawBuff = NULL;
160 "AxiStreamDma::openShared",
161 "Failed to map dma buffers. Increase vm map limit to : sudo sysctl -w vm.max_map_count=%i",
162 mapSize));
163 }
164 log->debug("Mapped buffers. bCount = %i, bSize=%i for %s", ret->bCount, ret->bSize, path.c_str());
165
166 // Open + map succeeded; count this caller and publish in the map.
167 ret->openCount++;
168 sharedBuffers_.insert(std::pair<std::string, rha::AxiStreamDmaSharedPtr>(path, ret));
169 return ret;
170}
171
173void rha::AxiStreamDma::closeShared(rha::AxiStreamDmaSharedPtr desc) {
174 rogue::GilRelease noGil;
175 std::lock_guard<std::mutex> lock(sharedBuffersMtx);
176 desc->openCount--;
177
178 if (desc->openCount == 0) {
179 if (desc->rawBuff != NULL) {
180 dmaUnMapDma(desc->fd, desc->rawBuff);
181 }
182
183 // Guard against close(-1). In the zeroCopyDisable() flow desc->fd is
184 // never opened, so calling ::close(desc->fd) here would issue a real
185 // syscall with fd=-1 and return EBADF. Skip the close when the fd
186 // was never held by this descriptor.
187 if (desc->fd != -1) {
188 ::close(desc->fd);
189 }
190 desc->fd = -1;
191 desc->bCount = 0;
192 desc->bSize = 0;
193 desc->rawBuff = NULL;
195}
196
198rha::AxiStreamDmaPtr rha::AxiStreamDma::create(std::string path, uint32_t dest, bool ssiEnable) {
199 rha::AxiStreamDmaPtr r = std::make_shared<rha::AxiStreamDma>(path, dest, ssiEnable);
200 return (r);
201}
202
203void rha::AxiStreamDma::zeroCopyDisable(std::string path) {
204 rogue::GilRelease noGil;
205 std::lock_guard<std::mutex> lock(sharedBuffersMtx);
206 std::map<std::string, rha::AxiStreamDmaSharedPtr>::iterator it;
207
208 // Entry already exists
209 if ((it = sharedBuffers_.find(path)) != sharedBuffers_.end())
210 throw(rogue::GeneralError("AxiStreamDma::zeroCopyDisable",
211 "zeroCopyDisable can't be called twice or after a device has been opened"));
212
213 // Create new record
214 rha::AxiStreamDmaSharedPtr ret = std::make_shared<rha::AxiStreamDmaShared>(path);
215 ret->zCopyEn = false;
216
217 sharedBuffers_.insert(std::pair<std::string, rha::AxiStreamDmaSharedPtr>(path, ret));
218}
219
221rha::AxiStreamDma::AxiStreamDma(std::string path, uint32_t dest, bool ssiEnable) {
222 uint8_t mask[DMA_MASK_SIZE];
223
224 dest_ = dest;
225 enSsi_ = ssiEnable;
226 retThold_ = 1;
227
228 // Create a shared pointer to use as a lock for runThread()
229 std::shared_ptr<int> scopePtr = std::make_shared<int>(0);
230
231 rogue::defaultTimeout(timeout_);
232
233 log_ = rogue::Logging::create("axi.AxiStreamDma");
234
235 rogue::GilRelease noGil;
236
237 // Attempt to open shared structure
238 desc_ = openShared(path, log_);
239
240 // Open non shared file descriptor
241 if ((fd_ = ::open(path.c_str(), O_RDWR)) < 0) {
242 closeShared(desc_);
243 throw(
244 rogue::GeneralError::create("AxiStreamDma::AxiStreamDma", "Failed to open device file: %s", path.c_str()));
245 }
246
247 // desc_->bCount and desc_->bSize live in the shared AxiStreamDmaShared
248 // instance returned by openShared(), which is also looked up under
249 // sharedBuffersMtx by other concurrent constructions on the same path.
250 // Two ctors racing here would otherwise both write the same fields
251 // without synchronisation - a data race even when the values written
252 // are identical. Take the same mutex that openShared / closeShared /
253 // zeroCopyDisable use, and BOTH read and write bCount under it (the
254 // earlier "read at function entry, write inside the conditional" form
255 // left the read racing with another ctor's write).
256 {
257 std::lock_guard<std::mutex> lock(sharedBuffersMtx);
258 if (desc_->rawBuff == NULL) {
259 desc_->bCount = dmaGetTxBuffCount(fd_) + dmaGetRxBuffCount(fd_);
260 desc_->bSize = dmaGetBuffSize(fd_);
261 }
262 if (desc_->bCount >= 1000) retThold_ = 80;
263 }
264
265 dmaInitMaskBytes(mask);
266 dmaAddMaskBytes(mask, dest_);
267
268 if (dmaSetMaskBytes(fd_, mask) < 0) {
269 closeShared(desc_);
270 ::close(fd_);
271 fd_ = -1;
272 throw(rogue::GeneralError::create("AxiStreamDma::AxiStreamDma",
273 "Failed to open device file %s with dest 0x%" PRIx32
274 "! Another process may already have it open!",
275 path.c_str(),
276 dest));
277 }
278
279 threadEn_ = true;
280 try {
281 thread_ = std::make_unique<std::thread>(&rha::AxiStreamDma::runThread, this, std::weak_ptr<int>(scopePtr));
282 } catch (...) {
283 threadEn_ = false;
284 closeShared(desc_);
285 ::close(fd_);
286 fd_ = -1;
287 throw;
288 }
289
290 // Set a thread name
291#ifndef __MACH__
292 pthread_setname_np(thread_->native_handle(), "AxiStreamDma");
293#endif
294}
295
297rha::AxiStreamDma::~AxiStreamDma() {
298 this->stop();
299
300 if (desc_) {
301 closeShared(desc_);
302 desc_.reset();
303 }
304}
305
306void rha::AxiStreamDma::stop() {
307 rogue::GilRelease noGil;
308
309 // Signal the worker to exit; idempotent so stop() can be called multiple
310 // times (user-initiated stop() followed by ~AxiStreamDma()).
311 threadEn_ = false;
312
313 // Join based on thread state, not threadEn_. runThread() can now exit on
314 // its own (e.g. the in-loop fd_ guard) by setting threadEn_ = false and
315 // returning, so gating cleanup on threadEn_ here would skip the join and
316 // ~unique_ptr<std::thread> on a still-joinable thread would call
317 // std::terminate(). Joining via joinable() handles both that early-exit
318 // path and the normal stop() path with the same code.
319 if (thread_ && thread_->joinable()) {
320 thread_->join();
321 }
322 thread_.reset();
323
324 // Close the per-instance fd exactly once. The shared DMA mapping remains
325 // open until ~AxiStreamDma() because zero-copy Rogue buffers can outlive a
326 // user-initiated stop() and still point into desc_->rawBuff.
327 {
328 std::lock_guard<std::mutex> lock(fdMtx_);
329 if (fd_ >= 0) {
330 ::close(fd_);
331 fd_ = -1;
332 }
334}
335
337void rha::AxiStreamDma::setTimeout(uint32_t timeout) {
338 if (timeout > 0) {
339 div_t divResult = div(timeout, 1000000);
340 timeout_.tv_sec = divResult.quot;
341 timeout_.tv_usec = divResult.rem;
343}
344
346void rha::AxiStreamDma::setDriverDebug(uint32_t level) {
347 dmaSetDebug(fd_, level);
348}
349
351void rha::AxiStreamDma::dmaAck() {
352 if (fd_ >= 0) axisReadAck(fd_);
353}
354
356ris::FramePtr rha::AxiStreamDma::acceptReq(uint32_t size, bool zeroCopyEn) {
357 int32_t res;
358 struct pollfd pfd;
359 uint32_t alloc;
360 ris::BufferPtr buff;
361 ris::FramePtr frame;
362 uint32_t buffSize;
363
364 // Reject use after stop() before dereferencing desc_ or entering the
365 // allocation path. stop() deliberately retains desc_ to preserve the
366 // shared DMA mapping, but closes fd_; report a clean lifecycle error
367 // instead of attempting a driver operation with the closed descriptor.
368 if (!desc_ || fd_ < 0)
369 throw rogue::GeneralError("AxiStreamDma::acceptReq",
370 "instance has been stopped or did not finish construction");
371
373 if (size > desc_->bSize)
374 buffSize = desc_->bSize;
375 else
376 buffSize = size;
377
378 // Zero copy is disabled. Allocate from memory.
379 if (zeroCopyEn == false || desc_->rawBuff == NULL) {
380 frame = reqLocalFrame(size, false);
381
382 // Allocate zero copy buffers from driver
383 } else {
384 rogue::GilRelease noGil;
385
386 // Create empty frame
387 frame = ris::Frame::create();
388 alloc = 0;
389
390 // Request may be serviced with multiple buffers
391 while (alloc < size) {
392 // Keep trying since poll call can fire
393 // but getIndex fails because we did not win the buffer lock
394 do {
395 // Check again immediately before poll() and the driver call. This
396 // provides a clean failure when the descriptor is already closed; it
397 // does not synchronize this operation with stop(), whose caller must
398 // first quiesce public driver operations.
399 // poll() imposes no FD_SETSIZE ceiling, so large fd values are fine.
400 if (fd_ < 0)
402 "AxiStreamDma::acceptReq",
403 "fd_=%d invalid; instance was stop()'d or never finished construction",
404 fd_);
405 pfd.fd = fd_;
406 pfd.events = POLLOUT;
407 pfd.revents = 0;
408
409 // Round up µs->ms so sub-ms timeouts do not collapse to a non-blocking poll().
410 int rc = poll(&pfd, 1, (timeout_.tv_sec * 1000) + ((timeout_.tv_usec + 999) / 1000));
411 // POLLERR/POLLHUP/POLLNVAL can be reported with POLLOUT also set on a fd in
412 // error state; surface that as a clear error instead of feeding a known-bad
413 // fd into dmaGetIndex() and looping on its return value.
414 if (rc > 0 && (pfd.revents & (POLLERR | POLLHUP | POLLNVAL)))
416 "AxiStreamDma::acceptReq",
417 "poll() reported fd error revents=0x%x; fd may be closed or in error state",
418 pfd.revents);
419 if (rc <= 0 || !(pfd.revents & POLLOUT)) {
420 log_->critical("AxiStreamDma::acceptReq: Timeout waiting for outbound buffer after %" PRIuLEAST32
421 ".%" PRIuLEAST32 " seconds! May be caused by outbound back pressure.",
422 timeout_.tv_sec,
423 timeout_.tv_usec);
424 res = -1;
425 } else {
426 // Attempt to get index.
427 // return of less than 0 is a failure to get a buffer
428 res = dmaGetIndex(fd_);
429 }
430 } while (res < 0);
431
432 // Mark zero copy meta with bit 31 set, lower bits are index
433 buff = createBuffer(desc_->rawBuff[res], 0x80000000 | res, buffSize, desc_->bSize);
434 frame->appendBuffer(buff);
435 alloc += buffSize;
436 }
437 }
438 return (frame);
439}
440
442void rha::AxiStreamDma::acceptFrame(ris::FramePtr frame) {
443 int32_t res;
444 struct pollfd pfd;
445 uint32_t meta;
446 uint32_t fuser;
447 uint32_t luser;
448 uint32_t cont;
449 bool emptyFrame;
450
451 // Reject use after stop() before locking or modifying the frame. Although
452 // the shared DMA mapping remains alive until destruction, the per-instance
453 // driver descriptor is closed by stop().
454 if (!desc_ || fd_ < 0)
455 throw rogue::GeneralError("AxiStreamDma::acceptFrame",
456 "instance has been stopped or did not finish construction");
457
458 rogue::GilRelease noGil;
459 ris::FrameLockPtr lock = frame->lock();
460 emptyFrame = false;
461
462 // Drop errored frames
463 if (frame->getError()) {
464 log_->warning("Dumping errored frame");
465 return;
466 }
467
468 // Go through each (*it)er in the frame
469 ris::Frame::BufferIterator it;
470 for (it = frame->beginBuffer(); it != frame->endBuffer(); ++it) {
471 (*it)->zeroHeader();
472
473 // First buffer
474 if (it == frame->beginBuffer()) {
475 fuser = frame->getFirstUser();
476 if (enSsi_) fuser |= 0x2;
477 } else {
478 fuser = 0;
479 }
480
481 // Last Buffer
482 if (it == (frame->endBuffer() - 1)) {
483 cont = 0;
484 luser = frame->getLastUser();
485
486 // Continue flag is set if this is not the last (*it)er
487 } else {
488 cont = 1;
489 luser = 0;
490 }
491
492 // Get (*it)er meta field
493 meta = (*it)->getMeta();
494
495 // Meta is zero copy as indicated by bit 31
496 if ((meta & 0x80000000) != 0) {
497 emptyFrame = true;
498
499 // Buffer is not already stale as indicates by bit 30
500 if ((meta & 0x40000000) == 0) {
501 // Write by passing (*it)er index to driver
502 if (dmaWriteIndex(fd_,
503 meta & 0x3FFFFFFF,
504 (*it)->getPayload(),
505 axisSetFlags(fuser, luser, cont),
506 dest_) <= 0) {
507 throw(rogue::GeneralError("AxiStreamDma::acceptFrame", "AXIS Write Call Failed"));
508 }
509
510 // Mark (*it)er as stale
511 meta |= 0x40000000;
512 (*it)->setMeta(meta);
513 }
514
515 // Write to pgp with (*it)er copy in driver
516 } else {
517 // Keep trying since poll call can fire
518 // but write fails because we did not win the (*it)er lock
519 do {
520 // Check again immediately before poll() and the driver call. This
521 // provides a clean failure when the descriptor is already closed; it
522 // does not synchronize this operation with stop(), whose caller must
523 // first quiesce public driver operations.
524 // poll() imposes no FD_SETSIZE ceiling, so large fd values are fine.
525 if (fd_ < 0)
527 "AxiStreamDma::acceptFrame",
528 "fd_=%d invalid; instance was stop()'d or never finished construction",
529 fd_);
530 pfd.fd = fd_;
531 pfd.events = POLLOUT;
532 pfd.revents = 0;
533
534 // Round up µs->ms so sub-ms timeouts do not collapse to a non-blocking poll().
535 int rc = poll(&pfd, 1, (timeout_.tv_sec * 1000) + ((timeout_.tv_usec + 999) / 1000));
536 // POLLERR/POLLHUP/POLLNVAL can be reported with POLLOUT also set on a fd in
537 // error state; surface that as a clear error instead of feeding a known-bad
538 // fd into dmaWrite() and reporting a generic "AXIS Write Call Failed".
539 if (rc > 0 && (pfd.revents & (POLLERR | POLLHUP | POLLNVAL)))
541 "AxiStreamDma::acceptFrame",
542 "poll() reported fd error revents=0x%x; fd may be closed or in error state",
543 pfd.revents);
544 if (rc <= 0 || !(pfd.revents & POLLOUT)) {
545 log_->critical("AxiStreamDma::acceptFrame: Timeout waiting for outbound write after %" PRIuLEAST32
546 ".%" PRIuLEAST32 " seconds! May be caused by outbound back pressure.",
547 timeout_.tv_sec,
548 timeout_.tv_usec);
549 res = 0;
550 } else {
551 // Write with (*it)er copy
552 if ((res =
553 dmaWrite(fd_, (*it)->begin(), (*it)->getPayload(), axisSetFlags(fuser, luser, 0), dest_)) <
554 0) {
555 throw(rogue::GeneralError("AxiStreamDma::acceptFrame", "AXIS Write Call Failed!!!!"));
556 }
557 }
558 } while (res == 0); // Exit out if return flag was set false
559 }
560 }
561
562 if (emptyFrame) frame->clear();
563}
564
566void rha::AxiStreamDma::retBuffer(uint8_t* data, uint32_t meta, uint32_t size) {
567 rogue::GilRelease noGil;
568
569 // Buffer is zero copy as indicated by bit 31
570 if ((meta & 0x80000000) != 0) {
571 // Return a non-stale zero-copy buffer only while the per-instance
572 // descriptor remains open. Bit 30 indicates that the buffer has
573 // already been returned to hardware. Synchronize the fd_ check and
574 // dmaRetIndex() with stop() closing the descriptor.
575 if ((meta & 0x40000000) == 0) {
576 std::lock_guard<std::mutex> lock(fdMtx_);
577 if (fd_ >= 0 && dmaRetIndex(fd_, meta & 0x3FFFFFFF) < 0)
578 throw(rogue::GeneralError("AxiStreamDma::retBuffer", "AXIS Return Buffer Call Failed!!!!"));
579 }
580 decCounter(size);
581
582 // Buffer is allocated from Pool class
583 } else {
584 Pool::retBuffer(data, meta, size);
585 }
586}
587
589void rha::AxiStreamDma::runThread(std::weak_ptr<int> lockPtr) {
590 ris::BufferPtr buff[RxBufferCount];
591 uint32_t meta[RxBufferCount];
592 uint32_t rxFlags[RxBufferCount];
593 uint32_t rxError[RxBufferCount];
594 int32_t rxSize[RxBufferCount];
595 int32_t rxCount;
596 int32_t x;
597 ris::FramePtr frame;
598 struct pollfd pfd;
599 uint8_t error;
600 uint32_t fuser;
601 uint32_t luser;
602 uint32_t cont;
603
604 fuser = 0;
605 luser = 0;
606 cont = 0;
607
608 // Wait until constructor completes
609 while (!lockPtr.expired()) continue;
610
611 // The inherited Pool needs shared_from_this() to create buffers, and the
612 // constructor starts this thread before a shared_ptr owns the object.
613 // Received data stays queued in the driver until then.
614 while (threadEn_) {
615 try {
616 ris::Pool::shared_from_this();
617 break;
618 } catch (const std::bad_weak_ptr&) {
619 std::this_thread::sleep_for(std::chrono::milliseconds(1));
620 }
621 }
622 if (!threadEn_) return;
623
624 log_->logThreadId();
625
626 // Preallocate empty frame
627 frame = ris::Frame::create();
628
629 while (threadEn_) {
630 // runThread() has no top-level catch, so a throw here would call
631 // std::terminate. This in-loop check only fires if stop()/close()
632 // races with the worker and toggles fd_ to -1; log and exit the worker
633 // cleanly so stop() can complete instead of crashing the process.
634 // poll() imposes no FD_SETSIZE ceiling, so large fd values are fine.
635 if (fd_ < 0) {
636 log_->error("AxiStreamDma::runThread: fd_ value %d invalid; exiting worker", fd_);
637 threadEn_ = false;
638 break;
639 }
640 pfd.fd = fd_;
641 pfd.events = POLLIN;
642 pfd.revents = 0;
643
644 // poll returns with available buffer (1 ms timeout)
645 if (poll(&pfd, 1, 1) > 0) {
646 // poll() can report POLLERR/POLLHUP/POLLNVAL with POLLIN unset (e.g. closed-fd race).
647 // runThread() has no top-level catch, so falling through to dmaRead on an invalid fd
648 // would throw and call std::terminate. Log and exit the worker cleanly instead.
649 if (pfd.revents & (POLLERR | POLLHUP | POLLNVAL)) {
650 log_->error("AxiStreamDma::runThread: poll reported fd error revents=0x%x; exiting worker",
651 pfd.revents);
652 threadEn_ = false;
653 break;
654 }
655 if (!(pfd.revents & POLLIN)) continue;
656 // Zero copy buffers were not allocated
657 if (desc_->rawBuff == NULL) {
658 // Allocate a buffer
659 buff[0] = allocBuffer(desc_->bSize, NULL);
660
661 // Attempt read, dest is not needed since only one lane/vc is open
662 rxSize[0] = dmaRead(fd_, buff[0]->begin(), buff[0]->getAvailable(), rxFlags, rxError, NULL);
663 if (rxSize[0] <= 0)
664 rxCount = rxSize[0];
665 else
666 rxCount = 1;
667
668 // Zero copy read
669 } else {
670 // Attempt read, dest is not needed since only one lane/vc is open
671 rxCount = dmaReadBulkIndex(fd_, RxBufferCount, rxSize, meta, rxFlags, rxError, NULL);
672
673 // Allocate a buffer, Mark zero copy meta with bit 31 set, lower bits are index
674 for (x = 0; x < rxCount; x++)
675 buff[x] = createBuffer(desc_->rawBuff[meta[x]], 0x80000000 | meta[x], desc_->bSize, desc_->bSize);
676 }
677
678 // A negative return means the driver read failed (ioctl error, driver
679 // unload, or fd closed by a concurrent stop()). runThread() has no
680 // top-level catch, so throwing here would call std::terminate. Log
681 // and exit the worker cleanly, matching the fd_/POLLERR paths above.
682 if (rxCount < 0) {
683 log_->error("AxiStreamDma::runThread: DMA read failed (ret=%" PRId32 "); exiting worker", rxCount);
684 threadEn_ = false;
685 break;
686 }
687
688 // Read was successful
689 for (x = 0; x < rxCount; x++) {
690 fuser = axisGetFuser(rxFlags[x]);
691 luser = axisGetLuser(rxFlags[x]);
692 cont = axisGetCont(rxFlags[x]);
693
694 buff[x]->setPayload(rxSize[x]);
695
696 error = frame->getError();
697
698 // Receive error
699 error |= (rxError[x] & 0xFF);
700
701 // First buffer of frame
702 if (frame->isEmpty()) frame->setFirstUser(fuser & 0xFF);
703
704 // Last buffer of frame
705 if (cont == 0) {
706 frame->setLastUser(luser & 0xFF);
707 if (enSsi_ && ((luser & 0x1) != 0)) error |= 0x80;
708 }
709
710 frame->setError(error);
711 frame->appendBuffer(buff[x]);
712 buff[x].reset();
713
714 // If continue flag is not set, push frame and get a new empty frame
715 if (cont == 0) {
716 sendFrame(frame);
717 frame = ris::Frame::create();
718 }
719 }
720 }
722}
723
725std::string rha::AxiStreamDma::getGitVersion() {
726 return dmaGetGitVersion(fd_);
727}
728
730uint32_t rha::AxiStreamDma::getApiVersion() {
731 return dmaGetApiVersion(fd_);
732}
733
735uint32_t rha::AxiStreamDma::getBuffSize() {
736 return dmaGetBuffSize(fd_);
737}
738
740uint32_t rha::AxiStreamDma::getRxBuffCount() {
741 return dmaGetRxBuffCount(fd_);
742}
743
745uint32_t rha::AxiStreamDma::getRxBuffinUserCount() {
747}
748
750uint32_t rha::AxiStreamDma::getRxBuffinHwCount() {
752}
753
755uint32_t rha::AxiStreamDma::getRxBuffinPreHwQCount() {
757}
758
760uint32_t rha::AxiStreamDma::getRxBuffinSwQCount() {
762}
763
765uint32_t rha::AxiStreamDma::getRxBuffMissCount() {
767}
768
770uint32_t rha::AxiStreamDma::getTxBuffCount() {
771 return dmaGetTxBuffCount(fd_);
772}
773
775uint32_t rha::AxiStreamDma::getTxBuffinUserCount() {
777}
778
780uint32_t rha::AxiStreamDma::getTxBuffinHwCount() {
782}
783
785uint32_t rha::AxiStreamDma::getTxBuffinPreHwQCount() {
787}
788
790uint32_t rha::AxiStreamDma::getTxBuffinSwQCount() {
792}
793
795uint32_t rha::AxiStreamDma::getTxBuffMissCount() {
796 return dmaGetTxBuffMissCount(fd_);
797}
798
799void rha::AxiStreamDma::setup_python() {
800#ifndef NO_PYTHON
801
802 bp::class_<rha::AxiStreamDma, rha::AxiStreamDmaPtr, bp::bases<ris::Master, ris::Slave>, boost::noncopyable>(
803 "AxiStreamDma",
804 bp::init<std::string, uint32_t, bool>())
805 .def("setDriverDebug", &rha::AxiStreamDma::setDriverDebug)
806 .def("dmaAck", &rha::AxiStreamDma::dmaAck)
807 .def("setTimeout", &rha::AxiStreamDma::setTimeout)
808 .def("getGitVersion", &rha::AxiStreamDma::getGitVersion)
809 .def("getApiVersion", &rha::AxiStreamDma::getApiVersion)
810 .def("getBuffSize", &rha::AxiStreamDma::getBuffSize)
811 .def("getRxBuffCount", &rha::AxiStreamDma::getRxBuffCount)
812 .def("getRxBuffinUserCount", &rha::AxiStreamDma::getRxBuffinUserCount)
813 .def("getRxBuffinHwCount", &rha::AxiStreamDma::getRxBuffinHwCount)
814 .def("getRxBuffinPreHwQCount", &rha::AxiStreamDma::getRxBuffinPreHwQCount)
815 .def("getRxBuffinSwQCount", &rha::AxiStreamDma::getRxBuffinSwQCount)
816 .def("getRxBuffMissCount", &rha::AxiStreamDma::getRxBuffMissCount)
817 .def("getTxBuffCount", &rha::AxiStreamDma::getTxBuffCount)
818 .def("getTxBuffinUserCount", &rha::AxiStreamDma::getTxBuffinUserCount)
819 .def("getTxBuffinHwCount", &rha::AxiStreamDma::getTxBuffinHwCount)
820 .def("getTxBuffinPreHwQCount", &rha::AxiStreamDma::getTxBuffinPreHwQCount)
821 .def("getTxBuffinSwQCount", &rha::AxiStreamDma::getTxBuffinSwQCount)
822 .def("getTxBuffMissCount", &rha::AxiStreamDma::getTxBuffMissCount)
823 .def("zeroCopyDisable", &rha::AxiStreamDma::zeroCopyDisable);
824
825 bp::implicitly_convertible<rha::AxiStreamDmaPtr, ris::MasterPtr>();
826 bp::implicitly_convertible<rha::AxiStreamDmaPtr, ris::SlavePtr>();
827#endif
828}
static std::mutex sharedBuffersMtx
static uint32_t axisGetLuser(uint32_t flags)
Extracts last-user flag from packed AXIS flags.
Definition AxisDriver.h:80
static uint32_t axisSetFlags(uint32_t fuser, uint32_t luser, uint32_t cont)
Packs AXIS first-user, last-user, and continuation bits.
Definition AxisDriver.h:51
static uint32_t axisGetFuser(uint32_t flags)
Extracts first-user flag from packed AXIS flags.
Definition AxisDriver.h:69
static void axisReadAck(int32_t fd)
Issues AXIS read-acknowledge ioctl command.
Definition AxisDriver.h:104
static uint32_t axisGetCont(uint32_t flags)
Extracts continuation flag from packed AXIS flags.
Definition AxisDriver.h:91
static std::string dmaGetGitVersion(int32_t fd)
Get the DMA Driver's Git Version.
Definition DmaDriver.h:727
static ssize_t dmaGetTxBuffinPreHwQCount(int32_t fd)
Get the transmit buffer count in pre-hardware queue.
Definition DmaDriver.h:662
static ssize_t dmaGetTxBuffCount(int32_t fd)
Get the transmit buffer count.
Definition DmaDriver.h:623
static ssize_t dmaGetTxBuffMissCount(int32_t fd)
Get the transmit buffer missing count.
Definition DmaDriver.h:688
static ssize_t dmaGetRxBuffinHwCount(int32_t fd)
Get the receive buffer count in hardware.
Definition DmaDriver.h:571
static ssize_t dmaGetApiVersion(int32_t fd)
Get API version of the DMA driver.
Definition DmaDriver.h:930
static ssize_t dmaWriteIndex(int32_t fd, uint32_t index, size_t size, uint32_t flags, uint32_t dest)
Writes data to a DMA channel using an index.
Definition DmaDriver.h:246
static void dmaInitMaskBytes(uint8_t *mask)
Initialize DMA mask byte array.
Definition DmaDriver.h:867
static void dmaAddMaskBytes(uint8_t *mask, uint32_t dest)
Add a destination to the DMA mask byte array.
Definition DmaDriver.h:881
static ssize_t dmaReadBulkIndex(int32_t fd, uint32_t count, int32_t *ret, uint32_t *index, uint32_t *flags, uint32_t *error, uint32_t *dest)
Receive frame and access memory-mapped buffer.
Definition DmaDriver.h:449
#define DMA_MASK_SIZE
Definition DmaDriver.h:118
static ssize_t dmaSetMaskBytes(int32_t fd, uint8_t *mask)
Set mask byte array to the driver.
Definition DmaDriver.h:902
static ssize_t dmaGetRxBuffinUserCount(int32_t fd)
Get the receive buffer count in user.
Definition DmaDriver.h:558
static ssize_t dmaGetTxBuffinHwCount(int32_t fd)
Get the transmit buffer count in hardware.
Definition DmaDriver.h:649
static ssize_t dmaGetBuffSize(int32_t fd)
Get the buffer size.
Definition DmaDriver.h:701
static ssize_t dmaGetRxBuffinSwQCount(int32_t fd)
Get the receive buffer count in software queue.
Definition DmaDriver.h:597
static ssize_t dmaGetRxBuffCount(int32_t fd)
Get the receive buffer count.
Definition DmaDriver.h:545
static int32_t dmaGetIndex(int32_t fd)
Get the current write buffer index.
Definition DmaDriver.h:519
static ssize_t dmaGetRxBuffinPreHwQCount(int32_t fd)
Get the receive buffer count in pre-hardware queue.
Definition DmaDriver.h:584
static ssize_t dmaRetIndex(int32_t fd, uint32_t index)
Post an index back to the DMA.
Definition DmaDriver.h:486
static ssize_t dmaSetDebug(int32_t fd, uint32_t level)
Set debugging level for DMA operations.
Definition DmaDriver.h:816
static ssize_t dmaGetTxBuffinSwQCount(int32_t fd)
Get the transmit buffer count in software queue.
Definition DmaDriver.h:675
static ssize_t dmaGetBuffCount(int32_t fd)
Get the buffer count.
Definition DmaDriver.h:714
static ssize_t dmaCheckVersion(int32_t fd)
Check API version of the DMA driver.
Definition DmaDriver.h:915
static void ** dmaMapDma(int32_t fd, uint32_t *count, uint32_t *size)
Map user space to DMA buffers.
Definition DmaDriver.h:747
static ssize_t dmaUnMapDma(int32_t fd, void **buffer)
Unmap user space from DMA buffers.
Definition DmaDriver.h:791
static ssize_t dmaRead(int32_t fd, void *buf, size_t maxSize, uint32_t *flags, uint32_t *error, uint32_t *dest)
Receive Frame.
Definition DmaDriver.h:380
static ssize_t dmaGetTxBuffinUserCount(int32_t fd)
Get the transmit buffer count in user.
Definition DmaDriver.h:636
static ssize_t dmaWrite(int32_t fd, const void *buf, size_t size, uint32_t flags, uint32_t dest)
Writes data to a DMA channel.
Definition DmaDriver.h:218
static ssize_t dmaGetRxBuffMissCount(int32_t fd)
Get the receive buffer missing count.
Definition DmaDriver.h:610
Generic Rogue exception type.
static GeneralError create(std::string src, const char *fmt,...)
Creates a formatted error instance.
RAII helper that releases the Python GIL for a scope.
Definition GilRelease.h:36
static std::shared_ptr< rogue::Logging > create(const std::string &name, bool quiet=false)
Creates a logger instance.
Definition Logging.cpp:95
std::shared_ptr< rogue::hardware::axi::AxiStreamDma > AxiStreamDmaPtr
std::shared_ptr< rogue::hardware::axi::AxiStreamDmaShared > AxiStreamDmaSharedPtr
std::shared_ptr< rogue::interfaces::stream::Buffer > BufferPtr
Shared pointer alias for Buffer.
Definition Buffer.h:270
std::shared_ptr< rogue::interfaces::stream::Frame > FramePtr
Shared pointer alias for Frame.
Definition Frame.h:549
std::shared_ptr< rogue::interfaces::stream::FrameLock > FrameLockPtr
Shared pointer alias for FrameLock.
Definition FrameLock.h:110
std::shared_ptr< rogue::Logging > LoggingPtr
Shared pointer alias for Logging.
Definition Logging.h:205
void defaultTimeout(struct timeval &tout)
Returns Rogue default timeout as a timeval.
Definition Helpers.h:49