rogue
Loading...
Searching...
No Matches
Core.cpp
Go to the documentation of this file.
1
17#include "rogue/Directives.h"
18
20
21#include <infiniband/verbs.h>
22#include <stdint.h>
23
24#include <cstring>
25#include <string>
26
27#include "rogue/GeneralError.h"
28#include "rogue/GilRelease.h"
29
31
32#ifndef NO_PYTHON
33 #include <boost/python.hpp>
34namespace bp = boost::python;
35#endif
36
38rpr::Core::Core(const std::string& deviceName,
39 uint8_t ibPort,
40 uint8_t gidIndex,
41 uint32_t maxPayload)
42 : ctx_(nullptr),
43 pd_(nullptr),
44 deviceName_(deviceName),
45 ibPort_(ibPort),
46 gidIndex_(gidIndex),
47 maxPayload_(maxPayload) {
48
49 log_ = rogue::Logging::create("rocev2.Core");
50
51 // Device discovery, open, and protection-domain allocation may block in
52 // the provider or kernel. This guard must live in Core: base construction
53 // finishes before the derived Server constructor body can release the GIL.
55
56 // -----------------------------------------------------------------------
57 // 1. Find and open the requested ibverbs device
58 // devList is freed on every exit path (success and failure) so a
59 // partial-construction throw never leaks the device list.
60 // -----------------------------------------------------------------------
61 int numDevices = 0;
62 struct ibv_device** devList = ibv_get_device_list(&numDevices);
63 if (!devList || numDevices == 0) {
64 if (devList) ibv_free_device_list(devList);
65 throw(rogue::GeneralError::create("rocev2::Core::Core",
66 "No RDMA devices found on this host"));
67 }
68
69 struct ibv_device* dev = nullptr;
70 for (int i = 0; i < numDevices; ++i) {
71 if (deviceName_ == ibv_get_device_name(devList[i])) {
72 dev = devList[i];
73 break;
74 }
75 }
76
77 if (!dev) {
78 ibv_free_device_list(devList);
79 throw(rogue::GeneralError::create("rocev2::Core::Core",
80 "RDMA device '%s' not found",
81 deviceName_.c_str()));
82 }
83
84 ctx_ = ibv_open_device(dev);
85 ibv_free_device_list(devList);
86
87 if (!ctx_)
88 throw(rogue::GeneralError::create("rocev2::Core::Core",
89 "Failed to open RDMA device '%s'",
90 deviceName_.c_str()));
91
92 // -----------------------------------------------------------------------
93 // 2. Allocate Protection Domain
94 // On failure we close ctx_ explicitly: the destructor does not run on
95 // a partially-constructed object, so without this the context would
96 // leak.
97 // -----------------------------------------------------------------------
98 pd_ = ibv_alloc_pd(ctx_);
99 if (!pd_) {
100 ibv_close_device(ctx_);
101 ctx_ = nullptr;
102 throw(rogue::GeneralError::create("rocev2::Core::Core",
103 "Failed to allocate protection domain"));
104 }
105
106 log_->info("Opened RoCEv2 device '%s', port %u, GID index %u",
107 deviceName_.c_str(), ibPort_, gidIndex_);
108}
109
111//
112// CQ / QP / MR are owned by the derived Server class and are torn down in
113// Server::stop() before the inherited Core destructor runs.
114rpr::Core::~Core() {
115 // Server::stop() has already returned (and reacquired the GIL) by the time
116 // this base destructor runs, so release it again for the final device
117 // teardown.
118 rogue::GilRelease noGil;
119
120 if (pd_) ibv_dealloc_pd(pd_);
121 if (ctx_) ibv_close_device(ctx_);
122}
123
124void rpr::Core::setup_python() {
125#ifndef NO_PYTHON
126 bp::class_<rpr::Core, rpr::CorePtr, boost::noncopyable>("Core", bp::no_init)
127 .def("maxPayload", &rpr::Core::maxPayload);
128
129 // Expose defaults as module-level attributes:
130 // rogue.protocols.rocev2.DefaultMaxPayload
131 // rogue.protocols.rocev2.DefaultRxQueueDepth
132 bp::scope().attr("DefaultMaxPayload") = rpr::DefaultMaxPayload;
133 bp::scope().attr("DefaultRxQueueDepth") = rpr::DefaultRxQueueDepth;
134#endif
135}
static GeneralError create(std::string src, const char *fmt,...)
Creates a formatted error instance.
RAII helper that releases the Python GIL for a scope.
Definition GilRelease.h:36
static std::shared_ptr< rogue::Logging > create(const std::string &name, bool quiet=false)
Creates a logger instance.
Definition Logging.cpp:95
std::shared_ptr< rogue::Logging > log_
Definition Core.h:55
struct ibv_pd * pd_
Definition Core.h:47
struct ibv_context * ctx_
Definition Core.h:46
static const uint32_t DefaultMaxPayload
Definition Core.h:40
static const uint32_t DefaultRxQueueDepth
Definition Core.h:35