rogue
Loading...
Searching...
No Matches
StreamWriter.cpp
Go to the documentation of this file.
1
31#include "rogue/Directives.h"
32
34
35#include <chrono>
36#include <errno.h>
37#include <fcntl.h>
38#include <inttypes.h>
39#include <stdint.h>
40#include <sys/time.h>
41#include <unistd.h>
42
43#include <cstring>
44#include <map>
45#include <memory>
46#include <string>
47#include <thread>
48
49#include "rogue/GeneralError.h"
50#include "rogue/GilRelease.h"
54
57
58#ifndef NO_PYTHON
59 #include <boost/python.hpp>
60namespace bp = boost::python;
61#endif
62
64ruf::StreamWriterPtr ruf::StreamWriter::create() {
65 ruf::StreamWriterPtr s = std::make_shared<ruf::StreamWriter>();
66 return (s);
67}
68
70void ruf::StreamWriter::setup_python() {
71#ifndef NO_PYTHON
72 bp::class_<ruf::StreamWriter, ruf::StreamWriterPtr, boost::noncopyable>("StreamWriter", bp::init<>())
73 .def("open", &ruf::StreamWriter::open)
74 .def("close", &ruf::StreamWriter::close)
75 .def("isOpen", &ruf::StreamWriter::isOpen)
76 .def("setRaw", &ruf::StreamWriter::setRaw)
77 .def("getRaw", &ruf::StreamWriter::getRaw)
78 .def("setBufferSize", &ruf::StreamWriter::setBufferSize)
79 .def("setMaxSize", &ruf::StreamWriter::setMaxSize)
80 .def("setDropErrors", &ruf::StreamWriter::setDropErrors)
81 .def("getChannel", &ruf::StreamWriter::getChannel)
82 .def("getTotalSize", &ruf::StreamWriter::getTotalSize)
83 .def("getCurrentSize", &ruf::StreamWriter::getCurrentSize)
84 .def("getBandwidth", &ruf::StreamWriter::getBandwidth)
85 .def("getFrameCount", &ruf::StreamWriter::getFrameCount)
86 .def("waitFrameCount", &ruf::StreamWriter::waitFrameCount);
87#endif
88}
89
91ruf::StreamWriter::StreamWriter() {
92 baseName_ = "";
93 fd_ = -1;
94 sizeLimit_ = 0;
95 buffSize_ = 0;
96 currSize_ = 0;
97 totSize_ = 0;
98 buffer_ = NULL;
99 frameCount_ = 0;
100 bandwidthBytes_ = 0;
101 currBuffer_ = 0;
102 dropErrors_ = false;
103 isOpen_ = false;
104 raw_ = false;
105
106 log_ = rogue::Logging::create("fileio.StreamWriter");
107}
108
110ruf::StreamWriter::~StreamWriter() {
111 this->close();
112 if (buffer_ != NULL) free(buffer_);
113 buffer_ = NULL;
114}
115
117void ruf::StreamWriter::open(std::string file) {
118 std::string name;
119
120 rogue::GilRelease noGil;
121 std::lock_guard<std::mutex> lock(mtx_);
122 isOpen_ = false;
123 flush();
124
125 // Close if open
126 if (fd_ >= 0) ::close(fd_);
127 fd_ = -1;
128
129 baseName_ = file;
130 name = file;
131 fdIdx_ = 1;
132
133 if (sizeLimit_ > 0) name.append(".1");
134
135 // The data file must not already exist. O_EXCL refuses to open an existing
136 // file so a previous session's data is never overwritten or appended to.
137 if ((fd_ = ::open(name.c_str(),
138 O_WRONLY | O_CREAT | O_EXCL,
139 S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)) < 0) {
140 if (errno == EEXIST)
141 throw(rogue::GeneralError::create("StreamWriter::open", "Data file already exists: %s", name.c_str()));
142 throw(rogue::GeneralError::create("StreamWriter::open", "Failed to open data file: %s", name.c_str()));
143 }
144
145 totSize_ = 0;
146 currSize_ = 0;
147 frameCount_ = 0;
148 bandwidthBytes_ = 0;
149 bandwidthHistory_.clear();
150 currBuffer_ = 0;
151
152 // Iterate over all channels and reset their frame counts
153 for (std::map<uint32_t, ruf::StreamWriterChannelPtr>::iterator it = channelMap_.begin(); it != channelMap_.end();
154 ++it) {
155 it->second->setFrameCount(0);
156 }
157 isOpen_ = true;
158}
159
161void ruf::StreamWriter::close() {
162 rogue::GilRelease noGil;
163 std::lock_guard<std::mutex> lock(mtx_);
164 isOpen_ = false;
165 flush();
166 bandwidthBytes_ = 0;
167 bandwidthHistory_.clear();
168 if (fd_ >= 0) ::close(fd_);
169 fd_ = -1;
170}
171
173bool ruf::StreamWriter::isOpen() {
174 return (isOpen_);
175}
176
178void ruf::StreamWriter::setRaw(bool raw) {
179 raw_ = raw;
180}
181
183bool ruf::StreamWriter::getRaw() {
184 return raw_;
185}
186
188void ruf::StreamWriter::setBufferSize(uint32_t size) {
189 rogue::GilRelease noGil;
190 std::lock_guard<std::mutex> lock(mtx_);
191
192 // No change
193 if (size != buffSize_) {
194 // Flush data out of current buffer
195 flush();
196
197 // Free old buffer
198 if (buffer_ != NULL) free(buffer_);
199 buffer_ = NULL;
200 buffSize_ = 0;
201
202 // Buffer is enabled
203 if (size != 0) {
204 // Create new buffer
205 if ((buffer_ = reinterpret_cast<uint8_t*>(malloc(size))) == NULL)
206 throw(rogue::GeneralError::create("StreamWriter::setBufferSize",
207 "Failed to allocate buffer with size = %" PRIu32,
208 size));
209 buffSize_ = size;
210 }
211 }
212}
213
215void ruf::StreamWriter::setMaxSize(uint64_t size) {
216 rogue::GilRelease noGil;
217 std::lock_guard<std::mutex> lock(mtx_);
218 sizeLimit_ = size;
219}
220
222void ruf::StreamWriter::setDropErrors(bool drop) {
223 dropErrors_ = drop;
224}
225
227ruf::StreamWriterChannelPtr ruf::StreamWriter::getChannel(uint8_t channel) {
228 rogue::GilRelease noGil;
229 std::lock_guard<std::mutex> lock(mtx_);
230 if (channelMap_.count(channel) == 0) {
231 channelMap_[channel] = ruf::StreamWriterChannel::create(shared_from_this(), channel);
232 }
233 return (channelMap_[channel]);
234}
235
237uint64_t ruf::StreamWriter::getTotalSize() {
238 rogue::GilRelease noGil;
239 std::lock_guard<std::mutex> lock(mtx_);
240 return (totSize_ + currBuffer_);
241}
242
244uint64_t ruf::StreamWriter::getCurrentSize() {
245 rogue::GilRelease noGil;
246 std::lock_guard<std::mutex> lock(mtx_);
247 return (currSize_ + currBuffer_);
248}
249
251double ruf::StreamWriter::getBandwidth() {
252 rogue::GilRelease noGil;
253 std::lock_guard<std::mutex> lock(mtx_);
254
255 auto now = std::chrono::steady_clock::now();
256 pruneBandwidth(now);
257
258 if (bandwidthHistory_.empty()) return (0.0);
259
260 auto windowStart = bandwidthHistory_.front().first;
261 double seconds = std::chrono::duration<double>(now - windowStart).count();
262 if (seconds <= 0.0) return (static_cast<double>(bandwidthBytes_));
263 return (static_cast<double>(bandwidthBytes_) / seconds);
264}
265
267uint32_t ruf::StreamWriter::getFrameCount() {
268 return (frameCount_);
269}
270
271bool ruf::StreamWriter::waitFrameCount(uint32_t count, uint64_t timeout) {
272 struct timeval endTime;
273 struct timeval sumTime;
274 struct timeval curTime;
275
276 rogue::GilRelease noGil;
277 std::unique_lock<std::mutex> lock(mtx_);
278
279 if (timeout != 0) {
280 gettimeofday(&curTime, NULL);
281
282 div_t divResult = div(timeout, 1000000);
283 sumTime.tv_sec = divResult.quot;
284 sumTime.tv_usec = divResult.rem;
285
286 timeradd(&curTime, &sumTime, &endTime);
287 }
288
289 while (frameCount_ < count) {
290 cond_.wait_for(lock, std::chrono::microseconds(1000));
291
292 if (timeout != 0) {
293 gettimeofday(&curTime, NULL);
294 if (timercmp(&curTime, &endTime, >)) break;
295 }
296 }
297
298 return (frameCount_ >= count);
299}
300
301void ruf::StreamWriter::pruneBandwidth(std::chrono::steady_clock::time_point now) {
302 auto cutoff = now - std::chrono::seconds(1);
303 while (!bandwidthHistory_.empty() && bandwidthHistory_.front().first < cutoff) {
304 bandwidthBytes_ -= bandwidthHistory_.front().second;
305 bandwidthHistory_.pop_front();
306 }
307}
308
309void ruf::StreamWriter::recordBandwidth(uint32_t size) {
310 if (size == 0) return;
311 auto now = std::chrono::steady_clock::now();
312 pruneBandwidth(now);
313 bandwidthHistory_.emplace_back(now, size);
314 bandwidthBytes_ += size;
315}
316
318void ruf::StreamWriter::writeFile(uint8_t channel, std::shared_ptr<rogue::interfaces::stream::Frame> frame) {
319 ris::Frame::BufferIterator it;
320 uint32_t value;
321 uint32_t size;
322
323 if ((frame->getPayload() == 0) || (dropErrors_ && (frame->getError() != 0))) return;
324
325 rogue::GilRelease noGil;
326 std::unique_lock<std::mutex> lock(mtx_);
327
328 if (fd_ >= 0) {
329 // Raw mode
330 if ( raw_ ) {
331 size = frame->getPayload();
332 checkSize(size);
333
334 // Written size has extra 4 bytes in non raw mode
335 // Check file size, including size header
336 } else {
337 size = frame->getPayload() + 4;
338 checkSize(size + 4);
339 }
340
341 if (!raw_) {
342 // First write size
343 intWrite(&size, 4);
344
345 // Create EVIO header
346 value = frame->getFlags();
347 value |= (frame->getError() << 16);
348 value |= (channel << 24);
349 intWrite(&value, 4);
350 }
351
352 // Write buffers
353 for (it = frame->beginBuffer(); it != frame->endBuffer(); ++it) intWrite((*it)->begin(), (*it)->getPayload());
354
355 // Update counters
356 frameCount_++;
357 cond_.notify_all();
358 }
359}
360
362void ruf::StreamWriter::intWrite(void* data, uint32_t size) {
363 if (fd_ < 0) return;
364
365 // New size is larger than buffer size, flush
366 if ((size + currBuffer_) > buffSize_) flush();
367
368 // Attempted write is larger than buffer, raw write
369 // This is called if buffer is disabled
370 if (size > buffSize_) {
371 if (write(fd_, data, size) != static_cast<int32_t>(size)) {
372 ::close(fd_);
373 fd_ = -1;
374 log_->error("Write failed, closing file!");
375 return;
376 }
377 currSize_ += size;
378 totSize_ += size;
379 // Append to buffer if non zero
380 } else if (buffSize_ > 0 && size > 0) {
381 std::memcpy(buffer_ + currBuffer_, data, size);
382 currBuffer_ += size;
383 }
384
385 recordBandwidth(size);
386}
387
389void ruf::StreamWriter::checkSize(uint32_t size) {
390 std::string name;
391
392 // No Limit
393 if (sizeLimit_ == 0) return;
394
395 // Bad configuration
396 if (size > sizeLimit_)
397 throw(rogue::GeneralError("StreamWriter::checkSize", "Frame size is larger than file size limit"));
398
399 // File size (including buffer) is larger than max size
400 if ((size + currBuffer_ + currSize_) > sizeLimit_) {
401 flush();
402
403 // Close and update index
404 ::close(fd_);
405 fdIdx_++;
406
407 name = baseName_ + "." + std::to_string(fdIdx_);
408
409 // Open new file. The rollover file must not already exist, see open().
410 // On failure the writer is left closed so no further data is written.
411 if ((fd_ = ::open(name.c_str(),
412 O_WRONLY | O_CREAT | O_EXCL,
413 S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)) < 0) {
414 isOpen_ = false;
415 if (errno == EEXIST)
416 throw(rogue::GeneralError::create("StreamWriter::checkSize", "Data file already exists: %s", name.c_str()));
417 throw(rogue::GeneralError::create("StreamWriter::checkSize", "Failed to open file %s", name.c_str()));
418 }
419
420 currSize_ = 0;
421 }
422}
423
425void ruf::StreamWriter::flush() {
426 if (currBuffer_ > 0) {
427 if (write(fd_, buffer_, currBuffer_) != static_cast<int32_t>(currBuffer_)) {
428 ::close(fd_);
429 fd_ = -1;
430 log_->error("Write failed, closing file!");
431 currBuffer_ = 0;
432 return;
433 }
434 currSize_ += currBuffer_;
435 totSize_ += currBuffer_;
436 currBuffer_ = 0;
437 }
438}
Generic Rogue exception type.
static GeneralError create(std::string src, const char *fmt,...)
Creates a formatted error instance.
RAII helper that releases the Python GIL for a scope.
Definition GilRelease.h:36
static std::shared_ptr< rogue::Logging > create(const std::string &name, bool quiet=false)
Creates a logger instance.
Definition Logging.cpp:95
std::shared_ptr< rogue::utilities::fileio::StreamWriter > StreamWriterPtr
std::shared_ptr< rogue::utilities::fileio::StreamWriterChannel > StreamWriterChannelPtr