Fixposition SDK 0.0.0-heads/main-0-g408dc89
Collection of c++ libraries and apps for use with Fixposition products on Linux
Loading...
Searching...
No Matches
fpl.hpp
Go to the documentation of this file.
1/**
2 * \verbatim
3 * ___ ___
4 * \ \ / /
5 * \ \/ / Copyright (c) Fixposition AG (www.fixposition.com) and contributors
6 * / /\ \ License: see the LICENSE file
7 * /__/ \__\
8 * \endverbatim
9 *
10 * @file
11 * @brief Fixposition SDK: .fpl logfile helpers
12 *
13 * @page FPSDK_COMMON_FPL .fpl logfile utilities
14 *
15 * **API**: fpsdk_common/fpl.hpp and fpsdk::common::fpl
16 *
17 */
18#ifndef __FPSDK_COMMON_FPL_HPP__
19#define __FPSDK_COMMON_FPL_HPP__
20
21/* LIBC/STL */
22#include <cstdint>
23#include <iostream>
24#include <memory>
25#include <string>
26#include <vector>
27
28/* EXTERNAL */
29#include <boost/iostreams/filtering_stream.hpp>
30#include <boost/iostreams/stream.hpp>
31
32/* PACKAGE */
33#include "time.hpp"
34
35namespace fpsdk {
36namespace common {
37/**
38 * @brief .fpl logfile utilities
39 */
40namespace fpl {
41/* ****************************************************************************************************************** */
42
43/**
44 * @brief FplMessage types
45 */
46enum class FplType : uint16_t
47{ // clang-format off
48 UNSPECIFIED = 0x0000, //!< Invalid
49 ROSMSGDEF = 0x0101, //!< Message definition for a topic (a.k.a. "connection header" in rosbag speak)
50 ROSMSGBIN = 0x0102, //!< Serialised ROS message with timestamp
51 LOGMETA = 0x0105, //!< Logfile meta data
52 STREAMMSG = 0x0106, //!< Stream message raw data with timestamp
53 LOGSTATUS = 0x0107, //!< Logging status
54 FILEDUMP = 0x0108, //!< Dump of an entire (small, and text) file
55 BLOB = 0xbaad, //!< Arbitrary data, not FplMessage framing
56 INT_D = 0xffaa, //!< Fixposition internal use only
57 INT_F = 0xffbb, //!< Fixposition internal use only
58 INT_X = 0xffcc, //!< Fixposition internal use only
59}; // clang-format on
60
61/**
62 * @brief Stringify type
63 *
64 * @param[in] type The type
65 *
66 * @returns a unique string identifying the type
67 */
68const char* FplTypeStr(const FplType type);
69
70// ---------------------------------------------------------------------------------------------------------------------
71
72/**
73 * @brief One message in the log
74 */
75class FplMessage
76{
77 public:
78 FplMessage();
79
80 /**
81 * @brief Parse message from buffer
82 *
83 * @param[in] data Pointer to start of message data
84 * @param[in] size Size of message data
85 *
86 * @returns 0 if the data is not a valid message,
87 * -1 if the size is too small to decide, or
88 * the message size (>0) if a valid message was found and loaded
89 */
90 int Parse(const uint8_t* data, const uint32_t size);
91
92 /**
93 * @brief Get message
94 *
95 * @returns a reference to the message data
96 */
97 const std::vector<uint8_t>& Raw() const;
98 /**
99 * @brief Get message data
100 *
101 * @returns a pointer to the message data (size = Size())
102 */
103 const uint8_t* RawData() const;
104
105 /**
106 * @brief Get message size
107 *
108 * @returns the size of the message (data = Data())
109 */
110 uint32_t RawSize() const;
111
112 /**
113 * @brief Get payload (log) type
114 *
115 * @returns type of the payload
116 */
118
119 /**
120 * @brief Get payload data
121 *
122 * @returns a pointer to the payload data
123 */
124 const uint8_t* PayloadData() const;
125
126 /**
127 * @brief Get payload size
128 *
129 * @returns size of the payload data
130 */
131 uint32_t PayloadSize() const;
132
133 // Debugging stuff, only with FplParser. For debugging.
134 uint64_t file_pos_; //!< Offset in logfile of the log message
135 uint64_t file_size_; //!< Size in the logfile of the log message
136 uint64_t file_seq_; //!< Message sequence counter in logfile
137
138 static constexpr std::uint32_t MAX_SIZE = 2000000; //!< Maximum message size
139 static constexpr std::size_t MAX_NAME_LEN = 100; //!< Maximum length of (file, topic, ...) names
140
141 /**
142 * @brief Make message a BLOB
143 *
144 * @param[in] data The data
145 * @param[in] size The size of the data
146 */
147 void SetBlob(const uint8_t* data, const uint32_t size);
148
149 private:
150 static constexpr uint16_t MAGIC = 0x55aa; //!< Start of message marker
151 static constexpr uint32_t HEADER_SIZE = 8; //!< Message header size
152 FplType payload_type_; //!< Payload type
153 uint32_t payload_size_; //!< Payload size
154 std::vector<uint8_t> message_; //!< Entire message
155};
156
157// ---------------------------------------------------------------------------------------------------------------------
158
159/**
160 * @brief Logfile parser
161 */
162class FplParser
163{
164 public:
165 FplParser();
166
167 /**
168 * @brief Add data to parser
169 *
170 * @param[in] data Pointer to data, can be NULL, in which case \c size is ignored
171 * @param[in] size Size of data, can be <= 0, in which case \c data is ignored
172 *
173 * The suggested chunk size to add to the parser is FplMessage::MAX_SIZE
174 *
175 * @returns true if data was added to the parser, false if there was not enough space left (parser overflow)
176 */
177 bool Add(const uint8_t* data, const std::size_t size);
178
179 /**
180 * @brief Reset parser
181 *
182 * Resets the parser state and discards all collected data.
183 */
184 void Reset();
185
186 /**
187 * @brief Process data in parser, return message
188 *
189 * @param[out] log_msg The detected message
190 *
191 * @returns true if a message was detected, false otherwise (meaning: not enough data in parser). Note that also in
192 * the false case the \c log_msg may be modified.
193 */
194 bool Process(FplMessage& log_msg);
195
196 private:
197 std::vector<uint8_t> buf_; //!< Working buffer
198 std::size_t offs_; //!< Current offset into buf_
199 std::size_t size_; //!< Current size of useful data in buf_ (counted from offs_)
200 std::size_t file_pos_; //!< Offset into "file", i.e. total number of bytes processed so far
201 std::size_t file_seq_; //!< Count of messages from "file"
202 void EmitBlob(FplMessage& log_msg); //!< Helper for Process()
203};
204
205// ---------------------------------------------------------------------------------------------------------------------
206
207/**
208 * @brief Read log messages from logfile
209 */
210class FplFileReader
211{
212 public:
213 FplFileReader();
214
215 /**
216 * @brief Open logfile
217 *
218 * @param[in] path The logfile
219 *
220 * @returns true on success, false otherwise
221 */
222 bool Open(const std::string& path);
223
224 /**
225 * @brief Get next log message
226 *
227 * @param[out] log_msg The log message
228 *
229 * @returns true if a next log message was found, false at end of file (or other problems)
230 */
231 bool Next(FplMessage& log_msg);
232
233 /**
234 * @brief Get progress report
235 *
236 * @param[out] progress Progress in [%] (approximate with gzipped files)
237 * @param[out] rate Rate in [MiB/s]
238 *
239 * @returns true if is a good time to print the progress report, false otherwise
240 */
241 bool GetProgress(double& progress, double& rate);
242
243 private:
244 std::string path_; //!< Logfile path
245 std::unique_ptr<std::istream> fh_; //!< File handle
246 FplParser parser_; //!< Parser
247 std::vector<uint8_t> buf_; //!< Read buffer
248 std::size_t pos_; //!< Current position in logfile
249 std::size_t count_; //!< Number of messages
250 std::size_t size_; //!< Size of logfile
251 bool is_gz_; //!< File is gzipped
252 double last_progress_; //!< GetProgress() state
253 uint64_t last_pos_; //!< GetProgress() state
254 double last_secs_; //!< GetProgress() state
255};
256
257// ---------------------------------------------------------------------------------------------------------------------
258
259/**
260 * @brief Helper for extracting meta data
261 */
263{
264 /**
265 * @brief Constructor
266 *
267 * @param[in] log_msg .fpl log message
268 */
269 LogMeta(const FplMessage& log_msg);
270 bool valid_ = false; //!< Data valid, successfully extracted from message
271 std::string info_; //!< Stringification of (some of the) data, for debugging
272 std::string hw_uid_; //!< Hardware UID
273 std::string product_model_; //!< Product model
274 std::string sw_version_; //!< Software version
275 uint32_t log_start_time_posix_ = 0; //!< Start time of logging
276 std::string log_start_time_iso_; //!< Start time of logging
277 std::string log_profile_; //!< Logging configuration profile name
278 std::string log_target_; //!< Loging target name
279 std::string log_filename_; //!< Logfile name
280 std::string yaml_; //!< Raw meta data YAML
281};
282
283// ---------------------------------------------------------------------------------------------------------------------
284
285/**
286 * @brief Helper for extracting recording status data
287 */
289{
290 /**
291 * @brief Constructor
292 *
293 * @param[in] log_msg .fpl log message
294 */
295 LogStatus(const FplMessage& log_msg);
296 bool valid_; //!< Data valid, successfully extracted from message
297
298 // Version 1 and later
299 std::string info_; //!< Stringification of (some of the) data, for debugging
300 std::string state_; //!< Logging state: "stopped", "logging", "stopping"
301 uint32_t queue_size_ = 0; //!< Queue size
302 uint32_t queue_peak_ = 0; //!< Queue peak size
303 uint32_t queue_skip_ = 0; //!< Number of skipped messages (queue full)
304 uint32_t queue_bsize_ = 0; //!< Output queue size [bytes]
305 uint32_t queue_bpeak_ = 0; //!< Output queue peak size [bytes]
306
307 uint32_t log_count_ = 0; //!< Number of logged messages
308 uint32_t log_errors_ = 0; //!< Number of messages failed to log (failed to write/send)
309 uint64_t log_size_ = 0; //!< Total size of logged messages [bytes]
310 uint32_t log_duration_ = 0; //!< Duration of logging [s]
311
312 // Version 2 and later
313 uint32_t log_time_posix_ = 0; //!< Approximate time
314 std::string log_time_iso_; //!< Approximate time
315 int8_t pos_source_ = 0; //!< Approximate sensor position source (see POS_SOURCE_... below)
316 int8_t pos_fix_type_ = 0; //!< Approximate sensor position fix type (see fpsdk::common::gnss::FixType)
317 double pos_lat_ = 0.0; //!< Approximate sensor position latitude [deg]
318 double pos_lon_ = 0.0; //!< Approximate sensor position longitude [deg]
319 double pos_height_ = 0.0; //!< Approximate sensor position height [m]
320
321 static constexpr int8_t POS_SOURCE_UNKNOWN = 0; //!< Position source unknown resp. not available
322 static constexpr int8_t POS_SOURCE_GNSS = 1; //!< Position source is GNSS
323 static constexpr int8_t POS_SOURCE_FUSION = 2; //!< Position source is Fusion
324
325 std::string yaml_; //!< Raw status data YAML
326};
327
328// ---------------------------------------------------------------------------------------------------------------------
329
330/**
331 * @brief Helper for extracting ROS message definition (the relevant fields from the "connection header")
332 */
334{
335 /**
336 * @brief Constructor
337 *
338 * @param[in] log_msg .fpl log message
339 */
340 RosMsgDef(const FplMessage& log_msg);
341 bool valid_; //!< Data valid, successfully extracted from message
342 std::string info_; //!< Stringification of (some of the) data, for debugging
343 std::string topic_name_; //!< The topic name
344 std::string msg_name_; //!< The message name (a.k.a. data type)
345 std::string msg_md5_; //!< The message definition MD5 sum
346 std::string msg_def_; //!< The message definition (the .msg file and also the defs for all the types)
347};
348
349// ---------------------------------------------------------------------------------------------------------------------
350
351/**
352 * @brief Helper for extracting a serialised ROS message
353 */
355{
356 /**
357 * @brief Constructor
358 *
359 * @param[in] log_msg .fpl log message
360 */
361 RosMsgBin(const FplMessage& log_msg);
362 bool valid_; //!< Data valid, successfully extracted from message
363 std::string info_; //!< Stringification of (some of the) data, for debugging
364 std::string topic_name_; //!< The topic name
365 time::RosTime rec_time_; //!< Recording timestamp
366 std::vector<uint8_t> msg_data_; //!< Serialised ROS message data
367};
368
369// ---------------------------------------------------------------------------------------------------------------------
370
371/**
372 * @brief Helper for extracting data of a stream message (NMEA, RTCM3, etc.)
373 */
375{
376 /**
377 * @brief Constructor
378 *
379 * @param[in] log_msg .fpl log message
380 */
381 StreamMsg(const FplMessage& log_msg);
382 bool valid_; //!< Data valid, successfully extracted from message
383 std::string info_; //!< Stringification of (some of the) data, for debugging
384 time::RosTime rec_time_; //!< Recording timestamp
385 std::string stream_name_; //!< Stream name
386 std::vector<uint8_t> msg_data_; //!< Message data
387};
388
389// ---------------------------------------------------------------------------------------------------------------------
390
391/**
392 * @brief Helper for extracting data of a stream message (NMEA, RTCM3, etc.)
393 */
395{
396 /**
397 * @brief Constructor
398 *
399 * @param[in] log_msg .fpl log message
400 */
401 FileDump(const FplMessage& log_msg);
402 bool valid_; //!< Data valid, successfully extracted from message
403 std::string info_; //!< Stringification of (some of the) data, for debugging
404 std::string filename_; //!< Filename
405 time::Time mtime_; //!< File modification time
406 std::vector<uint8_t> data_; //!< File contents
407};
408
409/* ****************************************************************************************************************** */
410} // namespace fpl
411} // namespace common
412} // namespace fpsdk
413#endif // __FPSDK_COMMON_FPL_HPP__
bool Next(FplMessage &log_msg)
Get next log message.
bool GetProgress(double &progress, double &rate)
Get progress report.
bool Open(const std::string &path)
Open logfile.
One message in the log.
Definition fpl.hpp:76
const std::vector< uint8_t > & Raw() const
Get message.
const uint8_t * RawData() const
Get message data.
int Parse(const uint8_t *data, const uint32_t size)
Parse message from buffer.
void SetBlob(const uint8_t *data, const uint32_t size)
Make message a BLOB.
uint64_t file_size_
Size in the logfile of the log message.
Definition fpl.hpp:135
static constexpr std::size_t MAX_NAME_LEN
Maximum length of (file, topic, ...) names.
Definition fpl.hpp:139
uint32_t PayloadSize() const
Get payload size.
uint64_t file_seq_
Message sequence counter in logfile.
Definition fpl.hpp:136
const uint8_t * PayloadData() const
Get payload data.
uint32_t RawSize() const
Get message size.
uint64_t file_pos_
Offset in logfile of the log message.
Definition fpl.hpp:134
FplType PayloadType() const
Get payload (log) type.
static constexpr std::uint32_t MAX_SIZE
Maximum message size.
Definition fpl.hpp:138
bool Process(FplMessage &log_msg)
Process data in parser, return message.
bool Add(const uint8_t *data, const std::size_t size)
Add data to parser.
void Reset()
Reset parser.
.fpl logfile utilities
Definition fpl.hpp:40
FplType
FplMessage types.
Definition fpl.hpp:47
@ INT_X
Fixposition internal use only.
Definition fpl.hpp:58
@ BLOB
Arbitrary data, not FplMessage framing.
Definition fpl.hpp:55
@ LOGSTATUS
Logging status.
Definition fpl.hpp:53
@ ROSMSGBIN
Serialised ROS message with timestamp.
Definition fpl.hpp:50
@ ROSMSGDEF
Message definition for a topic (a.k.a. "connection header" in rosbag speak)
Definition fpl.hpp:49
@ LOGMETA
Logfile meta data.
Definition fpl.hpp:51
@ INT_F
Fixposition internal use only.
Definition fpl.hpp:57
@ FILEDUMP
Dump of an entire (small, and text) file.
Definition fpl.hpp:54
@ INT_D
Fixposition internal use only.
Definition fpl.hpp:56
@ STREAMMSG
Stream message raw data with timestamp.
Definition fpl.hpp:52
const char * FplTypeStr(const FplType type)
Stringify type.
File and path utilities.
Definition path.hpp:37
Fixposition SDK: Common library.
Definition doc.hpp:21
Fixposition SDK.
time::Time mtime_
File modification time.
Definition fpl.hpp:405
std::string info_
Stringification of (some of the) data, for debugging.
Definition fpl.hpp:403
std::vector< uint8_t > data_
File contents.
Definition fpl.hpp:406
FileDump(const FplMessage &log_msg)
Constructor.
std::string filename_
Filename.
Definition fpl.hpp:404
bool valid_
Data valid, successfully extracted from message.
Definition fpl.hpp:402
uint32_t log_start_time_posix_
Start time of logging.
Definition fpl.hpp:275
std::string info_
Stringification of (some of the) data, for debugging.
Definition fpl.hpp:271
std::string yaml_
Raw meta data YAML.
Definition fpl.hpp:280
std::string log_filename_
Logfile name.
Definition fpl.hpp:279
std::string sw_version_
Software version.
Definition fpl.hpp:274
LogMeta(const FplMessage &log_msg)
Constructor.
std::string log_target_
Loging target name.
Definition fpl.hpp:278
std::string product_model_
Product model.
Definition fpl.hpp:273
std::string log_start_time_iso_
Start time of logging.
Definition fpl.hpp:276
bool valid_
Data valid, successfully extracted from message.
Definition fpl.hpp:270
std::string log_profile_
Logging configuration profile name.
Definition fpl.hpp:277
std::string hw_uid_
Hardware UID.
Definition fpl.hpp:272
static constexpr int8_t POS_SOURCE_GNSS
Position source is GNSS.
Definition fpl.hpp:322
std::string state_
Logging state: "stopped", "logging", "stopping".
Definition fpl.hpp:300
bool valid_
Data valid, successfully extracted from message.
Definition fpl.hpp:296
static constexpr int8_t POS_SOURCE_FUSION
Position source is Fusion.
Definition fpl.hpp:323
uint32_t queue_size_
Queue size.
Definition fpl.hpp:301
uint32_t log_count_
Number of logged messages.
Definition fpl.hpp:307
std::string yaml_
Raw status data YAML.
Definition fpl.hpp:325
uint32_t queue_peak_
Queue peak size.
Definition fpl.hpp:302
uint32_t queue_skip_
Number of skipped messages (queue full)
Definition fpl.hpp:303
uint32_t queue_bpeak_
Output queue peak size [bytes].
Definition fpl.hpp:305
static constexpr int8_t POS_SOURCE_UNKNOWN
Position source unknown resp. not available.
Definition fpl.hpp:321
uint32_t queue_bsize_
Output queue size [bytes].
Definition fpl.hpp:304
int8_t pos_source_
Approximate sensor position source (see POS_SOURCE_... below)
Definition fpl.hpp:315
uint32_t log_duration_
Duration of logging [s].
Definition fpl.hpp:310
uint64_t log_size_
Total size of logged messages [bytes].
Definition fpl.hpp:309
std::string info_
Stringification of (some of the) data, for debugging.
Definition fpl.hpp:299
int8_t pos_fix_type_
Approximate sensor position fix type (see fpsdk::common::gnss::FixType)
Definition fpl.hpp:316
LogStatus(const FplMessage &log_msg)
Constructor.
double pos_height_
Approximate sensor position height [m].
Definition fpl.hpp:319
uint32_t log_errors_
Number of messages failed to log (failed to write/send)
Definition fpl.hpp:308
double pos_lon_
Approximate sensor position longitude [deg].
Definition fpl.hpp:318
uint32_t log_time_posix_
Approximate time.
Definition fpl.hpp:313
std::string log_time_iso_
Approximate time.
Definition fpl.hpp:314
double pos_lat_
Approximate sensor position latitude [deg].
Definition fpl.hpp:317
std::string topic_name_
The topic name.
Definition fpl.hpp:364
std::string info_
Stringification of (some of the) data, for debugging.
Definition fpl.hpp:363
std::vector< uint8_t > msg_data_
Serialised ROS message data.
Definition fpl.hpp:366
bool valid_
Data valid, successfully extracted from message.
Definition fpl.hpp:362
RosMsgBin(const FplMessage &log_msg)
Constructor.
time::RosTime rec_time_
Recording timestamp.
Definition fpl.hpp:365
RosMsgDef(const FplMessage &log_msg)
Constructor.
std::string info_
Stringification of (some of the) data, for debugging.
Definition fpl.hpp:342
std::string msg_name_
The message name (a.k.a. data type)
Definition fpl.hpp:344
bool valid_
Data valid, successfully extracted from message.
Definition fpl.hpp:341
std::string msg_md5_
The message definition MD5 sum.
Definition fpl.hpp:345
std::string topic_name_
The topic name.
Definition fpl.hpp:343
std::string msg_def_
The message definition (the .msg file and also the defs for all the types)
Definition fpl.hpp:346
bool valid_
Data valid, successfully extracted from message.
Definition fpl.hpp:382
std::string stream_name_
Stream name.
Definition fpl.hpp:385
std::string info_
Stringification of (some of the) data, for debugging.
Definition fpl.hpp:383
StreamMsg(const FplMessage &log_msg)
Constructor.
std::vector< uint8_t > msg_data_
Message data.
Definition fpl.hpp:386
time::RosTime rec_time_
Recording timestamp.
Definition fpl.hpp:384
Minimal ros::Time() / rplcpp::Time implementation (that doesn't throw)
Definition time.hpp:156
Fixposition SDK: Time utilities.