Source code

001// Copyright (c) FIRST and other WPILib contributors.
002// Open Source Software; you can modify and/or share it under the terms of
003// the WPILib BSD license file in the root directory of this project.
004
005package edu.wpi.first.util.datalog;
006
007import edu.wpi.first.util.WPIUtilJNI;
008import edu.wpi.first.util.protobuf.Protobuf;
009import edu.wpi.first.util.struct.Struct;
010import java.nio.ByteBuffer;
011import java.util.HashSet;
012import java.util.Set;
013import java.util.concurrent.ConcurrentHashMap;
014import java.util.concurrent.ConcurrentMap;
015
016/**
017 * A data log. The log file is created immediately upon construction with a temporary filename. The
018 * file may be renamed at any time using the setFilename() function.
019 *
020 * <p>The data log is periodically flushed to disk. It can also be explicitly flushed to disk by
021 * using the flush() function.
022 *
023 * <p>The finish() function is needed only to indicate in the log that a particular entry is no
024 * longer being used (it releases the name to ID mapping). The finish() function is not required to
025 * be called for data to be flushed to disk; entries in the log are written as append() calls are
026 * being made. In fact, finish() does not need to be called at all.
027 *
028 * <p>DataLog calls are thread safe. DataLog uses a typical multiple-supplier, single-consumer
029 * setup. Writes to the log are atomic, but there is no guaranteed order in the log when multiple
030 * threads are writing to it; whichever thread grabs the write mutex first will get written first.
031 * For this reason (as well as the fact that timestamps can be set to arbitrary values), records in
032 * the log are not guaranteed to be sorted by timestamp.
033 */
034public final class DataLog implements AutoCloseable {
035  /**
036   * Construct a new Data Log. The log will be initially created with a temporary filename.
037   *
038   * @param dir directory to store the log
039   * @param filename filename to use; if none provided, a random filename is generated of the form
040   *     "wpilog_{}.wpilog"
041   * @param period time between automatic flushes to disk, in seconds; this is a time/storage
042   *     tradeoff
043   * @param extraHeader extra header data
044   */
045  public DataLog(String dir, String filename, double period, String extraHeader) {
046    m_impl = DataLogJNI.create(dir, filename, period, extraHeader);
047  }
048
049  /**
050   * Construct a new Data Log. The log will be initially created with a temporary filename.
051   *
052   * @param dir directory to store the log
053   * @param filename filename to use; if none provided, a random filename is generated of the form
054   *     "wpilog_{}.wpilog"
055   * @param period time between automatic flushes to disk, in seconds; this is a time/storage
056   *     tradeoff
057   */
058  public DataLog(String dir, String filename, double period) {
059    this(dir, filename, period, "");
060  }
061
062  /**
063   * Construct a new Data Log. The log will be initially created with a temporary filename.
064   *
065   * @param dir directory to store the log
066   * @param filename filename to use; if none provided, a random filename is generated of the form
067   *     "wpilog_{}.wpilog"
068   */
069  public DataLog(String dir, String filename) {
070    this(dir, filename, 0.25);
071  }
072
073  /**
074   * Construct a new Data Log. The log will be initially created with a temporary filename.
075   *
076   * @param dir directory to store the log
077   */
078  public DataLog(String dir) {
079    this(dir, "", 0.25);
080  }
081
082  /** Construct a new Data Log. The log will be initially created with a temporary filename. */
083  public DataLog() {
084    this("");
085  }
086
087  /**
088   * Change log filename.
089   *
090   * @param filename filename
091   */
092  public void setFilename(String filename) {
093    DataLogJNI.setFilename(m_impl, filename);
094  }
095
096  /** Explicitly flushes the log data to disk. */
097  public void flush() {
098    DataLogJNI.flush(m_impl);
099  }
100
101  /**
102   * Pauses appending of data records to the log. While paused, no data records are saved (e.g.
103   * AppendX is a no-op). Has no effect on entry starts / finishes / metadata changes.
104   */
105  public void pause() {
106    DataLogJNI.pause(m_impl);
107  }
108
109  /**
110   * Resumes appending of data records to the log. If called after stop(), opens a new file (with
111   * random name if SetFilename was not called after stop()) and appends Start records and schema
112   * data values for all previously started entries and schemas.
113   */
114  public void resume() {
115    DataLogJNI.resume(m_impl);
116  }
117
118  /** Stops appending all records to the log, and closes the log file. */
119  public void stop() {
120    DataLogJNI.stop(m_impl);
121  }
122
123  /**
124   * Returns whether there is a data schema already registered with the given name.
125   *
126   * @param name Name (the string passed as the data type for records using this schema)
127   * @return True if schema already registered
128   */
129  public boolean hasSchema(String name) {
130    return m_schemaMap.containsKey(name);
131  }
132
133  /**
134   * Registers a data schema. Data schemas provide information for how a certain data type string
135   * can be decoded. The type string of a data schema indicates the type of the schema itself (e.g.
136   * "protobuf" for protobuf schemas, "struct" for struct schemas, etc). In the data log, schemas
137   * are saved just like normal records, with the name being generated from the provided name:
138   * "/.schema/name". Duplicate calls to this function with the same name are silently ignored.
139   *
140   * @param name Name (the string passed as the data type for records using this schema)
141   * @param type Type of schema (e.g. "protobuf", "struct", etc)
142   * @param schema Schema data
143   * @param timestamp Time stamp (may be 0 to indicate now)
144   */
145  public void addSchema(String name, String type, byte[] schema, long timestamp) {
146    if (m_schemaMap.putIfAbsent(name, 1) != null) {
147      return;
148    }
149    DataLogJNI.addSchema(m_impl, name, type, schema, timestamp);
150  }
151
152  /**
153   * Registers a data schema. Data schemas provide information for how a certain data type string
154   * can be decoded. The type string of a data schema indicates the type of the schema itself (e.g.
155   * "protobuf" for protobuf schemas, "struct" for struct schemas, etc). In the data log, schemas
156   * are saved just like normal records, with the name being generated from the provided name:
157   * "/.schema/name". Duplicate calls to this function with the same name are silently ignored.
158   *
159   * @param name Name (the string passed as the data type for records using this schema)
160   * @param type Type of schema (e.g. "protobuf", "struct", etc)
161   * @param schema Schema data
162   */
163  public void addSchema(String name, String type, byte[] schema) {
164    addSchema(name, type, schema, 0);
165  }
166
167  /**
168   * Registers a data schema. Data schemas provide information for how a certain data type string
169   * can be decoded. The type string of a data schema indicates the type of the schema itself (e.g.
170   * "protobuf" for protobuf schemas, "struct" for struct schemas, etc). In the data log, schemas
171   * are saved just like normal records, with the name being generated from the provided name:
172   * "/.schema/name". Duplicate calls to this function with the same name are silently ignored.
173   *
174   * @param name Name (the string passed as the data type for records using this schema)
175   * @param type Type of schema (e.g. "protobuf", "struct", etc)
176   * @param schema Schema data
177   * @param timestamp Time stamp (may be 0 to indicate now)
178   */
179  public void addSchema(String name, String type, String schema, long timestamp) {
180    if (m_schemaMap.putIfAbsent(name, 1) != null) {
181      return;
182    }
183    DataLogJNI.addSchemaString(m_impl, name, type, schema, timestamp);
184  }
185
186  /**
187   * Registers a data schema. Data schemas provide information for how a certain data type string
188   * can be decoded. The type string of a data schema indicates the type of the schema itself (e.g.
189   * "protobuf" for protobuf schemas, "struct" for struct schemas, etc). In the data log, schemas
190   * are saved just like normal records, with the name being generated from the provided name:
191   * "/.schema/name". Duplicate calls to this function with the same name are silently ignored.
192   *
193   * @param name Name (the string passed as the data type for records using this schema)
194   * @param type Type of schema (e.g. "protobuf", "struct", etc)
195   * @param schema Schema data
196   */
197  public void addSchema(String name, String type, String schema) {
198    addSchema(name, type, schema, 0);
199  }
200
201  /**
202   * Registers a protobuf schema. Duplicate calls to this function with the same name are silently
203   * ignored.
204   *
205   * @param proto protobuf serialization object
206   * @param timestamp Time stamp (0 to indicate now)
207   */
208  public void addSchema(Protobuf<?, ?> proto, long timestamp) {
209    final long actualTimestamp = timestamp == 0 ? WPIUtilJNI.now() : timestamp;
210    proto.forEachDescriptor(
211        this::hasSchema,
212        (typeString, schema) ->
213            addSchema(typeString, "proto:FileDescriptorProto", schema, actualTimestamp));
214  }
215
216  /**
217   * Registers a protobuf schema. Duplicate calls to this function with the same name are silently
218   * ignored.
219   *
220   * @param proto protobuf serialization object
221   */
222  public void addSchema(Protobuf<?, ?> proto) {
223    addSchema(proto, 0);
224  }
225
226  /**
227   * Registers a struct schema. Duplicate calls to this function with the same name are silently
228   * ignored.
229   *
230   * @param struct struct serialization object
231   * @param timestamp Time stamp (0 to indicate now)
232   */
233  public void addSchema(Struct<?> struct, long timestamp) {
234    addSchemaImpl(struct, timestamp == 0 ? WPIUtilJNI.now() : timestamp, new HashSet<>());
235  }
236
237  /**
238   * Registers a struct schema. Duplicate calls to this function with the same name are silently
239   * ignored.
240   *
241   * @param struct struct serialization object
242   */
243  public void addSchema(Struct<?> struct) {
244    addSchema(struct, 0);
245  }
246
247  /**
248   * Start an entry. Duplicate names are allowed (with the same type), and result in the same index
249   * being returned (start/finish are reference counted). A duplicate name with a different type
250   * will result in an error message being printed to the console and 0 being returned (which will
251   * be ignored by the append functions).
252   *
253   * @param name Name
254   * @param type Data type
255   * @param metadata Initial metadata (e.g. data properties)
256   * @param timestamp Time stamp (0 to indicate now)
257   * @return Entry index
258   */
259  public int start(String name, String type, String metadata, long timestamp) {
260    return DataLogJNI.start(m_impl, name, type, metadata, timestamp);
261  }
262
263  /**
264   * Start an entry. Duplicate names are allowed (with the same type), and result in the same index
265   * being returned (start/finish are reference counted). A duplicate name with a different type
266   * will result in an error message being printed to the console and 0 being returned (which will
267   * be ignored by the append functions).
268   *
269   * @param name Name
270   * @param type Data type
271   * @param metadata Initial metadata (e.g. data properties)
272   * @return Entry index
273   */
274  public int start(String name, String type, String metadata) {
275    return start(name, type, metadata, 0);
276  }
277
278  /**
279   * Start an entry. Duplicate names are allowed (with the same type), and result in the same index
280   * being returned (start/finish are reference counted). A duplicate name with a different type
281   * will result in an error message being printed to the console and 0 being returned (which will
282   * be ignored by the append functions).
283   *
284   * @param name Name
285   * @param type Data type
286   * @return Entry index
287   */
288  public int start(String name, String type) {
289    return start(name, type, "");
290  }
291
292  /**
293   * Finish an entry.
294   *
295   * @param entry Entry index
296   * @param timestamp Time stamp (0 to indicate now)
297   */
298  public void finish(int entry, long timestamp) {
299    DataLogJNI.finish(m_impl, entry, timestamp);
300  }
301
302  /**
303   * Finish an entry.
304   *
305   * @param entry Entry index
306   */
307  public void finish(int entry) {
308    finish(entry, 0);
309  }
310
311  /**
312   * Updates the metadata for an entry.
313   *
314   * @param entry Entry index
315   * @param metadata New metadata for the entry
316   * @param timestamp Time stamp (0 to indicate now)
317   */
318  public void setMetadata(int entry, String metadata, long timestamp) {
319    DataLogJNI.setMetadata(m_impl, entry, metadata, timestamp);
320  }
321
322  /**
323   * Updates the metadata for an entry.
324   *
325   * @param entry Entry index
326   * @param metadata New metadata for the entry
327   */
328  public void setMetadata(int entry, String metadata) {
329    setMetadata(entry, metadata, 0);
330  }
331
332  /**
333   * Appends a raw record to the log.
334   *
335   * @param entry Entry index, as returned by start()
336   * @param data Byte array to record; will send entire array contents
337   * @param timestamp Time stamp (0 to indicate now)
338   */
339  public void appendRaw(int entry, byte[] data, long timestamp) {
340    appendRaw(entry, data, 0, data.length, timestamp);
341  }
342
343  /**
344   * Appends a record to the log.
345   *
346   * @param entry Entry index, as returned by start()
347   * @param data Byte array to record
348   * @param start Start position of data (in byte array)
349   * @param len Length of data (must be less than or equal to data.length - start)
350   * @param timestamp Time stamp (0 to indicate now)
351   */
352  public void appendRaw(int entry, byte[] data, int start, int len, long timestamp) {
353    DataLogJNI.appendRaw(m_impl, entry, data, start, len, timestamp);
354  }
355
356  /**
357   * Appends a record to the log.
358   *
359   * @param entry Entry index, as returned by start()
360   * @param data Buffer to record; will send from data.position() to data.limit()
361   * @param timestamp Time stamp (0 to indicate now)
362   */
363  public void appendRaw(int entry, ByteBuffer data, long timestamp) {
364    int pos = data.position();
365    appendRaw(entry, data, pos, data.limit() - pos, timestamp);
366  }
367
368  /**
369   * Appends a record to the log.
370   *
371   * @param entry Entry index, as returned by start()
372   * @param data Buffer to record
373   * @param start Start position of data (in buffer)
374   * @param len Length of data (must be less than or equal to data.capacity() - start)
375   * @param timestamp Time stamp (0 to indicate now)
376   */
377  public void appendRaw(int entry, ByteBuffer data, int start, int len, long timestamp) {
378    DataLogJNI.appendRaw(m_impl, entry, data, start, len, timestamp);
379  }
380
381  @Override
382  public void close() {
383    DataLogJNI.close(m_impl);
384    m_impl = 0;
385  }
386
387  /**
388   * Appends a boolean record to the log.
389   *
390   * @param entry Entry index, as returned by start()
391   * @param value Boolean value to record
392   * @param timestamp Time stamp (0 to indicate now)
393   */
394  public void appendBoolean(int entry, boolean value, long timestamp) {
395    DataLogJNI.appendBoolean(m_impl, entry, value, timestamp);
396  }
397
398  /**
399   * Appends an integer record to the log.
400   *
401   * @param entry Entry index, as returned by start()
402   * @param value Integer value to record
403   * @param timestamp Time stamp (0 to indicate now)
404   */
405  public void appendInteger(int entry, long value, long timestamp) {
406    DataLogJNI.appendInteger(m_impl, entry, value, timestamp);
407  }
408
409  /**
410   * Appends a float record to the log.
411   *
412   * @param entry Entry index, as returned by start()
413   * @param value Float value to record
414   * @param timestamp Time stamp (0 to indicate now)
415   */
416  public void appendFloat(int entry, float value, long timestamp) {
417    DataLogJNI.appendFloat(m_impl, entry, value, timestamp);
418  }
419
420  /**
421   * Appends a double record to the log.
422   *
423   * @param entry Entry index, as returned by start()
424   * @param value Double value to record
425   * @param timestamp Time stamp (0 to indicate now)
426   */
427  public void appendDouble(int entry, double value, long timestamp) {
428    DataLogJNI.appendDouble(m_impl, entry, value, timestamp);
429  }
430
431  /**
432   * Appends a string record to the log.
433   *
434   * @param entry Entry index, as returned by start()
435   * @param value String value to record
436   * @param timestamp Time stamp (0 to indicate now)
437   */
438  public void appendString(int entry, String value, long timestamp) {
439    DataLogJNI.appendString(m_impl, entry, value, timestamp);
440  }
441
442  /**
443   * Appends a boolean array record to the log.
444   *
445   * @param entry Entry index, as returned by start()
446   * @param arr Boolean array to record
447   * @param timestamp Time stamp (0 to indicate now)
448   */
449  public void appendBooleanArray(int entry, boolean[] arr, long timestamp) {
450    DataLogJNI.appendBooleanArray(m_impl, entry, arr, timestamp);
451  }
452
453  /**
454   * Appends an integer array record to the log.
455   *
456   * @param entry Entry index, as returned by start()
457   * @param arr Integer array to record
458   * @param timestamp Time stamp (0 to indicate now)
459   */
460  public void appendIntegerArray(int entry, long[] arr, long timestamp) {
461    DataLogJNI.appendIntegerArray(m_impl, entry, arr, timestamp);
462  }
463
464  /**
465   * Appends a float array record to the log.
466   *
467   * @param entry Entry index, as returned by start()
468   * @param arr Float array to record
469   * @param timestamp Time stamp (0 to indicate now)
470   */
471  public void appendFloatArray(int entry, float[] arr, long timestamp) {
472    DataLogJNI.appendFloatArray(m_impl, entry, arr, timestamp);
473  }
474
475  /**
476   * Appends a double array record to the log.
477   *
478   * @param entry Entry index, as returned by start()
479   * @param arr Double array to record
480   * @param timestamp Time stamp (0 to indicate now)
481   */
482  public void appendDoubleArray(int entry, double[] arr, long timestamp) {
483    DataLogJNI.appendDoubleArray(m_impl, entry, arr, timestamp);
484  }
485
486  /**
487   * Appends a string array record to the log.
488   *
489   * @param entry Entry index, as returned by start()
490   * @param arr String array to record
491   * @param timestamp Time stamp (0 to indicate now)
492   */
493  public void appendStringArray(int entry, String[] arr, long timestamp) {
494    DataLogJNI.appendStringArray(m_impl, entry, arr, timestamp);
495  }
496
497  public long getImpl() {
498    return m_impl;
499  }
500
501  private void addSchemaImpl(Struct<?> struct, long timestamp, Set<String> seen) {
502    String typeString = struct.getTypeString();
503    if (hasSchema(typeString)) {
504      return;
505    }
506    if (!seen.add(typeString)) {
507      throw new UnsupportedOperationException(typeString + ": circular reference with " + seen);
508    }
509    addSchema(typeString, "structschema", struct.getSchema(), timestamp);
510    for (Struct<?> inner : struct.getNested()) {
511      addSchemaImpl(inner, timestamp, seen);
512    }
513    seen.remove(typeString);
514  }
515
516  private long m_impl;
517  private final ConcurrentMap<String, Integer> m_schemaMap = new ConcurrentHashMap<>();
518}