Source code

001// Copyright (c) FIRST and other WPILib contributors.
002// Open Source Software; you can modify and/or share it under the terms of
003// the WPILib BSD license file in the root directory of this project.
004
005package edu.wpi.first.wpilibj;
006
007import edu.wpi.first.networktables.NetworkTableInstance;
008import edu.wpi.first.util.WPIUtilJNI;
009import edu.wpi.first.util.concurrent.Event;
010import edu.wpi.first.util.datalog.DataLog;
011import edu.wpi.first.util.datalog.IntegerLogEntry;
012import edu.wpi.first.util.datalog.StringLogEntry;
013import java.io.File;
014import java.io.IOException;
015import java.nio.file.Files;
016import java.nio.file.Path;
017import java.nio.file.Paths;
018import java.time.LocalDateTime;
019import java.time.ZoneId;
020import java.time.format.DateTimeFormatter;
021import java.util.Arrays;
022import java.util.Comparator;
023import java.util.Random;
024
025/**
026 * Centralized data log that provides automatic data log file management. It automatically cleans up
027 * old files when disk space is low and renames the file based either on current date/time or (if
028 * available) competition match number. The data file will be saved to a USB flash drive in a folder
029 * named "logs" if one is attached, or to /home/lvuser/logs otherwise.
030 *
031 * <p>Log files are initially named "FRC_TBD_{random}.wpilog" until the DS connects. After the DS
032 * connects, the log file is renamed to "FRC_yyyyMMdd_HHmmss.wpilog" (where the date/time is UTC).
033 * If the FMS is connected and provides a match number, the log file is renamed to
034 * "FRC_yyyyMMdd_HHmmss_{event}_{match}.wpilog".
035 *
036 * <p>On startup, all existing FRC_TBD log files are deleted. If there is less than 50 MB of free
037 * space on the target storage, FRC_ log files are deleted (oldest to newest) until there is 50 MB
038 * free OR there are 10 files remaining.
039 *
040 * <p>By default, all NetworkTables value changes are stored to the data log.
041 */
042public final class DataLogManager {
043  private static DataLog m_log;
044  private static boolean m_stopped;
045  private static String m_logDir;
046  private static boolean m_filenameOverride;
047  private static Thread m_thread;
048  private static final ZoneId m_utc = ZoneId.of("UTC");
049  private static final DateTimeFormatter m_timeFormatter =
050      DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss").withZone(m_utc);
051  private static boolean m_ntLoggerEnabled = true;
052  private static int m_ntEntryLogger;
053  private static int m_ntConnLogger;
054  private static StringLogEntry m_messageLog;
055
056  // if less than this much free space, delete log files until there is this much free space
057  // OR there are this many files remaining.
058  private static final long kFreeSpaceThreshold = 50000000L;
059  private static final int kFileCountThreshold = 10;
060
061  private DataLogManager() {}
062
063  /** Start data log manager with default directory location. */
064  public static synchronized void start() {
065    start("", "", 0.25);
066  }
067
068  /**
069   * Start data log manager. The parameters have no effect if the data log manager was already
070   * started (e.g. by calling another static function).
071   *
072   * @param dir if not empty, directory to use for data log storage
073   */
074  public static synchronized void start(String dir) {
075    start(dir, "", 0.25);
076  }
077
078  /**
079   * Start data log manager. The parameters have no effect if the data log manager was already
080   * started (e.g. by calling another static function).
081   *
082   * @param dir if not empty, directory to use for data log storage
083   * @param filename filename to use; if none provided, the filename is automatically generated
084   */
085  public static synchronized void start(String dir, String filename) {
086    start(dir, filename, 0.25);
087  }
088
089  /**
090   * Start data log manager. The parameters have no effect if the data log manager was already
091   * started (e.g. by calling another static function).
092   *
093   * @param dir if not empty, directory to use for data log storage
094   * @param filename filename to use; if none provided, the filename is automatically generated
095   * @param period time between automatic flushes to disk, in seconds; this is a time/storage
096   *     tradeoff
097   */
098  public static synchronized void start(String dir, String filename, double period) {
099    if (m_log == null) {
100      m_logDir = makeLogDir(dir);
101      m_filenameOverride = !filename.isEmpty();
102
103      // Delete all previously existing FRC_TBD_*.wpilog files. These only exist when the robot
104      // never connects to the DS, so they are very unlikely to have useful data and just clutter
105      // the filesystem.
106      File[] files =
107          new File(m_logDir)
108              .listFiles((d, name) -> name.startsWith("FRC_TBD_") && name.endsWith(".wpilog"));
109      if (files != null) {
110        for (File file : files) {
111          if (!file.delete()) {
112            System.err.println("DataLogManager: could not delete " + file);
113          }
114        }
115      }
116      m_log = new DataLog(m_logDir, makeLogFilename(filename), period);
117      m_messageLog = new StringLogEntry(m_log, "messages");
118
119      // Log all NT entries and connections
120      if (m_ntLoggerEnabled) {
121        startNtLog();
122      }
123    } else if (m_stopped) {
124      m_log.setFilename(makeLogFilename(filename));
125      m_log.resume();
126      m_stopped = false;
127    }
128
129    if (m_thread == null) {
130      m_thread = new Thread(DataLogManager::logMain, "DataLogDS");
131      m_thread.setDaemon(true);
132      m_thread.start();
133    }
134  }
135
136  /** Stop data log manager. */
137  public static synchronized void stop() {
138    if (m_thread != null) {
139      m_thread.interrupt();
140      m_thread = null;
141    }
142    if (m_log != null) {
143      m_log.stop();
144      m_stopped = true;
145    }
146  }
147
148  /**
149   * Log a message to the "messages" entry. The message is also printed to standard output (followed
150   * by a newline).
151   *
152   * @param message message
153   */
154  public static synchronized void log(String message) {
155    if (m_messageLog == null) {
156      start();
157    }
158    m_messageLog.append(message);
159    System.out.println(message);
160  }
161
162  /**
163   * Get the managed data log (for custom logging). Starts the data log manager if not already
164   * started.
165   *
166   * @return data log
167   */
168  public static synchronized DataLog getLog() {
169    if (m_log == null) {
170      start();
171    }
172    return m_log;
173  }
174
175  /**
176   * Get the log directory.
177   *
178   * @return log directory, or empty string if logging not yet started
179   */
180  public static synchronized String getLogDir() {
181    if (m_logDir == null) {
182      return "";
183    }
184    return m_logDir;
185  }
186
187  /**
188   * Enable or disable logging of NetworkTables data. Note that unlike the network interface for
189   * NetworkTables, this will capture every value change. Defaults to enabled.
190   *
191   * @param enabled true to enable, false to disable
192   */
193  public static synchronized void logNetworkTables(boolean enabled) {
194    boolean wasEnabled = m_ntLoggerEnabled;
195    m_ntLoggerEnabled = enabled;
196    if (m_log == null) {
197      start();
198      return;
199    }
200    if (enabled && !wasEnabled) {
201      startNtLog();
202    } else if (!enabled && wasEnabled) {
203      stopNtLog();
204    }
205  }
206
207  private static String makeLogDir(String dir) {
208    if (!dir.isEmpty()) {
209      return dir;
210    }
211
212    if (RobotBase.isReal()) {
213      try {
214        // prefer a mounted USB drive if one is accessible
215        Path usbDir = Paths.get("/u").toRealPath();
216        if (Files.isWritable(usbDir)) {
217          if (!new File("/u/logs").mkdir()) {
218            // ignored
219          }
220          return "/u/logs";
221        }
222      } catch (IOException ex) {
223        // ignored
224      }
225      if (RobotBase.getRuntimeType() == RuntimeType.kRoboRIO) {
226        DriverStation.reportWarning(
227            "DataLogManager: Logging to RoboRIO 1 internal storage is not recommended!"
228                + " Plug in a FAT32 formatted flash drive!",
229            false);
230      }
231      if (!new File("/home/lvuser/logs").mkdir()) {
232        // ignored
233      }
234      return "/home/lvuser/logs";
235    }
236    String logDir = Filesystem.getOperatingDirectory().getAbsolutePath() + "/logs";
237    if (!new File(logDir).mkdir()) {
238      // ignored
239    }
240    return logDir;
241  }
242
243  private static String makeLogFilename(String filenameOverride) {
244    if (!filenameOverride.isEmpty()) {
245      return filenameOverride;
246    }
247    Random rnd = new Random();
248    StringBuilder filename = new StringBuilder();
249    filename.append("FRC_TBD_");
250    for (int i = 0; i < 4; i++) {
251      filename.append(String.format("%04x", rnd.nextInt(0x10000)));
252    }
253    filename.append(".wpilog");
254    return filename.toString();
255  }
256
257  private static void startNtLog() {
258    NetworkTableInstance inst = NetworkTableInstance.getDefault();
259    m_ntEntryLogger = inst.startEntryDataLog(m_log, "", "NT:");
260    m_ntConnLogger = inst.startConnectionDataLog(m_log, "NTConnection");
261  }
262
263  private static void stopNtLog() {
264    NetworkTableInstance.stopEntryDataLog(m_ntEntryLogger);
265    NetworkTableInstance.stopConnectionDataLog(m_ntConnLogger);
266  }
267
268  private static void logMain() {
269    // based on free disk space, scan for "old" FRC_*.wpilog files and remove
270    {
271      File logDir = new File(m_logDir);
272      long freeSpace = logDir.getFreeSpace();
273      if (freeSpace < kFreeSpaceThreshold) {
274        // Delete oldest FRC_*.wpilog files (ignore FRC_TBD_*.wpilog as we just created one)
275        File[] files =
276            logDir.listFiles(
277                (dir, name) ->
278                    name.startsWith("FRC_")
279                        && name.endsWith(".wpilog")
280                        && !name.startsWith("FRC_TBD_"));
281        if (files != null) {
282          Arrays.sort(files, Comparator.comparingLong(File::lastModified));
283          int count = files.length;
284          for (File file : files) {
285            --count;
286            if (count < kFileCountThreshold) {
287              break;
288            }
289            long length = file.length();
290            if (file.delete()) {
291              DriverStation.reportWarning("DataLogManager: Deleted " + file.getName(), false);
292              freeSpace += length;
293              if (freeSpace >= kFreeSpaceThreshold) {
294                break;
295              }
296            } else {
297              System.err.println("DataLogManager: could not delete " + file);
298            }
299          }
300        }
301      } else if (freeSpace < 2 * kFreeSpaceThreshold) {
302        DriverStation.reportWarning(
303            "DataLogManager: Log storage device has "
304                + freeSpace / 1000000
305                + " MB of free space remaining! Logs will get deleted below "
306                + kFreeSpaceThreshold / 1000000
307                + " MB of free space."
308                + "Consider deleting logs off the storage device.",
309            false);
310      }
311    }
312
313    int timeoutCount = 0;
314    boolean paused = false;
315    int dsAttachCount = 0;
316    int fmsAttachCount = 0;
317    boolean dsRenamed = m_filenameOverride;
318    boolean fmsRenamed = m_filenameOverride;
319    int sysTimeCount = 0;
320    IntegerLogEntry sysTimeEntry =
321        new IntegerLogEntry(
322            m_log, "systemTime", "{\"source\":\"DataLogManager\",\"format\":\"time_t_us\"}");
323
324    Event newDataEvent = new Event();
325    DriverStation.provideRefreshedDataEventHandle(newDataEvent.getHandle());
326    while (!Thread.interrupted()) {
327      boolean timedOut;
328      try {
329        timedOut = WPIUtilJNI.waitForObjectTimeout(newDataEvent.getHandle(), 0.25);
330      } catch (InterruptedException e) {
331        break;
332      }
333      if (Thread.interrupted()) {
334        break;
335      }
336      if (timedOut) {
337        timeoutCount++;
338        // pause logging after being disconnected for 10 seconds
339        if (timeoutCount > 40 && !paused) {
340          timeoutCount = 0;
341          paused = true;
342          m_log.pause();
343        }
344        continue;
345      }
346      // when we connect to the DS, resume logging
347      timeoutCount = 0;
348      if (paused) {
349        paused = false;
350        m_log.resume();
351      }
352
353      if (!dsRenamed) {
354        // track DS attach
355        if (DriverStation.isDSAttached()) {
356          dsAttachCount++;
357        } else {
358          dsAttachCount = 0;
359        }
360        if (dsAttachCount > 50) { // 1 second
361          if (RobotController.isSystemTimeValid()) {
362            LocalDateTime now = LocalDateTime.now(m_utc);
363            m_log.setFilename("FRC_" + m_timeFormatter.format(now) + ".wpilog");
364            dsRenamed = true;
365          } else {
366            dsAttachCount = 0; // wait a bit and try again
367          }
368        }
369      }
370
371      if (!fmsRenamed) {
372        // track FMS attach
373        if (DriverStation.isFMSAttached()) {
374          fmsAttachCount++;
375        } else {
376          fmsAttachCount = 0;
377        }
378        if (fmsAttachCount > 250) { // 5 seconds
379          // match info comes through TCP, so we need to double-check we've
380          // actually received it
381          DriverStation.MatchType matchType = DriverStation.getMatchType();
382          if (matchType != DriverStation.MatchType.None) {
383            // rename per match info
384            char matchTypeChar;
385            switch (matchType) {
386              case Practice:
387                matchTypeChar = 'P';
388                break;
389              case Qualification:
390                matchTypeChar = 'Q';
391                break;
392              case Elimination:
393                matchTypeChar = 'E';
394                break;
395              default:
396                matchTypeChar = '_';
397                break;
398            }
399            m_log.setFilename(
400                "FRC_"
401                    + m_timeFormatter.format(LocalDateTime.now(m_utc))
402                    + "_"
403                    + DriverStation.getEventName()
404                    + "_"
405                    + matchTypeChar
406                    + DriverStation.getMatchNumber()
407                    + ".wpilog");
408            fmsRenamed = true;
409            dsRenamed = true; // don't override FMS rename
410          }
411        }
412      }
413
414      // Write system time every ~5 seconds
415      sysTimeCount++;
416      if (sysTimeCount >= 250) {
417        sysTimeCount = 0;
418        if (RobotController.isSystemTimeValid()) {
419          sysTimeEntry.append(WPIUtilJNI.getSystemTime(), WPIUtilJNI.now());
420        }
421      }
422    }
423    newDataEvent.close();
424  }
425}