001// Copyright (c) FIRST and other WPILib contributors.
002// Open Source Software; you can modify and/or share it under the terms of
003// the WPILib BSD license file in the root directory of this project.
004
005package edu.wpi.first.wpilibj;
006
007import edu.wpi.first.datalog.DataLog;
008import edu.wpi.first.datalog.DataLogBackgroundWriter;
009import edu.wpi.first.datalog.FileLogger;
010import edu.wpi.first.datalog.IntegerLogEntry;
011import edu.wpi.first.datalog.StringLogEntry;
012import edu.wpi.first.hal.HAL;
013import edu.wpi.first.networktables.NetworkTableInstance;
014import edu.wpi.first.util.WPIUtilJNI;
015import edu.wpi.first.util.concurrent.Event;
016import java.io.File;
017import java.io.IOException;
018import java.nio.file.Files;
019import java.nio.file.Path;
020import java.nio.file.Paths;
021import java.time.LocalDateTime;
022import java.time.ZoneId;
023import java.time.format.DateTimeFormatter;
024import java.util.Arrays;
025import java.util.Comparator;
026import java.util.Random;
027
028/**
029 * Centralized data log that provides automatic data log file management. It automatically cleans up
030 * old files when disk space is low and renames the file based either on current date/time or (if
031 * available) competition match number. The data file will be saved to a USB flash drive in a folder
032 * named "logs" if one is attached, or to /home/systemcore/logs otherwise.
033 *
034 * <p>Log files are initially named "FRC_TBD_{random}.wpilog" until the DS connects. After the DS
035 * connects, the log file is renamed to "FRC_yyyyMMdd_HHmmss.wpilog" (where the date/time is UTC).
036 * If the FMS is connected and provides a match number, the log file is renamed to
037 * "FRC_yyyyMMdd_HHmmss_{event}_{match}.wpilog".
038 *
039 * <p>On startup, all existing FRC_TBD log files are deleted. If there is less than 50 MB of free
040 * space on the target storage, FRC_ log files are deleted (oldest to newest) until there is 50 MB
041 * free OR there are 10 files remaining.
042 *
043 * <p>By default, all NetworkTables value changes are stored to the data log.
044 */
045public final class DataLogManager {
046  private static DataLogBackgroundWriter m_log;
047  private static boolean m_stopped;
048  private static String m_logDir;
049  private static boolean m_filenameOverride;
050  private static Thread m_thread;
051  private static final ZoneId m_utc = ZoneId.of("UTC");
052  private static final DateTimeFormatter m_timeFormatter =
053      DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss").withZone(m_utc);
054  private static boolean m_ntLoggerEnabled = true;
055  private static int m_ntEntryLogger;
056  private static int m_ntConnLogger;
057  private static boolean m_consoleLoggerEnabled = true;
058  private static FileLogger m_consoleLogger;
059  private static StringLogEntry m_messageLog;
060
061  // if less than this much free space, delete log files until there is this much free space
062  // OR there are this many files remaining.
063  private static final long kFreeSpaceThreshold = 50000000L;
064  private static final int kFileCountThreshold = 10;
065
066  private DataLogManager() {}
067
068  /** Start data log manager with default directory location. */
069  public static synchronized void start() {
070    start("", "", 0.25);
071  }
072
073  /**
074   * Start data log manager. The parameters have no effect if the data log manager was already
075   * started (e.g. by calling another static function).
076   *
077   * @param dir if not empty, directory to use for data log storage
078   */
079  public static synchronized void start(String dir) {
080    start(dir, "", 0.25);
081  }
082
083  /**
084   * Start data log manager. The parameters have no effect if the data log manager was already
085   * started (e.g. by calling another static function).
086   *
087   * @param dir if not empty, directory to use for data log storage
088   * @param filename filename to use; if none provided, the filename is automatically generated
089   */
090  public static synchronized void start(String dir, String filename) {
091    start(dir, filename, 0.25);
092  }
093
094  /**
095   * Start data log manager. The parameters have no effect if the data log manager was already
096   * started (e.g. by calling another static function).
097   *
098   * @param dir if not empty, directory to use for data log storage
099   * @param filename filename to use; if none provided, the filename is automatically generated
100   * @param period time between automatic flushes to disk, in seconds; this is a time/storage
101   *     tradeoff
102   */
103  public static synchronized void start(String dir, String filename, double period) {
104    if (m_log == null) {
105      m_logDir = makeLogDir(dir);
106      m_filenameOverride = !filename.isEmpty();
107
108      // Delete all previously existing FRC_TBD_*.wpilog files. These only exist when the robot
109      // never connects to the DS, so they are very unlikely to have useful data and just clutter
110      // the filesystem.
111      File[] files =
112          new File(m_logDir)
113              .listFiles((d, name) -> name.startsWith("FRC_TBD_") && name.endsWith(".wpilog"));
114      if (files != null) {
115        for (File file : files) {
116          if (!file.delete()) {
117            System.err.println("DataLogManager: could not delete " + file);
118          }
119        }
120      }
121      m_log = new DataLogBackgroundWriter(m_logDir, makeLogFilename(filename), period);
122      m_messageLog = new StringLogEntry(m_log, "messages");
123
124      // Log all NT entries and connections
125      if (m_ntLoggerEnabled) {
126        startNtLog();
127      }
128      // Log console output
129      if (m_consoleLoggerEnabled) {
130        startConsoleLog();
131      }
132    } else if (m_stopped) {
133      m_log.setFilename(makeLogFilename(filename));
134      m_log.resume();
135      m_stopped = false;
136    }
137
138    if (m_thread == null) {
139      m_thread = new Thread(DataLogManager::logMain, "DataLogDS");
140      m_thread.setDaemon(true);
141      m_thread.start();
142    }
143  }
144
145  /** Stop data log manager. */
146  public static synchronized void stop() {
147    if (m_thread != null) {
148      m_thread.interrupt();
149      m_thread = null;
150    }
151    if (m_log != null) {
152      m_log.stop();
153      m_stopped = true;
154    }
155  }
156
157  /**
158   * Log a message to the "messages" entry. The message is also printed to standard output (followed
159   * by a newline).
160   *
161   * @param message message
162   */
163  public static synchronized void log(String message) {
164    if (m_messageLog == null) {
165      start();
166    }
167    m_messageLog.append(message);
168    System.out.println(message);
169  }
170
171  /**
172   * Get the managed data log (for custom logging). Starts the data log manager if not already
173   * started.
174   *
175   * @return data log
176   */
177  public static synchronized DataLog getLog() {
178    if (m_log == null) {
179      start();
180    }
181    return m_log;
182  }
183
184  /**
185   * Get the log directory.
186   *
187   * @return log directory, or empty string if logging not yet started
188   */
189  public static synchronized String getLogDir() {
190    if (m_logDir == null) {
191      return "";
192    }
193    return m_logDir;
194  }
195
196  /**
197   * Enable or disable logging of NetworkTables data. Note that unlike the network interface for
198   * NetworkTables, this will capture every value change. Defaults to enabled.
199   *
200   * @param enabled true to enable, false to disable
201   */
202  public static synchronized void logNetworkTables(boolean enabled) {
203    boolean wasEnabled = m_ntLoggerEnabled;
204    m_ntLoggerEnabled = enabled;
205    if (m_log == null) {
206      start();
207      return;
208    }
209    if (enabled && !wasEnabled) {
210      startNtLog();
211    } else if (!enabled && wasEnabled) {
212      stopNtLog();
213    }
214  }
215
216  /**
217   * Enable or disable logging of the console output. Defaults to enabled.
218   *
219   * @param enabled true to enable, false to disable
220   */
221  public static synchronized void logConsoleOutput(boolean enabled) {
222    boolean wasEnabled = m_consoleLoggerEnabled;
223    m_consoleLoggerEnabled = enabled;
224    if (m_log == null) {
225      start();
226      return;
227    }
228    if (enabled && !wasEnabled) {
229      startConsoleLog();
230    } else if (!enabled && wasEnabled) {
231      stopConsoleLog();
232    }
233  }
234
235  private static String makeLogDir(String dir) {
236    if (!dir.isEmpty()) {
237      return dir;
238    }
239
240    if (RobotBase.isReal()) {
241      try {
242        // prefer a mounted USB drive if one is accessible
243        Path usbDir = Paths.get("/u").toRealPath();
244        if (Files.isWritable(usbDir)) {
245          if (!new File("/u/logs").mkdir()) {
246            // ignored
247          }
248          HAL.reportUsage("DataLogManager", "USB");
249          return "/u/logs";
250        }
251      } catch (IOException ex) {
252        // ignored
253      }
254      if (!new File("/home/systemcore/logs").mkdir()) {
255        // ignored
256      }
257      HAL.reportUsage("DataLogManager", "Onboard");
258      return "/home/systemcore/logs";
259    }
260    String logDir = Filesystem.getOperatingDirectory().getAbsolutePath() + "/logs";
261    if (!new File(logDir).mkdir()) {
262      // ignored
263    }
264    return logDir;
265  }
266
267  private static String makeLogFilename(String filenameOverride) {
268    if (!filenameOverride.isEmpty()) {
269      return filenameOverride;
270    }
271    Random rnd = new Random();
272    StringBuilder filename = new StringBuilder();
273    filename.append("FRC_TBD_");
274    for (int i = 0; i < 4; i++) {
275      filename.append(String.format("%04x", rnd.nextInt(0x10000)));
276    }
277    filename.append(".wpilog");
278    return filename.toString();
279  }
280
281  private static void startNtLog() {
282    NetworkTableInstance inst = NetworkTableInstance.getDefault();
283    m_ntEntryLogger = inst.startEntryDataLog(m_log, "", "NT:");
284    m_ntConnLogger = inst.startConnectionDataLog(m_log, "NTConnection");
285  }
286
287  private static void stopNtLog() {
288    NetworkTableInstance.stopEntryDataLog(m_ntEntryLogger);
289    NetworkTableInstance.stopConnectionDataLog(m_ntConnLogger);
290  }
291
292  private static void startConsoleLog() {
293    if (RobotBase.isReal()) {
294      m_consoleLogger = new FileLogger("/home/systemcore/FRC_UserProgram.log", m_log, "console");
295    }
296  }
297
298  private static void stopConsoleLog() {
299    if (RobotBase.isReal()) {
300      m_consoleLogger.close();
301    }
302  }
303
304  private static void logMain() {
305    // based on free disk space, scan for "old" FRC_*.wpilog files and remove
306    {
307      File logDir = new File(m_logDir);
308      long freeSpace = logDir.getUsableSpace();
309      if (freeSpace < kFreeSpaceThreshold) {
310        // Delete oldest FRC_*.wpilog files (ignore FRC_TBD_*.wpilog as we just created one)
311        File[] files =
312            logDir.listFiles(
313                (dir, name) ->
314                    name.startsWith("FRC_")
315                        && name.endsWith(".wpilog")
316                        && !name.startsWith("FRC_TBD_"));
317        if (files != null) {
318          Arrays.sort(files, Comparator.comparingLong(File::lastModified));
319          int count = files.length;
320          for (File file : files) {
321            --count;
322            if (count < kFileCountThreshold) {
323              break;
324            }
325            long length = file.length();
326            if (file.delete()) {
327              DriverStation.reportWarning("DataLogManager: Deleted " + file.getName(), false);
328              freeSpace += length;
329              if (freeSpace >= kFreeSpaceThreshold) {
330                break;
331              }
332            } else {
333              System.err.println("DataLogManager: could not delete " + file);
334            }
335          }
336        }
337      } else if (freeSpace < 2 * kFreeSpaceThreshold) {
338        DriverStation.reportWarning(
339            "DataLogManager: Log storage device has "
340                + freeSpace / 1000000
341                + " MB of free space remaining! Logs will get deleted below "
342                + kFreeSpaceThreshold / 1000000
343                + " MB of free space. "
344                + "Consider deleting logs off the storage device.",
345            false);
346      }
347    }
348
349    int timeoutCount = 0;
350    boolean paused = false;
351    int dsAttachCount = 0;
352    int fmsAttachCount = 0;
353    boolean dsRenamed = m_filenameOverride;
354    boolean fmsRenamed = m_filenameOverride;
355    int sysTimeCount = 0;
356    IntegerLogEntry sysTimeEntry =
357        new IntegerLogEntry(
358            m_log, "systemTime", "{\"source\":\"DataLogManager\",\"format\":\"time_t_us\"}");
359
360    Event newDataEvent = new Event();
361    DriverStation.provideRefreshedDataEventHandle(newDataEvent.getHandle());
362    while (!Thread.interrupted()) {
363      boolean timedOut;
364      try {
365        timedOut = WPIUtilJNI.waitForObjectTimeout(newDataEvent.getHandle(), 0.25);
366      } catch (InterruptedException e) {
367        break;
368      }
369      if (Thread.interrupted()) {
370        break;
371      }
372      if (timedOut) {
373        timeoutCount++;
374        // pause logging after being disconnected for 10 seconds
375        if (timeoutCount > 40 && !paused) {
376          timeoutCount = 0;
377          paused = true;
378          m_log.pause();
379        }
380        continue;
381      }
382      // when we connect to the DS, resume logging
383      timeoutCount = 0;
384      if (paused) {
385        paused = false;
386        m_log.resume();
387      }
388
389      if (!dsRenamed) {
390        // track DS attach
391        if (DriverStation.isDSAttached()) {
392          dsAttachCount++;
393        } else {
394          dsAttachCount = 0;
395        }
396        if (dsAttachCount > 50) { // 1 second
397          if (RobotController.isSystemTimeValid()) {
398            LocalDateTime now = LocalDateTime.now(m_utc);
399            m_log.setFilename("FRC_" + m_timeFormatter.format(now) + ".wpilog");
400            dsRenamed = true;
401          } else {
402            dsAttachCount = 0; // wait a bit and try again
403          }
404        }
405      }
406
407      if (!fmsRenamed) {
408        // track FMS attach
409        if (DriverStation.isFMSAttached()) {
410          fmsAttachCount++;
411        } else {
412          fmsAttachCount = 0;
413        }
414        if (fmsAttachCount > 250) { // 5 seconds
415          // match info comes through TCP, so we need to double-check we've
416          // actually received it
417          DriverStation.MatchType matchType = DriverStation.getMatchType();
418          if (matchType != DriverStation.MatchType.None) {
419            // rename per match info
420            char matchTypeChar =
421                switch (matchType) {
422                  case Practice -> 'P';
423                  case Qualification -> 'Q';
424                  case Elimination -> 'E';
425                  default -> '_';
426                };
427            m_log.setFilename(
428                "FRC_"
429                    + m_timeFormatter.format(LocalDateTime.now(m_utc))
430                    + "_"
431                    + DriverStation.getEventName()
432                    + "_"
433                    + matchTypeChar
434                    + DriverStation.getMatchNumber()
435                    + ".wpilog");
436            fmsRenamed = true;
437            dsRenamed = true; // don't override FMS rename
438          }
439        }
440      }
441
442      // Write system time every ~5 seconds
443      sysTimeCount++;
444      if (sysTimeCount >= 250) {
445        sysTimeCount = 0;
446        if (RobotController.isSystemTimeValid()) {
447          sysTimeEntry.append(WPIUtilJNI.getSystemTime(), WPIUtilJNI.now());
448        }
449      }
450    }
451    newDataEvent.close();
452  }
453}