001// Copyright (c) FIRST and other WPILib contributors. 002// Open Source Software; you can modify and/or share it under the terms of 003// the WPILib BSD license file in the root directory of this project. 004 005package edu.wpi.first.wpilibj; 006 007import edu.wpi.first.datalog.DataLog; 008import edu.wpi.first.datalog.DataLogBackgroundWriter; 009import edu.wpi.first.datalog.FileLogger; 010import edu.wpi.first.datalog.IntegerLogEntry; 011import edu.wpi.first.datalog.StringLogEntry; 012import edu.wpi.first.hal.HAL; 013import edu.wpi.first.networktables.NetworkTableInstance; 014import edu.wpi.first.util.WPIUtilJNI; 015import edu.wpi.first.util.concurrent.Event; 016import java.io.File; 017import java.io.IOException; 018import java.nio.file.Files; 019import java.nio.file.Path; 020import java.nio.file.Paths; 021import java.time.LocalDateTime; 022import java.time.ZoneId; 023import java.time.format.DateTimeFormatter; 024import java.util.Arrays; 025import java.util.Comparator; 026import java.util.Random; 027 028/** 029 * Centralized data log that provides automatic data log file management. It automatically cleans up 030 * old files when disk space is low and renames the file based either on current date/time or (if 031 * available) competition match number. The data file will be saved to a USB flash drive in a folder 032 * named "logs" if one is attached, or to /home/systemcore/logs otherwise. 033 * 034 * <p>Log files are initially named "FRC_TBD_{random}.wpilog" until the DS connects. After the DS 035 * connects, the log file is renamed to "FRC_yyyyMMdd_HHmmss.wpilog" (where the date/time is UTC). 036 * If the FMS is connected and provides a match number, the log file is renamed to 037 * "FRC_yyyyMMdd_HHmmss_{event}_{match}.wpilog". 038 * 039 * <p>On startup, all existing FRC_TBD log files are deleted. If there is less than 50 MB of free 040 * space on the target storage, FRC_ log files are deleted (oldest to newest) until there is 50 MB 041 * free OR there are 10 files remaining. 042 * 043 * <p>By default, all NetworkTables value changes are stored to the data log. 044 */ 045public final class DataLogManager { 046 private static DataLogBackgroundWriter m_log; 047 private static boolean m_stopped; 048 private static String m_logDir; 049 private static boolean m_filenameOverride; 050 private static Thread m_thread; 051 private static final ZoneId m_utc = ZoneId.of("UTC"); 052 private static final DateTimeFormatter m_timeFormatter = 053 DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss").withZone(m_utc); 054 private static boolean m_ntLoggerEnabled = true; 055 private static int m_ntEntryLogger; 056 private static int m_ntConnLogger; 057 private static boolean m_consoleLoggerEnabled = true; 058 private static FileLogger m_consoleLogger; 059 private static StringLogEntry m_messageLog; 060 061 // if less than this much free space, delete log files until there is this much free space 062 // OR there are this many files remaining. 063 private static final long kFreeSpaceThreshold = 50000000L; 064 private static final int kFileCountThreshold = 10; 065 066 private DataLogManager() {} 067 068 /** Start data log manager with default directory location. */ 069 public static synchronized void start() { 070 start("", "", 0.25); 071 } 072 073 /** 074 * Start data log manager. The parameters have no effect if the data log manager was already 075 * started (e.g. by calling another static function). 076 * 077 * @param dir if not empty, directory to use for data log storage 078 */ 079 public static synchronized void start(String dir) { 080 start(dir, "", 0.25); 081 } 082 083 /** 084 * Start data log manager. The parameters have no effect if the data log manager was already 085 * started (e.g. by calling another static function). 086 * 087 * @param dir if not empty, directory to use for data log storage 088 * @param filename filename to use; if none provided, the filename is automatically generated 089 */ 090 public static synchronized void start(String dir, String filename) { 091 start(dir, filename, 0.25); 092 } 093 094 /** 095 * Start data log manager. The parameters have no effect if the data log manager was already 096 * started (e.g. by calling another static function). 097 * 098 * @param dir if not empty, directory to use for data log storage 099 * @param filename filename to use; if none provided, the filename is automatically generated 100 * @param period time between automatic flushes to disk, in seconds; this is a time/storage 101 * tradeoff 102 */ 103 public static synchronized void start(String dir, String filename, double period) { 104 if (m_log == null) { 105 m_logDir = makeLogDir(dir); 106 m_filenameOverride = !filename.isEmpty(); 107 108 // Delete all previously existing FRC_TBD_*.wpilog files. These only exist when the robot 109 // never connects to the DS, so they are very unlikely to have useful data and just clutter 110 // the filesystem. 111 File[] files = 112 new File(m_logDir) 113 .listFiles((d, name) -> name.startsWith("FRC_TBD_") && name.endsWith(".wpilog")); 114 if (files != null) { 115 for (File file : files) { 116 if (!file.delete()) { 117 System.err.println("DataLogManager: could not delete " + file); 118 } 119 } 120 } 121 m_log = new DataLogBackgroundWriter(m_logDir, makeLogFilename(filename), period); 122 m_messageLog = new StringLogEntry(m_log, "messages"); 123 124 // Log all NT entries and connections 125 if (m_ntLoggerEnabled) { 126 startNtLog(); 127 } 128 // Log console output 129 if (m_consoleLoggerEnabled) { 130 startConsoleLog(); 131 } 132 } else if (m_stopped) { 133 m_log.setFilename(makeLogFilename(filename)); 134 m_log.resume(); 135 m_stopped = false; 136 } 137 138 if (m_thread == null) { 139 m_thread = new Thread(DataLogManager::logMain, "DataLogDS"); 140 m_thread.setDaemon(true); 141 m_thread.start(); 142 } 143 } 144 145 /** Stop data log manager. */ 146 public static synchronized void stop() { 147 if (m_thread != null) { 148 m_thread.interrupt(); 149 m_thread = null; 150 } 151 if (m_log != null) { 152 m_log.stop(); 153 m_stopped = true; 154 } 155 } 156 157 /** 158 * Log a message to the "messages" entry. The message is also printed to standard output (followed 159 * by a newline). 160 * 161 * @param message message 162 */ 163 public static synchronized void log(String message) { 164 if (m_messageLog == null) { 165 start(); 166 } 167 m_messageLog.append(message); 168 System.out.println(message); 169 } 170 171 /** 172 * Get the managed data log (for custom logging). Starts the data log manager if not already 173 * started. 174 * 175 * @return data log 176 */ 177 public static synchronized DataLog getLog() { 178 if (m_log == null) { 179 start(); 180 } 181 return m_log; 182 } 183 184 /** 185 * Get the log directory. 186 * 187 * @return log directory, or empty string if logging not yet started 188 */ 189 public static synchronized String getLogDir() { 190 if (m_logDir == null) { 191 return ""; 192 } 193 return m_logDir; 194 } 195 196 /** 197 * Enable or disable logging of NetworkTables data. Note that unlike the network interface for 198 * NetworkTables, this will capture every value change. Defaults to enabled. 199 * 200 * @param enabled true to enable, false to disable 201 */ 202 public static synchronized void logNetworkTables(boolean enabled) { 203 boolean wasEnabled = m_ntLoggerEnabled; 204 m_ntLoggerEnabled = enabled; 205 if (m_log == null) { 206 start(); 207 return; 208 } 209 if (enabled && !wasEnabled) { 210 startNtLog(); 211 } else if (!enabled && wasEnabled) { 212 stopNtLog(); 213 } 214 } 215 216 /** 217 * Enable or disable logging of the console output. Defaults to enabled. 218 * 219 * @param enabled true to enable, false to disable 220 */ 221 public static synchronized void logConsoleOutput(boolean enabled) { 222 boolean wasEnabled = m_consoleLoggerEnabled; 223 m_consoleLoggerEnabled = enabled; 224 if (m_log == null) { 225 start(); 226 return; 227 } 228 if (enabled && !wasEnabled) { 229 startConsoleLog(); 230 } else if (!enabled && wasEnabled) { 231 stopConsoleLog(); 232 } 233 } 234 235 private static String makeLogDir(String dir) { 236 if (!dir.isEmpty()) { 237 return dir; 238 } 239 240 if (RobotBase.isReal()) { 241 try { 242 // prefer a mounted USB drive if one is accessible 243 Path usbDir = Paths.get("/u").toRealPath(); 244 if (Files.isWritable(usbDir)) { 245 if (!new File("/u/logs").mkdir()) { 246 // ignored 247 } 248 HAL.reportUsage("DataLogManager", "USB"); 249 return "/u/logs"; 250 } 251 } catch (IOException ex) { 252 // ignored 253 } 254 if (!new File("/home/systemcore/logs").mkdir()) { 255 // ignored 256 } 257 HAL.reportUsage("DataLogManager", "Onboard"); 258 return "/home/systemcore/logs"; 259 } 260 String logDir = Filesystem.getOperatingDirectory().getAbsolutePath() + "/logs"; 261 if (!new File(logDir).mkdir()) { 262 // ignored 263 } 264 return logDir; 265 } 266 267 private static String makeLogFilename(String filenameOverride) { 268 if (!filenameOverride.isEmpty()) { 269 return filenameOverride; 270 } 271 Random rnd = new Random(); 272 StringBuilder filename = new StringBuilder(); 273 filename.append("FRC_TBD_"); 274 for (int i = 0; i < 4; i++) { 275 filename.append(String.format("%04x", rnd.nextInt(0x10000))); 276 } 277 filename.append(".wpilog"); 278 return filename.toString(); 279 } 280 281 private static void startNtLog() { 282 NetworkTableInstance inst = NetworkTableInstance.getDefault(); 283 m_ntEntryLogger = inst.startEntryDataLog(m_log, "", "NT:"); 284 m_ntConnLogger = inst.startConnectionDataLog(m_log, "NTConnection"); 285 } 286 287 private static void stopNtLog() { 288 NetworkTableInstance.stopEntryDataLog(m_ntEntryLogger); 289 NetworkTableInstance.stopConnectionDataLog(m_ntConnLogger); 290 } 291 292 private static void startConsoleLog() { 293 if (RobotBase.isReal()) { 294 m_consoleLogger = new FileLogger("/home/systemcore/FRC_UserProgram.log", m_log, "console"); 295 } 296 } 297 298 private static void stopConsoleLog() { 299 if (RobotBase.isReal()) { 300 m_consoleLogger.close(); 301 } 302 } 303 304 private static void logMain() { 305 // based on free disk space, scan for "old" FRC_*.wpilog files and remove 306 { 307 File logDir = new File(m_logDir); 308 long freeSpace = logDir.getUsableSpace(); 309 if (freeSpace < kFreeSpaceThreshold) { 310 // Delete oldest FRC_*.wpilog files (ignore FRC_TBD_*.wpilog as we just created one) 311 File[] files = 312 logDir.listFiles( 313 (dir, name) -> 314 name.startsWith("FRC_") 315 && name.endsWith(".wpilog") 316 && !name.startsWith("FRC_TBD_")); 317 if (files != null) { 318 Arrays.sort(files, Comparator.comparingLong(File::lastModified)); 319 int count = files.length; 320 for (File file : files) { 321 --count; 322 if (count < kFileCountThreshold) { 323 break; 324 } 325 long length = file.length(); 326 if (file.delete()) { 327 DriverStation.reportWarning("DataLogManager: Deleted " + file.getName(), false); 328 freeSpace += length; 329 if (freeSpace >= kFreeSpaceThreshold) { 330 break; 331 } 332 } else { 333 System.err.println("DataLogManager: could not delete " + file); 334 } 335 } 336 } 337 } else if (freeSpace < 2 * kFreeSpaceThreshold) { 338 DriverStation.reportWarning( 339 "DataLogManager: Log storage device has " 340 + freeSpace / 1000000 341 + " MB of free space remaining! Logs will get deleted below " 342 + kFreeSpaceThreshold / 1000000 343 + " MB of free space. " 344 + "Consider deleting logs off the storage device.", 345 false); 346 } 347 } 348 349 int timeoutCount = 0; 350 boolean paused = false; 351 int dsAttachCount = 0; 352 int fmsAttachCount = 0; 353 boolean dsRenamed = m_filenameOverride; 354 boolean fmsRenamed = m_filenameOverride; 355 int sysTimeCount = 0; 356 IntegerLogEntry sysTimeEntry = 357 new IntegerLogEntry( 358 m_log, "systemTime", "{\"source\":\"DataLogManager\",\"format\":\"time_t_us\"}"); 359 360 Event newDataEvent = new Event(); 361 DriverStation.provideRefreshedDataEventHandle(newDataEvent.getHandle()); 362 while (!Thread.interrupted()) { 363 boolean timedOut; 364 try { 365 timedOut = WPIUtilJNI.waitForObjectTimeout(newDataEvent.getHandle(), 0.25); 366 } catch (InterruptedException e) { 367 break; 368 } 369 if (Thread.interrupted()) { 370 break; 371 } 372 if (timedOut) { 373 timeoutCount++; 374 // pause logging after being disconnected for 10 seconds 375 if (timeoutCount > 40 && !paused) { 376 timeoutCount = 0; 377 paused = true; 378 m_log.pause(); 379 } 380 continue; 381 } 382 // when we connect to the DS, resume logging 383 timeoutCount = 0; 384 if (paused) { 385 paused = false; 386 m_log.resume(); 387 } 388 389 if (!dsRenamed) { 390 // track DS attach 391 if (DriverStation.isDSAttached()) { 392 dsAttachCount++; 393 } else { 394 dsAttachCount = 0; 395 } 396 if (dsAttachCount > 50) { // 1 second 397 if (RobotController.isSystemTimeValid()) { 398 LocalDateTime now = LocalDateTime.now(m_utc); 399 m_log.setFilename("FRC_" + m_timeFormatter.format(now) + ".wpilog"); 400 dsRenamed = true; 401 } else { 402 dsAttachCount = 0; // wait a bit and try again 403 } 404 } 405 } 406 407 if (!fmsRenamed) { 408 // track FMS attach 409 if (DriverStation.isFMSAttached()) { 410 fmsAttachCount++; 411 } else { 412 fmsAttachCount = 0; 413 } 414 if (fmsAttachCount > 250) { // 5 seconds 415 // match info comes through TCP, so we need to double-check we've 416 // actually received it 417 DriverStation.MatchType matchType = DriverStation.getMatchType(); 418 if (matchType != DriverStation.MatchType.None) { 419 // rename per match info 420 char matchTypeChar = 421 switch (matchType) { 422 case Practice -> 'P'; 423 case Qualification -> 'Q'; 424 case Elimination -> 'E'; 425 default -> '_'; 426 }; 427 m_log.setFilename( 428 "FRC_" 429 + m_timeFormatter.format(LocalDateTime.now(m_utc)) 430 + "_" 431 + DriverStation.getEventName() 432 + "_" 433 + matchTypeChar 434 + DriverStation.getMatchNumber() 435 + ".wpilog"); 436 fmsRenamed = true; 437 dsRenamed = true; // don't override FMS rename 438 } 439 } 440 } 441 442 // Write system time every ~5 seconds 443 sysTimeCount++; 444 if (sysTimeCount >= 250) { 445 sysTimeCount = 0; 446 if (RobotController.isSystemTimeValid()) { 447 sysTimeEntry.append(WPIUtilJNI.getSystemTime(), WPIUtilJNI.now()); 448 } 449 } 450 } 451 newDataEvent.close(); 452 } 453}