001// Copyright (c) FIRST and other WPILib contributors. 002// Open Source Software; you can modify and/or share it under the terms of 003// the WPILib BSD license file in the root directory of this project. 004 005package edu.wpi.first.wpilibj; 006 007import edu.wpi.first.networktables.NetworkTableInstance; 008import edu.wpi.first.util.WPIUtilJNI; 009import edu.wpi.first.util.concurrent.Event; 010import edu.wpi.first.util.datalog.DataLog; 011import edu.wpi.first.util.datalog.DataLogBackgroundWriter; 012import edu.wpi.first.util.datalog.IntegerLogEntry; 013import edu.wpi.first.util.datalog.StringLogEntry; 014import java.io.File; 015import java.io.IOException; 016import java.nio.file.Files; 017import java.nio.file.Path; 018import java.nio.file.Paths; 019import java.time.LocalDateTime; 020import java.time.ZoneId; 021import java.time.format.DateTimeFormatter; 022import java.util.Arrays; 023import java.util.Comparator; 024import java.util.Random; 025 026/** 027 * Centralized data log that provides automatic data log file management. It automatically cleans up 028 * old files when disk space is low and renames the file based either on current date/time or (if 029 * available) competition match number. The data file will be saved to a USB flash drive in a folder 030 * named "logs" if one is attached, or to /home/lvuser/logs otherwise. 031 * 032 * <p>Log files are initially named "FRC_TBD_{random}.wpilog" until the DS connects. After the DS 033 * connects, the log file is renamed to "FRC_yyyyMMdd_HHmmss.wpilog" (where the date/time is UTC). 034 * If the FMS is connected and provides a match number, the log file is renamed to 035 * "FRC_yyyyMMdd_HHmmss_{event}_{match}.wpilog". 036 * 037 * <p>On startup, all existing FRC_TBD log files are deleted. If there is less than 50 MB of free 038 * space on the target storage, FRC_ log files are deleted (oldest to newest) until there is 50 MB 039 * free OR there are 10 files remaining. 040 * 041 * <p>By default, all NetworkTables value changes are stored to the data log. 042 */ 043public final class DataLogManager { 044 private static DataLogBackgroundWriter m_log; 045 private static boolean m_stopped; 046 private static String m_logDir; 047 private static boolean m_filenameOverride; 048 private static Thread m_thread; 049 private static final ZoneId m_utc = ZoneId.of("UTC"); 050 private static final DateTimeFormatter m_timeFormatter = 051 DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss").withZone(m_utc); 052 private static boolean m_ntLoggerEnabled = true; 053 private static int m_ntEntryLogger; 054 private static int m_ntConnLogger; 055 private static StringLogEntry m_messageLog; 056 057 // if less than this much free space, delete log files until there is this much free space 058 // OR there are this many files remaining. 059 private static final long kFreeSpaceThreshold = 50000000L; 060 private static final int kFileCountThreshold = 10; 061 062 private DataLogManager() {} 063 064 /** Start data log manager with default directory location. */ 065 public static synchronized void start() { 066 start("", "", 0.25); 067 } 068 069 /** 070 * Start data log manager. The parameters have no effect if the data log manager was already 071 * started (e.g. by calling another static function). 072 * 073 * @param dir if not empty, directory to use for data log storage 074 */ 075 public static synchronized void start(String dir) { 076 start(dir, "", 0.25); 077 } 078 079 /** 080 * Start data log manager. The parameters have no effect if the data log manager was already 081 * started (e.g. by calling another static function). 082 * 083 * @param dir if not empty, directory to use for data log storage 084 * @param filename filename to use; if none provided, the filename is automatically generated 085 */ 086 public static synchronized void start(String dir, String filename) { 087 start(dir, filename, 0.25); 088 } 089 090 /** 091 * Start data log manager. The parameters have no effect if the data log manager was already 092 * started (e.g. by calling another static function). 093 * 094 * @param dir if not empty, directory to use for data log storage 095 * @param filename filename to use; if none provided, the filename is automatically generated 096 * @param period time between automatic flushes to disk, in seconds; this is a time/storage 097 * tradeoff 098 */ 099 public static synchronized void start(String dir, String filename, double period) { 100 if (m_log == null) { 101 m_logDir = makeLogDir(dir); 102 m_filenameOverride = !filename.isEmpty(); 103 104 // Delete all previously existing FRC_TBD_*.wpilog files. These only exist when the robot 105 // never connects to the DS, so they are very unlikely to have useful data and just clutter 106 // the filesystem. 107 File[] files = 108 new File(m_logDir) 109 .listFiles((d, name) -> name.startsWith("FRC_TBD_") && name.endsWith(".wpilog")); 110 if (files != null) { 111 for (File file : files) { 112 if (!file.delete()) { 113 System.err.println("DataLogManager: could not delete " + file); 114 } 115 } 116 } 117 m_log = new DataLogBackgroundWriter(m_logDir, makeLogFilename(filename), period); 118 m_messageLog = new StringLogEntry(m_log, "messages"); 119 120 // Log all NT entries and connections 121 if (m_ntLoggerEnabled) { 122 startNtLog(); 123 } 124 } else if (m_stopped) { 125 m_log.setFilename(makeLogFilename(filename)); 126 m_log.resume(); 127 m_stopped = false; 128 } 129 130 if (m_thread == null) { 131 m_thread = new Thread(DataLogManager::logMain, "DataLogDS"); 132 m_thread.setDaemon(true); 133 m_thread.start(); 134 } 135 } 136 137 /** Stop data log manager. */ 138 public static synchronized void stop() { 139 if (m_thread != null) { 140 m_thread.interrupt(); 141 m_thread = null; 142 } 143 if (m_log != null) { 144 m_log.stop(); 145 m_stopped = true; 146 } 147 } 148 149 /** 150 * Log a message to the "messages" entry. The message is also printed to standard output (followed 151 * by a newline). 152 * 153 * @param message message 154 */ 155 public static synchronized void log(String message) { 156 if (m_messageLog == null) { 157 start(); 158 } 159 m_messageLog.append(message); 160 System.out.println(message); 161 } 162 163 /** 164 * Get the managed data log (for custom logging). Starts the data log manager if not already 165 * started. 166 * 167 * @return data log 168 */ 169 public static synchronized DataLog getLog() { 170 if (m_log == null) { 171 start(); 172 } 173 return m_log; 174 } 175 176 /** 177 * Get the log directory. 178 * 179 * @return log directory, or empty string if logging not yet started 180 */ 181 public static synchronized String getLogDir() { 182 if (m_logDir == null) { 183 return ""; 184 } 185 return m_logDir; 186 } 187 188 /** 189 * Enable or disable logging of NetworkTables data. Note that unlike the network interface for 190 * NetworkTables, this will capture every value change. Defaults to enabled. 191 * 192 * @param enabled true to enable, false to disable 193 */ 194 public static synchronized void logNetworkTables(boolean enabled) { 195 boolean wasEnabled = m_ntLoggerEnabled; 196 m_ntLoggerEnabled = enabled; 197 if (m_log == null) { 198 start(); 199 return; 200 } 201 if (enabled && !wasEnabled) { 202 startNtLog(); 203 } else if (!enabled && wasEnabled) { 204 stopNtLog(); 205 } 206 } 207 208 private static String makeLogDir(String dir) { 209 if (!dir.isEmpty()) { 210 return dir; 211 } 212 213 if (RobotBase.isReal()) { 214 try { 215 // prefer a mounted USB drive if one is accessible 216 Path usbDir = Paths.get("/u").toRealPath(); 217 if (Files.isWritable(usbDir)) { 218 if (!new File("/u/logs").mkdir()) { 219 // ignored 220 } 221 return "/u/logs"; 222 } 223 } catch (IOException ex) { 224 // ignored 225 } 226 if (RobotBase.getRuntimeType() == RuntimeType.kRoboRIO) { 227 DriverStation.reportWarning( 228 "DataLogManager: Logging to RoboRIO 1 internal storage is not recommended!" 229 + " Plug in a FAT32 formatted flash drive!", 230 false); 231 } 232 if (!new File("/home/lvuser/logs").mkdir()) { 233 // ignored 234 } 235 return "/home/lvuser/logs"; 236 } 237 String logDir = Filesystem.getOperatingDirectory().getAbsolutePath() + "/logs"; 238 if (!new File(logDir).mkdir()) { 239 // ignored 240 } 241 return logDir; 242 } 243 244 private static String makeLogFilename(String filenameOverride) { 245 if (!filenameOverride.isEmpty()) { 246 return filenameOverride; 247 } 248 Random rnd = new Random(); 249 StringBuilder filename = new StringBuilder(); 250 filename.append("FRC_TBD_"); 251 for (int i = 0; i < 4; i++) { 252 filename.append(String.format("%04x", rnd.nextInt(0x10000))); 253 } 254 filename.append(".wpilog"); 255 return filename.toString(); 256 } 257 258 private static void startNtLog() { 259 NetworkTableInstance inst = NetworkTableInstance.getDefault(); 260 m_ntEntryLogger = inst.startEntryDataLog(m_log, "", "NT:"); 261 m_ntConnLogger = inst.startConnectionDataLog(m_log, "NTConnection"); 262 } 263 264 private static void stopNtLog() { 265 NetworkTableInstance.stopEntryDataLog(m_ntEntryLogger); 266 NetworkTableInstance.stopConnectionDataLog(m_ntConnLogger); 267 } 268 269 private static void logMain() { 270 // based on free disk space, scan for "old" FRC_*.wpilog files and remove 271 { 272 File logDir = new File(m_logDir); 273 long freeSpace = logDir.getUsableSpace(); 274 if (freeSpace < kFreeSpaceThreshold) { 275 // Delete oldest FRC_*.wpilog files (ignore FRC_TBD_*.wpilog as we just created one) 276 File[] files = 277 logDir.listFiles( 278 (dir, name) -> 279 name.startsWith("FRC_") 280 && name.endsWith(".wpilog") 281 && !name.startsWith("FRC_TBD_")); 282 if (files != null) { 283 Arrays.sort(files, Comparator.comparingLong(File::lastModified)); 284 int count = files.length; 285 for (File file : files) { 286 --count; 287 if (count < kFileCountThreshold) { 288 break; 289 } 290 long length = file.length(); 291 if (file.delete()) { 292 DriverStation.reportWarning("DataLogManager: Deleted " + file.getName(), false); 293 freeSpace += length; 294 if (freeSpace >= kFreeSpaceThreshold) { 295 break; 296 } 297 } else { 298 System.err.println("DataLogManager: could not delete " + file); 299 } 300 } 301 } 302 } else if (freeSpace < 2 * kFreeSpaceThreshold) { 303 DriverStation.reportWarning( 304 "DataLogManager: Log storage device has " 305 + freeSpace / 1000000 306 + " MB of free space remaining! Logs will get deleted below " 307 + kFreeSpaceThreshold / 1000000 308 + " MB of free space. " 309 + "Consider deleting logs off the storage device.", 310 false); 311 } 312 } 313 314 int timeoutCount = 0; 315 boolean paused = false; 316 int dsAttachCount = 0; 317 int fmsAttachCount = 0; 318 boolean dsRenamed = m_filenameOverride; 319 boolean fmsRenamed = m_filenameOverride; 320 int sysTimeCount = 0; 321 IntegerLogEntry sysTimeEntry = 322 new IntegerLogEntry( 323 m_log, "systemTime", "{\"source\":\"DataLogManager\",\"format\":\"time_t_us\"}"); 324 325 Event newDataEvent = new Event(); 326 DriverStation.provideRefreshedDataEventHandle(newDataEvent.getHandle()); 327 while (!Thread.interrupted()) { 328 boolean timedOut; 329 try { 330 timedOut = WPIUtilJNI.waitForObjectTimeout(newDataEvent.getHandle(), 0.25); 331 } catch (InterruptedException e) { 332 break; 333 } 334 if (Thread.interrupted()) { 335 break; 336 } 337 if (timedOut) { 338 timeoutCount++; 339 // pause logging after being disconnected for 10 seconds 340 if (timeoutCount > 40 && !paused) { 341 timeoutCount = 0; 342 paused = true; 343 m_log.pause(); 344 } 345 continue; 346 } 347 // when we connect to the DS, resume logging 348 timeoutCount = 0; 349 if (paused) { 350 paused = false; 351 m_log.resume(); 352 } 353 354 if (!dsRenamed) { 355 // track DS attach 356 if (DriverStation.isDSAttached()) { 357 dsAttachCount++; 358 } else { 359 dsAttachCount = 0; 360 } 361 if (dsAttachCount > 50) { // 1 second 362 if (RobotController.isSystemTimeValid()) { 363 LocalDateTime now = LocalDateTime.now(m_utc); 364 m_log.setFilename("FRC_" + m_timeFormatter.format(now) + ".wpilog"); 365 dsRenamed = true; 366 } else { 367 dsAttachCount = 0; // wait a bit and try again 368 } 369 } 370 } 371 372 if (!fmsRenamed) { 373 // track FMS attach 374 if (DriverStation.isFMSAttached()) { 375 fmsAttachCount++; 376 } else { 377 fmsAttachCount = 0; 378 } 379 if (fmsAttachCount > 250) { // 5 seconds 380 // match info comes through TCP, so we need to double-check we've 381 // actually received it 382 DriverStation.MatchType matchType = DriverStation.getMatchType(); 383 if (matchType != DriverStation.MatchType.None) { 384 // rename per match info 385 char matchTypeChar; 386 switch (matchType) { 387 case Practice: 388 matchTypeChar = 'P'; 389 break; 390 case Qualification: 391 matchTypeChar = 'Q'; 392 break; 393 case Elimination: 394 matchTypeChar = 'E'; 395 break; 396 default: 397 matchTypeChar = '_'; 398 break; 399 } 400 m_log.setFilename( 401 "FRC_" 402 + m_timeFormatter.format(LocalDateTime.now(m_utc)) 403 + "_" 404 + DriverStation.getEventName() 405 + "_" 406 + matchTypeChar 407 + DriverStation.getMatchNumber() 408 + ".wpilog"); 409 fmsRenamed = true; 410 dsRenamed = true; // don't override FMS rename 411 } 412 } 413 } 414 415 // Write system time every ~5 seconds 416 sysTimeCount++; 417 if (sysTimeCount >= 250) { 418 sysTimeCount = 0; 419 if (RobotController.isSystemTimeValid()) { 420 sysTimeEntry.append(WPIUtilJNI.getSystemTime(), WPIUtilJNI.now()); 421 } 422 } 423 } 424 newDataEvent.close(); 425 } 426}