001// Copyright (c) FIRST and other WPILib contributors. 002// Open Source Software; you can modify and/or share it under the terms of 003// the WPILib BSD license file in the root directory of this project. 004 005package edu.wpi.first.cscore; 006 007import edu.wpi.first.util.PixelFormat; 008import edu.wpi.first.util.RawFrame; 009import edu.wpi.first.util.TimestampSource; 010import java.nio.ByteBuffer; 011import org.opencv.core.CvType; 012import org.opencv.core.Mat; 013 014/** 015 * A sink for user code to accept video frames as OpenCV images. These sinks require the WPILib 016 * OpenCV builds. For an alternate OpenCV, see the documentation how to build your own with RawSink. 017 */ 018public class CvSink extends ImageSink { 019 private final RawFrame m_frame = new RawFrame(); 020 private Mat m_tmpMat; 021 private ByteBuffer m_origByteBuffer; 022 private int m_width; 023 private int m_height; 024 private PixelFormat m_pixelFormat; 025 026 @Override 027 public void close() { 028 if (m_tmpMat != null) { 029 m_tmpMat.release(); 030 } 031 m_frame.close(); 032 super.close(); 033 } 034 035 private int getCVFormat(PixelFormat pixelFormat) { 036 return switch (pixelFormat) { 037 case kYUYV, kRGB565, kY16, kUYVY -> CvType.CV_8UC2; 038 case kBGR -> CvType.CV_8UC3; 039 case kBGRA -> CvType.CV_8UC4; 040 case kGray, kMJPEG, kUnknown -> CvType.CV_8UC1; 041 }; 042 } 043 044 /** 045 * Create a sink for accepting OpenCV images. grabFrame() must be called on the created sink to 046 * get each new image. 047 * 048 * @param name Sink name (arbitrary unique identifier) 049 * @param pixelFormat Source pixel format 050 */ 051 public CvSink(String name, PixelFormat pixelFormat) { 052 super(CameraServerJNI.createRawSink(name, true)); 053 m_pixelFormat = pixelFormat; 054 OpenCvLoader.forceStaticLoad(); 055 } 056 057 /** 058 * Create a sink for accepting OpenCV images. WaitForFrame() must be called on the created sink to 059 * get each new image. Defaults to kBGR for pixelFormat 060 * 061 * @param name Source name (arbitrary unique identifier) 062 */ 063 public CvSink(String name) { 064 this(name, PixelFormat.kBGR); 065 } 066 067 /** 068 * Wait for the next frame and get the image. Times out (returning 0) after 0.225 seconds. The 069 * provided image will have the pixelFormat this class was constructed with. 070 * 071 * @param image Where to store the image. 072 * @return Frame time, or 0 on error (call GetError() to obtain the error message) 073 */ 074 public long grabFrame(Mat image) { 075 return grabFrame(image, 0.225); 076 } 077 078 /** 079 * Wait for the next frame and get the image. Times out (returning 0) after timeout seconds. The 080 * provided image will have the pixelFormat this class was constructed with. 081 * 082 * @param image Where to store the image. 083 * @param timeout Retrieval timeout in seconds. 084 * @return Frame time, or 0 on error (call GetError() to obtain the error message); the frame time 085 * is in 1 us increments. 086 */ 087 public long grabFrame(Mat image, double timeout) { 088 long rv = grabFrameDirect(timeout); 089 if (rv <= 0) { 090 return rv; 091 } 092 m_tmpMat.copyTo(image); 093 return rv; 094 } 095 096 /** 097 * Wait for the next frame and get the image. May block forever. The provided image will have the 098 * pixelFormat this class was constructed with. 099 * 100 * @param image Where to store the image. 101 * @return Frame time, or 0 on error (call GetError() to obtain the error message); the frame time 102 * is in 1 us increments. 103 */ 104 public long grabFrameNoTimeout(Mat image) { 105 long rv = grabFrameNoTimeoutDirect(); 106 if (rv <= 0) { 107 return rv; 108 } 109 m_tmpMat.copyTo(image); 110 return rv; 111 } 112 113 /** 114 * Get the direct backing mat for this sink. 115 * 116 * <p>This mat can be invalidated any time any of the grab* methods are called, or when the CvSink 117 * is closed. 118 * 119 * @return The backing mat. 120 */ 121 public Mat getDirectMat() { 122 return m_tmpMat; 123 } 124 125 /** 126 * Wait for the next frame and store the image. Times out (returning 0) after 0.225 seconds. The 127 * provided image will have the pixelFormat this class was constructed with. Use getDirectMat() to 128 * grab the image. 129 * 130 * @return Frame time, or 0 on error (call GetError() to obtain the error message) 131 */ 132 public long grabFrameDirect() { 133 return grabFrameDirect(0.225); 134 } 135 136 /** 137 * Wait for the next frame and store the image. Times out (returning 0) after timeout seconds. The 138 * provided image will have the pixelFormat this class was constructed with. Use getDirectMat() to 139 * grab the image. 140 * 141 * @param timeout Retrieval timeout in seconds. 142 * @return Frame time, or 0 on error (call GetError() to obtain the error message); the frame time 143 * is in 1 us increments. 144 */ 145 @SuppressWarnings("PMD.CompareObjectsWithEquals") 146 public long grabFrameDirect(double timeout) { 147 m_frame.setInfo(0, 0, 0, m_pixelFormat); 148 long rv = 149 CameraServerJNI.grabRawSinkFrameTimeout(m_handle, m_frame, m_frame.getNativeObj(), timeout); 150 if (rv <= 0) { 151 return rv; 152 } 153 154 if (m_frame.getData() != m_origByteBuffer 155 || m_width != m_frame.getWidth() 156 || m_height != m_frame.getHeight() 157 || m_pixelFormat != m_frame.getPixelFormat()) { 158 m_origByteBuffer = m_frame.getData(); 159 m_height = m_frame.getHeight(); 160 m_width = m_frame.getWidth(); 161 m_pixelFormat = m_frame.getPixelFormat(); 162 if (m_frame.getStride() == 0) { 163 m_tmpMat = 164 new Mat( 165 m_frame.getHeight(), 166 m_frame.getWidth(), 167 getCVFormat(m_pixelFormat), 168 m_origByteBuffer); 169 } else { 170 m_tmpMat = 171 new Mat( 172 m_frame.getHeight(), 173 m_frame.getWidth(), 174 getCVFormat(m_pixelFormat), 175 m_origByteBuffer, 176 m_frame.getStride()); 177 } 178 } 179 return rv; 180 } 181 182 /** 183 * Wait for the next frame and store the image. May block forever. The provided image will have 184 * the pixelFormat this class was constructed with. Use getDirectMat() to grab the image. 185 * 186 * @return Frame time, or 0 on error (call GetError() to obtain the error message); the frame time 187 * is in 1 us increments. 188 */ 189 @SuppressWarnings("PMD.CompareObjectsWithEquals") 190 public long grabFrameNoTimeoutDirect() { 191 m_frame.setInfo(0, 0, 0, m_pixelFormat); 192 long rv = CameraServerJNI.grabRawSinkFrame(m_handle, m_frame, m_frame.getNativeObj()); 193 if (rv <= 0) { 194 return rv; 195 } 196 197 if (m_frame.getData() != m_origByteBuffer 198 || m_width != m_frame.getWidth() 199 || m_height != m_frame.getHeight() 200 || m_pixelFormat != m_frame.getPixelFormat()) { 201 m_origByteBuffer = m_frame.getData(); 202 m_height = m_frame.getHeight(); 203 m_width = m_frame.getWidth(); 204 m_pixelFormat = m_frame.getPixelFormat(); 205 if (m_frame.getStride() == 0) { 206 m_tmpMat = 207 new Mat( 208 m_frame.getHeight(), 209 m_frame.getWidth(), 210 getCVFormat(m_pixelFormat), 211 m_origByteBuffer); 212 } else { 213 m_tmpMat = 214 new Mat( 215 m_frame.getHeight(), 216 m_frame.getWidth(), 217 getCVFormat(m_pixelFormat), 218 m_origByteBuffer, 219 m_frame.getStride()); 220 } 221 } 222 return rv; 223 } 224 225 /** 226 * Get the last time a frame was grabbed. This uses the same time base as wpi::Now(). 227 * 228 * @return Time in 1 us increments. 229 */ 230 public long getLastFrameTime() { 231 return m_frame.getTimestamp(); 232 } 233 234 /** 235 * Get the time source for the timestamp the last frame was grabbed at. 236 * 237 * @return Time source 238 */ 239 public TimestampSource getLastFrameTimeSource() { 240 return m_frame.getTimestampSource(); 241 } 242}