001// Copyright (c) FIRST and other WPILib contributors.
002// Open Source Software; you can modify and/or share it under the terms of
003// the WPILib BSD license file in the root directory of this project.
004
005package edu.wpi.first.cscore;
006
007import edu.wpi.first.util.PixelFormat;
008import edu.wpi.first.util.RawFrame;
009import edu.wpi.first.util.TimestampSource;
010import java.nio.ByteBuffer;
011import org.opencv.core.CvType;
012import org.opencv.core.Mat;
013
014/**
015 * A sink for user code to accept video frames as OpenCV images. These sinks require the WPILib
016 * OpenCV builds. For an alternate OpenCV, see the documentation how to build your own with RawSink.
017 */
018public class CvSink extends ImageSink {
019  private final RawFrame m_frame = new RawFrame();
020  private Mat m_tmpMat;
021  private ByteBuffer m_origByteBuffer;
022  private int m_width;
023  private int m_height;
024  private PixelFormat m_pixelFormat;
025
026  @Override
027  public void close() {
028    if (m_tmpMat != null) {
029      m_tmpMat.release();
030    }
031    m_frame.close();
032    super.close();
033  }
034
035  private int getCVFormat(PixelFormat pixelFormat) {
036    return switch (pixelFormat) {
037      case kYUYV, kRGB565, kY16, kUYVY -> CvType.CV_8UC2;
038      case kBGR -> CvType.CV_8UC3;
039      case kBGRA -> CvType.CV_8UC4;
040      case kGray, kMJPEG, kUnknown -> CvType.CV_8UC1;
041    };
042  }
043
044  /**
045   * Create a sink for accepting OpenCV images. grabFrame() must be called on the created sink to
046   * get each new image.
047   *
048   * @param name Sink name (arbitrary unique identifier)
049   * @param pixelFormat Source pixel format
050   */
051  public CvSink(String name, PixelFormat pixelFormat) {
052    super(CameraServerJNI.createRawSink(name, true));
053    m_pixelFormat = pixelFormat;
054    OpenCvLoader.forceStaticLoad();
055  }
056
057  /**
058   * Create a sink for accepting OpenCV images. WaitForFrame() must be called on the created sink to
059   * get each new image. Defaults to kBGR for pixelFormat
060   *
061   * @param name Source name (arbitrary unique identifier)
062   */
063  public CvSink(String name) {
064    this(name, PixelFormat.kBGR);
065  }
066
067  /**
068   * Wait for the next frame and get the image. Times out (returning 0) after 0.225 seconds. The
069   * provided image will have the pixelFormat this class was constructed with.
070   *
071   * @param image Where to store the image.
072   * @return Frame time, or 0 on error (call GetError() to obtain the error message)
073   */
074  public long grabFrame(Mat image) {
075    return grabFrame(image, 0.225);
076  }
077
078  /**
079   * Wait for the next frame and get the image. Times out (returning 0) after timeout seconds. The
080   * provided image will have the pixelFormat this class was constructed with.
081   *
082   * @param image Where to store the image.
083   * @param timeout Retrieval timeout in seconds.
084   * @return Frame time, or 0 on error (call GetError() to obtain the error message); the frame time
085   *     is in 1 us increments.
086   */
087  public long grabFrame(Mat image, double timeout) {
088    long rv = grabFrameDirect(timeout);
089    if (rv <= 0) {
090      return rv;
091    }
092    m_tmpMat.copyTo(image);
093    return rv;
094  }
095
096  /**
097   * Wait for the next frame and get the image. May block forever. The provided image will have the
098   * pixelFormat this class was constructed with.
099   *
100   * @param image Where to store the image.
101   * @return Frame time, or 0 on error (call GetError() to obtain the error message); the frame time
102   *     is in 1 us increments.
103   */
104  public long grabFrameNoTimeout(Mat image) {
105    long rv = grabFrameNoTimeoutDirect();
106    if (rv <= 0) {
107      return rv;
108    }
109    m_tmpMat.copyTo(image);
110    return rv;
111  }
112
113  /**
114   * Get the direct backing mat for this sink.
115   *
116   * <p>This mat can be invalidated any time any of the grab* methods are called, or when the CvSink
117   * is closed.
118   *
119   * @return The backing mat.
120   */
121  public Mat getDirectMat() {
122    return m_tmpMat;
123  }
124
125  /**
126   * Wait for the next frame and store the image. Times out (returning 0) after 0.225 seconds. The
127   * provided image will have the pixelFormat this class was constructed with. Use getDirectMat() to
128   * grab the image.
129   *
130   * @return Frame time, or 0 on error (call GetError() to obtain the error message)
131   */
132  public long grabFrameDirect() {
133    return grabFrameDirect(0.225);
134  }
135
136  /**
137   * Wait for the next frame and store the image. Times out (returning 0) after timeout seconds. The
138   * provided image will have the pixelFormat this class was constructed with. Use getDirectMat() to
139   * grab the image.
140   *
141   * @param timeout Retrieval timeout in seconds.
142   * @return Frame time, or 0 on error (call GetError() to obtain the error message); the frame time
143   *     is in 1 us increments.
144   */
145  @SuppressWarnings("PMD.CompareObjectsWithEquals")
146  public long grabFrameDirect(double timeout) {
147    m_frame.setInfo(0, 0, 0, m_pixelFormat);
148    long rv =
149        CameraServerJNI.grabRawSinkFrameTimeout(m_handle, m_frame, m_frame.getNativeObj(), timeout);
150    if (rv <= 0) {
151      return rv;
152    }
153
154    if (m_frame.getData() != m_origByteBuffer
155        || m_width != m_frame.getWidth()
156        || m_height != m_frame.getHeight()
157        || m_pixelFormat != m_frame.getPixelFormat()) {
158      m_origByteBuffer = m_frame.getData();
159      m_height = m_frame.getHeight();
160      m_width = m_frame.getWidth();
161      m_pixelFormat = m_frame.getPixelFormat();
162      if (m_frame.getStride() == 0) {
163        m_tmpMat =
164            new Mat(
165                m_frame.getHeight(),
166                m_frame.getWidth(),
167                getCVFormat(m_pixelFormat),
168                m_origByteBuffer);
169      } else {
170        m_tmpMat =
171            new Mat(
172                m_frame.getHeight(),
173                m_frame.getWidth(),
174                getCVFormat(m_pixelFormat),
175                m_origByteBuffer,
176                m_frame.getStride());
177      }
178    }
179    return rv;
180  }
181
182  /**
183   * Wait for the next frame and store the image. May block forever. The provided image will have
184   * the pixelFormat this class was constructed with. Use getDirectMat() to grab the image.
185   *
186   * @return Frame time, or 0 on error (call GetError() to obtain the error message); the frame time
187   *     is in 1 us increments.
188   */
189  @SuppressWarnings("PMD.CompareObjectsWithEquals")
190  public long grabFrameNoTimeoutDirect() {
191    m_frame.setInfo(0, 0, 0, m_pixelFormat);
192    long rv = CameraServerJNI.grabRawSinkFrame(m_handle, m_frame, m_frame.getNativeObj());
193    if (rv <= 0) {
194      return rv;
195    }
196
197    if (m_frame.getData() != m_origByteBuffer
198        || m_width != m_frame.getWidth()
199        || m_height != m_frame.getHeight()
200        || m_pixelFormat != m_frame.getPixelFormat()) {
201      m_origByteBuffer = m_frame.getData();
202      m_height = m_frame.getHeight();
203      m_width = m_frame.getWidth();
204      m_pixelFormat = m_frame.getPixelFormat();
205      if (m_frame.getStride() == 0) {
206        m_tmpMat =
207            new Mat(
208                m_frame.getHeight(),
209                m_frame.getWidth(),
210                getCVFormat(m_pixelFormat),
211                m_origByteBuffer);
212      } else {
213        m_tmpMat =
214            new Mat(
215                m_frame.getHeight(),
216                m_frame.getWidth(),
217                getCVFormat(m_pixelFormat),
218                m_origByteBuffer,
219                m_frame.getStride());
220      }
221    }
222    return rv;
223  }
224
225  /**
226   * Get the last time a frame was grabbed. This uses the same time base as wpi::Now().
227   *
228   * @return Time in 1 us increments.
229   */
230  public long getLastFrameTime() {
231    return m_frame.getTimestamp();
232  }
233
234  /**
235   * Get the time source for the timestamp the last frame was grabbed at.
236   *
237   * @return Time source
238   */
239  public TimestampSource getLastFrameTimeSource() {
240    return m_frame.getTimestampSource();
241  }
242}