001// Copyright (c) FIRST and other WPILib contributors.
002// Open Source Software; you can modify and/or share it under the terms of
003// the WPILib BSD license file in the root directory of this project.
004
005package edu.wpi.first.cscore;
006
007import edu.wpi.first.util.PixelFormat;
008import edu.wpi.first.util.RawFrame;
009import java.nio.ByteBuffer;
010import org.opencv.core.CvType;
011import org.opencv.core.Mat;
012
013/**
014 * A sink for user code to accept video frames as OpenCV images. These sinks require the WPILib
015 * OpenCV builds. For an alternate OpenCV, see the documentation how to build your own with RawSink.
016 */
017public class CvSink extends ImageSink {
018  private final RawFrame m_frame = new RawFrame();
019  private Mat m_tmpMat;
020  private ByteBuffer m_origByteBuffer;
021  private int m_width;
022  private int m_height;
023  private PixelFormat m_pixelFormat;
024
025  @Override
026  public void close() {
027    if (m_tmpMat != null) {
028      m_tmpMat.release();
029    }
030    m_frame.close();
031    super.close();
032  }
033
034  private int getCVFormat(PixelFormat pixelFormat) {
035    int type = 0;
036    switch (pixelFormat) {
037      case kYUYV:
038      case kRGB565:
039      case kY16:
040      case kUYVY:
041        type = CvType.CV_8UC2;
042        break;
043      case kBGR:
044        type = CvType.CV_8UC3;
045        break;
046      case kBGRA:
047        type = CvType.CV_8UC4;
048        break;
049      case kGray:
050      case kMJPEG:
051      default:
052        type = CvType.CV_8UC1;
053        break;
054    }
055    return type;
056  }
057
058  /**
059   * Create a sink for accepting OpenCV images. grabFrame() must be called on the created sink to
060   * get each new image.
061   *
062   * @param name Source name (arbitrary unique identifier)
063   * @param pixelFormat Source pixel format
064   */
065  public CvSink(String name, PixelFormat pixelFormat) {
066    super(CameraServerJNI.createRawSink(name, true));
067    m_pixelFormat = pixelFormat;
068    OpenCvLoader.forceStaticLoad();
069  }
070
071  /**
072   * Create a sink for accepting OpenCV images. WaitForFrame() must be called on the created sink to
073   * get each new image. Defaults to kBGR for pixelFormat
074   *
075   * @param name Source name (arbitrary unique identifier)
076   */
077  public CvSink(String name) {
078    this(name, PixelFormat.kBGR);
079  }
080
081  /**
082   * Wait for the next frame and get the image. Times out (returning 0) after 0.225 seconds. The
083   * provided image will have the pixelFormat this class was constructed with.
084   *
085   * @param image Where to store the image.
086   * @return Frame time, or 0 on error (call GetError() to obtain the error message)
087   */
088  public long grabFrame(Mat image) {
089    return grabFrame(image, 0.225);
090  }
091
092  /**
093   * Wait for the next frame and get the image. Times out (returning 0) after timeout seconds. The
094   * provided image will have the pixelFormat this class was constructed with.
095   *
096   * @param image Where to store the image.
097   * @param timeout Retrieval timeout in seconds.
098   * @return Frame time, or 0 on error (call GetError() to obtain the error message); the frame time
099   *     is in 1 us increments.
100   */
101  public long grabFrame(Mat image, double timeout) {
102    long rv = grabFrameDirect(timeout);
103    if (rv <= 0) {
104      return rv;
105    }
106    m_tmpMat.copyTo(image);
107    return rv;
108  }
109
110  /**
111   * Wait for the next frame and get the image. May block forever. The provided image will have the
112   * pixelFormat this class was constructed with.
113   *
114   * @param image Where to store the image.
115   * @return Frame time, or 0 on error (call GetError() to obtain the error message); the frame time
116   *     is in 1 us increments.
117   */
118  public long grabFrameNoTimeout(Mat image) {
119    long rv = grabFrameNoTimeoutDirect();
120    if (rv <= 0) {
121      return rv;
122    }
123    m_tmpMat.copyTo(image);
124    return rv;
125  }
126
127  /**
128   * Get the direct backing mat for this sink.
129   *
130   * <p>This mat can be invalidated any time any of the grab* methods are called, or when the CvSink
131   * is closed.
132   *
133   * @return The backing mat.
134   */
135  public Mat getDirectMat() {
136    return m_tmpMat;
137  }
138
139  /**
140   * Wait for the next frame and store the image. Times out (returning 0) after 0.225 seconds. The
141   * provided image will have the pixelFormat this class was constructed with. Use getDirectMat() to
142   * grab the image.
143   *
144   * @return Frame time, or 0 on error (call GetError() to obtain the error message)
145   */
146  public long grabFrameDirect() {
147    return grabFrameDirect(0.225);
148  }
149
150  /**
151   * Wait for the next frame and store the image. Times out (returning 0) after timeout seconds. The
152   * provided image will have the pixelFormat this class was constructed with. Use getDirectMat() to
153   * grab the image.
154   *
155   * @param timeout Retrieval timeout in seconds.
156   * @return Frame time, or 0 on error (call GetError() to obtain the error message); the frame time
157   *     is in 1 us increments.
158   */
159  @SuppressWarnings("PMD.CompareObjectsWithEquals")
160  public long grabFrameDirect(double timeout) {
161    m_frame.setInfo(0, 0, 0, m_pixelFormat);
162    long rv =
163        CameraServerJNI.grabRawSinkFrameTimeout(m_handle, m_frame, m_frame.getNativeObj(), timeout);
164    if (rv <= 0) {
165      return rv;
166    }
167
168    if (m_frame.getData() != m_origByteBuffer
169        || m_width != m_frame.getWidth()
170        || m_height != m_frame.getHeight()
171        || m_pixelFormat != m_frame.getPixelFormat()) {
172      m_origByteBuffer = m_frame.getData();
173      m_height = m_frame.getHeight();
174      m_width = m_frame.getWidth();
175      m_pixelFormat = m_frame.getPixelFormat();
176      if (m_frame.getStride() == 0) {
177        m_tmpMat =
178            new Mat(
179                m_frame.getHeight(),
180                m_frame.getWidth(),
181                getCVFormat(m_pixelFormat),
182                m_origByteBuffer);
183      } else {
184        m_tmpMat =
185            new Mat(
186                m_frame.getHeight(),
187                m_frame.getWidth(),
188                getCVFormat(m_pixelFormat),
189                m_origByteBuffer,
190                m_frame.getStride());
191      }
192    }
193    return rv;
194  }
195
196  /**
197   * Wait for the next frame and store the image. May block forever. The provided image will have
198   * the pixelFormat this class was constructed with. Use getDirectMat() to grab the image.
199   *
200   * @return Frame time, or 0 on error (call GetError() to obtain the error message); the frame time
201   *     is in 1 us increments.
202   */
203  @SuppressWarnings("PMD.CompareObjectsWithEquals")
204  public long grabFrameNoTimeoutDirect() {
205    m_frame.setInfo(0, 0, 0, m_pixelFormat);
206    long rv = CameraServerJNI.grabRawSinkFrame(m_handle, m_frame, m_frame.getNativeObj());
207    if (rv <= 0) {
208      return rv;
209    }
210
211    if (m_frame.getData() != m_origByteBuffer
212        || m_width != m_frame.getWidth()
213        || m_height != m_frame.getHeight()
214        || m_pixelFormat != m_frame.getPixelFormat()) {
215      m_origByteBuffer = m_frame.getData();
216      m_height = m_frame.getHeight();
217      m_width = m_frame.getWidth();
218      m_pixelFormat = m_frame.getPixelFormat();
219      if (m_frame.getStride() == 0) {
220        m_tmpMat =
221            new Mat(
222                m_frame.getHeight(),
223                m_frame.getWidth(),
224                getCVFormat(m_pixelFormat),
225                m_origByteBuffer);
226      } else {
227        m_tmpMat =
228            new Mat(
229                m_frame.getHeight(),
230                m_frame.getWidth(),
231                getCVFormat(m_pixelFormat),
232                m_origByteBuffer,
233                m_frame.getStride());
234      }
235    }
236    return rv;
237  }
238}