lavu/fifo: return errors on trying to read/write too much

Trying to write too much will currently overwrite previous data. Trying
to read too much will either av_assert2() in av_fifo_drain() or return
old data. Trying to peek too much will either av_assert2() in
av_fifo_generic_peek_at() or return old data.

Return an error code in all these cases, which is safer and more
consistent.

libavutil/fifo.c

@@ -134,6 +134,9 @@ int av_fifo_generic_write(AVFifoBuffer *f, void *src, int size,
     uint32_t wndx= f->wndx;
     uint8_t *wptr= f->wptr;
 
+    if (size > av_fifo_space(f))
+        return AVERROR(ENOSPC);
+
     do {
         int len = FFMIN(f->end - wptr, size);
         if (func) {
@@ -159,13 +162,8 @@ int av_fifo_generic_peek_at(AVFifoBuffer *f, void *dest, int offset, int buf_siz
 {
     uint8_t *rptr = f->rptr;
 
-    av_assert2(offset >= 0);
-
-    /*
-     * *ndx are indexes modulo 2^32, they are intended to overflow,
-     * to handle *ndx greater than 4gb.
-     */
-    av_assert2(buf_size + (unsigned)offset <= f->wndx - f->rndx);
+    if (offset < 0 || buf_size > av_fifo_size(f) - offset)
+        return AVERROR(EINVAL);
 
     if (offset >= f->end - rptr)
         rptr += offset - (f->end - f->buffer);
@@ -198,6 +196,9 @@ int av_fifo_generic_peek(AVFifoBuffer *f, void *dest, int buf_size,
 {
     uint8_t *rptr = f->rptr;
 
+    if (buf_size > av_fifo_size(f))
+        return AVERROR(EINVAL);
+
     do {
         int len = FFMIN(f->end - rptr, buf_size);
         if (func)
@@ -218,6 +219,9 @@ int av_fifo_generic_peek(AVFifoBuffer *f, void *dest, int buf_size,
 int av_fifo_generic_read(AVFifoBuffer *f, void *dest, int buf_size,
                          void (*func)(void *, void *, int))
 {
+    if (buf_size > av_fifo_size(f))
+        return AVERROR(EINVAL);
+
     do {
         int len = FFMIN(f->end - f->rptr, buf_size);
         if (func)

libavutil/fifo.h

@@ -91,6 +91,8 @@ int av_fifo_space(const AVFifoBuffer *f);
  * @param buf_size number of bytes to read
  * @param func generic read function
  * @param dest data destination
+ *
+ * @return a non-negative number on success, a negative error code on failure
  */
 int av_fifo_generic_peek_at(AVFifoBuffer *f, void *dest, int offset, int buf_size, void (*func)(void*, void*, int));
 
@@ -101,6 +103,8 @@ int av_fifo_generic_peek_at(AVFifoBuffer *f, void *dest, int offset, int buf_siz
  * @param buf_size number of bytes to read
  * @param func generic read function
  * @param dest data destination
+ *
+ * @return a non-negative number on success, a negative error code on failure
  */
 int av_fifo_generic_peek(AVFifoBuffer *f, void *dest, int buf_size, void (*func)(void*, void*, int));
 
@@ -110,6 +114,8 @@ int av_fifo_generic_peek(AVFifoBuffer *f, void *dest, int buf_size, void (*func)
  * @param buf_size number of bytes to read
  * @param func generic read function
  * @param dest data destination
+ *
+ * @return a non-negative number on success, a negative error code on failure
  */
 int av_fifo_generic_read(AVFifoBuffer *f, void *dest, int buf_size, void (*func)(void*, void*, int));
 
@@ -124,7 +130,7 @@ int av_fifo_generic_read(AVFifoBuffer *f, void *dest, int buf_size, void (*func)
  * func must return the number of bytes written to dest_buf, or <= 0 to
  * indicate no more data available to write.
  * If func is NULL, src is interpreted as a simple byte array for source data.
- * @return the number of bytes written to the FIFO
+ * @return the number of bytes written to the FIFO or a negative error code on failure
  */
 int av_fifo_generic_write(AVFifoBuffer *f, void *src, int size, int (*func)(void*, void*, int));