avfilter/formats: proper error handling in ff_channel_layouts_ref() and ff_formats_ref()

Also make sure the allocation and its check are properly done.

libavfilter/formats.c

@@ -400,21 +400,21 @@ AVFilterChannelLayouts *ff_all_channel_counts(void)
     return ret;
 }
 
-#define FORMATS_REF(f, ref)                                          \
-do {                                                                 \
-    *ref = f;                                                        \
-    f->refs = av_realloc(f->refs, sizeof(*f->refs) * ++f->refcount); \
-    if (!f->refs)                                                    \
-        return;                                                      \
-    f->refs[f->refcount-1] = ref;                                    \
-} while (0)
-
-void ff_channel_layouts_ref(AVFilterChannelLayouts *f, AVFilterChannelLayouts **ref)
+#define FORMATS_REF(f, ref)                                                     \
+    void *tmp = av_realloc_array(f->refs, sizeof(*f->refs), f->refcount + 1);   \
+    if (!tmp)                                                                   \
+        return AVERROR(ENOMEM);                                                 \
+    f->refs = tmp;                                                              \
+    f->refs[f->refcount++] = ref;                                               \
+    *ref = f;                                                                   \
+    return 0
+
+int ff_channel_layouts_ref(AVFilterChannelLayouts *f, AVFilterChannelLayouts **ref)
 {
     FORMATS_REF(f, ref);
 }
 
-void ff_formats_ref(AVFilterFormats *f, AVFilterFormats **ref)
+int ff_formats_ref(AVFilterFormats *f, AVFilterFormats **ref)
 {
     FORMATS_REF(f, ref);
 }

libavfilter/formats.h

@@ -159,8 +159,8 @@ int ff_add_channel_layout(AVFilterChannelLayouts **l, uint64_t channel_layout);
 /**
  * Add *ref as a new reference to f.
  */
-void ff_channel_layouts_ref(AVFilterChannelLayouts *f,
-                            AVFilterChannelLayouts **ref);
+int ff_channel_layouts_ref(AVFilterChannelLayouts *f,
+                           AVFilterChannelLayouts **ref);
 
 /**
  * Remove a reference to a channel layouts list.
@@ -233,7 +233,7 @@ AVFilterFormats *ff_merge_formats(AVFilterFormats *a, AVFilterFormats *b,
  *  | |____| |    | |____|
  *  |________|    |________________________
  */
-void ff_formats_ref(AVFilterFormats *formats, AVFilterFormats **ref);
+int ff_formats_ref(AVFilterFormats *formats, AVFilterFormats **ref);
 
 /**
  * If *ref is non-NULL, remove *ref as a reference to the format list