avformat/format: Fix registering a format more than once and related races

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit 4cc896ea)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavformat/format.c

@@ -62,20 +62,24 @@ void av_register_input_format(AVInputFormat *format)
 {
     AVInputFormat **p = last_iformat;
 
-    format->next = NULL;
-    while(*p || avpriv_atomic_ptr_cas((void * volatile *)p, NULL, format))
+    // Note, format could be added after the first 2 checks but that implies that *p is no longer NULL
+    while(p != &format->next && !format->next && avpriv_atomic_ptr_cas((void * volatile *)p, NULL, format))
         p = &(*p)->next;
-    last_iformat = &format->next;
+
+    if (!format->next)
+        last_iformat = &format->next;
 }
 
 void av_register_output_format(AVOutputFormat *format)
 {
     AVOutputFormat **p = last_oformat;
 
-    format->next = NULL;
-    while(*p || avpriv_atomic_ptr_cas((void * volatile *)p, NULL, format))
+    // Note, format could be added after the first 2 checks but that implies that *p is no longer NULL
+    while(p != &format->next && !format->next && avpriv_atomic_ptr_cas((void * volatile *)p, NULL, format))
         p = &(*p)->next;
-    last_oformat = &format->next;
+
+    if (!format->next)
+        last_oformat = &format->next;
 }
 
 int av_match_ext(const char *filename, const char *extensions)