lavu: add Vulkan hwcontext code

This commit adds the necessary code to initialize and use a Vulkan device
within the hwcontext libavutil framework.
Currently direct mapping to VAAPI and DRM frames is functional, and
transfers to CUDA and native frames are supported.

Lets hope the future Vulkan video decode extension fits well within this
framework.

configure

@@ -316,6 +316,7 @@ External library support:
   --disable-securetransport disable Secure Transport, needed for TLS support
                            on OSX if openssl and gnutls are not used [autodetect]
   --enable-vapoursynth     enable VapourSynth demuxer [no]
+  --enable-vulkan          enable Vulkan code [no]
   --disable-xlib           disable xlib [autodetect]
   --disable-zlib           disable zlib [autodetect]
 
@@ -1854,6 +1855,7 @@ HWACCEL_LIBRARY_LIST="
     mmal
     omx
     opencl
+    vulkan
 "
 
 DOCUMENT_LIST="
@@ -3641,7 +3643,7 @@ avformat_deps="avcodec avutil"
 avformat_suggest="libm network zlib"
 avresample_deps="avutil"
 avresample_suggest="libm"
-avutil_suggest="clock_gettime ffnvcodec libm libdrm libmfx opencl user32 vaapi videotoolbox corefoundation corevideo coremedia bcrypt"
+avutil_suggest="clock_gettime ffnvcodec libm libdrm libmfx opencl user32 vaapi vulkan videotoolbox corefoundation corevideo coremedia bcrypt"
 postproc_deps="avutil gpl"
 postproc_suggest="libm"
 swresample_deps="avutil"
@@ -6629,6 +6631,9 @@ enabled vdpau &&
 
 enabled crystalhd && check_lib crystalhd "stdint.h libcrystalhd/libcrystalhd_if.h" DtsCrystalHDVersion -lcrystalhd
 
+enabled vulkan &&
+    require_pkg_config vulkan "vulkan >= 1.1.97" "vulkan/vulkan.h" vkCreateInstance
+
 if enabled x86; then
     case $target_os in
         mingw32*|mingw64*|win32|win64|linux|cygwin*)

doc/APIchanges

@@ -15,6 +15,10 @@ libavutil:     2017-10-21
 
 API changes, most recent first:
 
+2020-ww-xx - xxxxxxxxxx - lavu yy.yy.yyy - hwcontext.h
+  Add AV_PIX_FMT_VULKAN
+  Add AV_HWDEVICE_TYPE_VULKAN and implementation.
+
 2020-01-30 - xxxxxxxxxx - lavf 58.37.100 - avio.h
   Add avio_protocol_get_class().
 

libavutil/Makefile

@@ -43,6 +43,7 @@ HEADERS = adler32.h                                                     \
           hwcontext_vaapi.h                                             \
           hwcontext_videotoolbox.h                                      \
           hwcontext_vdpau.h                                             \
+          hwcontext_vulkan.h                                            \
           imgutils.h                                                    \
           intfloat.h                                                    \
           intreadwrite.h                                                \
@@ -175,6 +176,7 @@ OBJS-$(CONFIG_QSV)                      += hwcontext_qsv.o
 OBJS-$(CONFIG_VAAPI)                    += hwcontext_vaapi.o
 OBJS-$(CONFIG_VIDEOTOOLBOX)             += hwcontext_videotoolbox.o
 OBJS-$(CONFIG_VDPAU)                    += hwcontext_vdpau.o
+OBJS-$(CONFIG_VULKAN)                   += hwcontext_vulkan.o
 
 OBJS += $(COMPAT_OBJS:%=../compat/%)
 
@@ -191,6 +193,7 @@ SKIPHEADERS-$(CONFIG_OPENCL)           += hwcontext_opencl.h
 SKIPHEADERS-$(CONFIG_VAAPI)            += hwcontext_vaapi.h
 SKIPHEADERS-$(CONFIG_VIDEOTOOLBOX)     += hwcontext_videotoolbox.h
 SKIPHEADERS-$(CONFIG_VDPAU)            += hwcontext_vdpau.h
+SKIPHEADERS-$(CONFIG_VULKAN)           += hwcontext_vulkan.h
 
 TESTPROGS = adler32                                                     \
             aes                                                         \

libavutil/hwcontext.c

@@ -58,6 +58,9 @@ static const HWContextType * const hw_table[] = {
 #endif
 #if CONFIG_MEDIACODEC
     &ff_hwcontext_type_mediacodec,
+#endif
+#if CONFIG_VULKAN
+    &ff_hwcontext_type_vulkan,
 #endif
     NULL,
 };
@@ -73,6 +76,7 @@ static const char *const hw_type_names[] = {
     [AV_HWDEVICE_TYPE_VDPAU]  = "vdpau",
     [AV_HWDEVICE_TYPE_VIDEOTOOLBOX] = "videotoolbox",
     [AV_HWDEVICE_TYPE_MEDIACODEC] = "mediacodec",
+    [AV_HWDEVICE_TYPE_VULKAN] = "vulkan",
 };
 
 enum AVHWDeviceType av_hwdevice_find_type_by_name(const char *name)

libavutil/hwcontext.h

@@ -36,6 +36,7 @@ enum AVHWDeviceType {
     AV_HWDEVICE_TYPE_DRM,
     AV_HWDEVICE_TYPE_OPENCL,
     AV_HWDEVICE_TYPE_MEDIACODEC,
+    AV_HWDEVICE_TYPE_VULKAN,
 };
 
 typedef struct AVHWDeviceInternal AVHWDeviceInternal;

libavutil/hwcontext_cuda.c

@@ -21,6 +21,9 @@
 #include "hwcontext.h"
 #include "hwcontext_internal.h"
 #include "hwcontext_cuda_internal.h"
+#if CONFIG_VULKAN
+#include "hwcontext_vulkan.h"
+#endif
 #include "cuda_check.h"
 #include "mem.h"
 #include "pixdesc.h"
@@ -42,6 +45,9 @@ static const enum AVPixelFormat supported_formats[] = {
     AV_PIX_FMT_YUV444P16,
     AV_PIX_FMT_0RGB32,
     AV_PIX_FMT_0BGR32,
+#if CONFIG_VULKAN
+    AV_PIX_FMT_VULKAN,
+#endif
 };
 
 #define CHECK_CU(x) FF_CUDA_CHECK_DL(device_ctx, cu, x)
@@ -205,6 +211,10 @@ static int cuda_transfer_data_from(AVHWFramesContext *ctx, AVFrame *dst,
     CUcontext dummy;
     int i, ret;
 
+    /* We don't support transfers to HW devices. */
+    if (dst->hw_frames_ctx)
+        return AVERROR(ENOSYS);
+
     ret = CHECK_CU(cu->cuCtxPushCurrent(hwctx->cuda_ctx));
     if (ret < 0)
         return ret;
@@ -247,6 +257,10 @@ static int cuda_transfer_data_to(AVHWFramesContext *ctx, AVFrame *dst,
     CUcontext dummy;
     int i, ret;
 
+    /* We don't support transfers from HW devices. */
+    if (src->hw_frames_ctx)
+        return AVERROR(ENOSYS);
+
     ret = CHECK_CU(cu->cuCtxPushCurrent(hwctx->cuda_ctx));
     if (ret < 0)
         return ret;
@@ -389,6 +403,123 @@ error:
     return AVERROR_UNKNOWN;
 }
 
+static int cuda_device_derive(AVHWDeviceContext *device_ctx,
+                              AVHWDeviceContext *src_ctx,
+                              int flags) {
+    AVCUDADeviceContext *hwctx = device_ctx->hwctx;
+    CudaFunctions *cu;
+    const char *src_uuid = NULL;
+    CUcontext dummy;
+    int ret, i, device_count, dev_active = 0;
+    unsigned int dev_flags = 0;
+
+    const unsigned int desired_flags = CU_CTX_SCHED_BLOCKING_SYNC;
+
+#if CONFIG_VULKAN
+    VkPhysicalDeviceIDProperties vk_idp = {
+        .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES,
+    };
+#endif
+
+    switch (src_ctx->type) {
+#if CONFIG_VULKAN
+    case AV_HWDEVICE_TYPE_VULKAN: {
+        AVVulkanDeviceContext *vkctx = src_ctx->hwctx;
+        VkPhysicalDeviceProperties2 vk_dev_props = {
+            .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2,
+            .pNext = &vk_idp,
+        };
+        vkGetPhysicalDeviceProperties2(vkctx->phys_dev, &vk_dev_props);
+        src_uuid = vk_idp.deviceUUID;
+        break;
+    }
+#endif
+    default:
+        return AVERROR(ENOSYS);
+    }
+
+    if (!src_uuid) {
+        av_log(device_ctx, AV_LOG_ERROR,
+               "Failed to get UUID of source device.\n");
+        goto error;
+    }
+
+    if (cuda_device_init(device_ctx) < 0)
+        goto error;
+
+    cu = hwctx->internal->cuda_dl;
+
+    ret = CHECK_CU(cu->cuInit(0));
+    if (ret < 0)
+        goto error;
+
+    ret = CHECK_CU(cu->cuDeviceGetCount(&device_count));
+    if (ret < 0)
+        goto error;
+
+    hwctx->internal->cuda_device = -1;
+    for (i = 0; i < device_count; i++) {
+        CUdevice dev;
+        CUuuid uuid;
+
+        ret = CHECK_CU(cu->cuDeviceGet(&dev, i));
+        if (ret < 0)
+            goto error;
+
+        ret = CHECK_CU(cu->cuDeviceGetUuid(&uuid, dev));
+        if (ret < 0)
+            goto error;
+
+        if (memcmp(src_uuid, uuid.bytes, sizeof (uuid.bytes)) == 0) {
+            hwctx->internal->cuda_device = dev;
+            break;
+        }
+    }
+
+    if (hwctx->internal->cuda_device == -1) {
+        av_log(device_ctx, AV_LOG_ERROR, "Could not derive CUDA device.\n");
+        goto error;
+    }
+
+    hwctx->internal->flags = flags;
+
+    if (flags & AV_CUDA_USE_PRIMARY_CONTEXT) {
+        ret = CHECK_CU(cu->cuDevicePrimaryCtxGetState(hwctx->internal->cuda_device, &dev_flags, &dev_active));
+        if (ret < 0)
+            goto error;
+
+        if (dev_active && dev_flags != desired_flags) {
+            av_log(device_ctx, AV_LOG_ERROR, "Primary context already active with incompatible flags.\n");
+            goto error;
+        } else if (dev_flags != desired_flags) {
+            ret = CHECK_CU(cu->cuDevicePrimaryCtxSetFlags(hwctx->internal->cuda_device, desired_flags));
+            if (ret < 0)
+                goto error;
+        }
+
+        ret = CHECK_CU(cu->cuDevicePrimaryCtxRetain(&hwctx->cuda_ctx, hwctx->internal->cuda_device));
+        if (ret < 0)
+            goto error;
+    } else {
+        ret = CHECK_CU(cu->cuCtxCreate(&hwctx->cuda_ctx, desired_flags, hwctx->internal->cuda_device));
+        if (ret < 0)
+            goto error;
+
+        CHECK_CU(cu->cuCtxPopCurrent(&dummy));
+    }
+
+    hwctx->internal->is_allocated = 1;
+
+    // Setting stream to NULL will make functions automatically use the default CUstream
+    hwctx->stream = NULL;
+
+    return 0;
+
+error:
+    cuda_device_uninit(device_ctx);
+    return AVERROR_UNKNOWN;
+}
+
 const HWContextType ff_hwcontext_type_cuda = {
     .type                 = AV_HWDEVICE_TYPE_CUDA,
     .name                 = "CUDA",
@@ -397,6 +528,7 @@ const HWContextType ff_hwcontext_type_cuda = {
     .frames_priv_size     = sizeof(CUDAFramesContext),
 
     .device_create        = cuda_device_create,
+    .device_derive        = cuda_device_derive,
     .device_init          = cuda_device_init,
     .device_uninit        = cuda_device_uninit,
     .frames_get_constraints = cuda_frames_get_constraints,

libavutil/hwcontext_internal.h

@@ -172,5 +172,6 @@ extern const HWContextType ff_hwcontext_type_vaapi;
 extern const HWContextType ff_hwcontext_type_vdpau;
 extern const HWContextType ff_hwcontext_type_videotoolbox;
 extern const HWContextType ff_hwcontext_type_mediacodec;
+extern const HWContextType ff_hwcontext_type_vulkan;
 
 #endif /* AVUTIL_HWCONTEXT_INTERNAL_H */

libavutil/hwcontext_vulkan.h

+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef AVUTIL_HWCONTEXT_VULKAN_H
+#define AVUTIL_HWCONTEXT_VULKAN_H
+
+#include <vulkan/vulkan.h>
+
+/**
+ * @file
+ * API-specific header for AV_HWDEVICE_TYPE_VULKAN.
+ *
+ * For user-allocated pools, AVHWFramesContext.pool must return AVBufferRefs
+ * with the data pointer set to an AVVkFrame.
+ */
+
+/**
+ * Main Vulkan context, allocated as AVHWDeviceContext.hwctx.
+ * All of these can be set before init to change what the context uses
+ */
+typedef struct AVVulkanDeviceContext {
+    /**
+     * Custom memory allocator, else NULL
+     */
+    const VkAllocationCallbacks *alloc;
+    /**
+     * Instance
+     */
+    VkInstance inst;
+    /**
+     * Physical device
+     */
+    VkPhysicalDevice phys_dev;
+    /**
+     * Active device
+     */
+    VkDevice act_dev;
+    /**
+     * Queue family index for graphics
+     * @note av_hwdevice_create() will set all 3 queue indices if unset
+     * If there is no dedicated queue for compute or transfer operations,
+     * they will be set to the graphics queue index which can handle both.
+     */
+    int queue_family_index;
+    /**
+     * Queue family index for transfer ops only
+     */
+    int queue_family_tx_index;
+    /**
+     * Queue family index for compute ops
+     */
+    int queue_family_comp_index;
+} AVVulkanDeviceContext;
+
+/**
+ * Allocated as AVHWFramesContext.hwctx, used to set pool-specific options
+ */
+typedef struct AVVulkanFramesContext {
+    /**
+     * Controls the tiling of output frames.
+     */
+    VkImageTiling tiling;
+    /**
+     * Defines extra usage of output frames. This is bitwise OR'd with the
+     * standard usage flags (SAMPLED, STORAGE, TRANSFER_SRC and TRANSFER_DST).
+     */
+    VkImageUsageFlagBits usage;
+    /**
+     * Extension data for image creation. By default, if the extension is
+     * available, this will be chained to a VkImageFormatListCreateInfoKHR.
+     */
+    void *create_pnext;
+    /**
+     * Extension data for memory allocation. Must have as many entries as
+     * the number of planes of the sw_format.
+     * This will be chained to VkExportMemoryAllocateInfo, which is used
+     * to make all pool images exportable to other APIs.
+     */
+    void *alloc_pnext[AV_NUM_DATA_POINTERS];
+} AVVulkanFramesContext;
+
+/*
+ * Frame structure, the VkFormat of the image will always match
+ * the pool's sw_format.
+ * All frames, imported or allocated, will be created with the
+ * VK_IMAGE_CREATE_ALIAS_BIT flag set, so the memory may be aliased if needed.
+ *
+ * @note the size of this structure is not part of the ABI, to allocate
+ * you must use @av_vk_frame_alloc().
+ */
+typedef struct AVVkFrame {
+    /**
+     * Vulkan images to which the memory is bound to.
+     */
+    VkImage img[AV_NUM_DATA_POINTERS];
+
+    /**
+     * Same tiling must be used for all images.
+     */
+    VkImageTiling tiling;
+
+    /**
+     * Memory backing the images. Could be less than the amount of images
+     * if importing from a DRM or VAAPI frame.
+     */
+    VkDeviceMemory mem[AV_NUM_DATA_POINTERS];
+    size_t size[AV_NUM_DATA_POINTERS];
+
+    /**
+     * OR'd flags for all memory allocated
+     */
+    VkMemoryPropertyFlagBits flags;
+
+    /**
+     * Updated after every barrier
+     */
+    VkAccessFlagBits access[AV_NUM_DATA_POINTERS];
+    VkImageLayout layout[AV_NUM_DATA_POINTERS];
+
+    /**
+     * Per-image semaphores. Must not be freed manually. Must be waited on
+     * and signalled at every queue submission.
+     */
+    VkSemaphore sem[AV_NUM_DATA_POINTERS];
+
+    /**
+     * Internal data.
+     */
+    struct AVVkFrameInternal *internal;
+} AVVkFrame;
+
+/**
+ * Allocates a single AVVkFrame and initializes everything as 0.
+ * @note Must be freed via av_free()
+ */
+AVVkFrame *av_vk_frame_alloc(void);
+
+/**
+ * Returns the format of each image up to the number of planes for a given sw_format.
+ */
+const VkFormat *av_vkfmt_from_pixfmt(enum AVPixelFormat p);
+
+#endif /* AVUTIL_HWCONTEXT_VULKAN_H */

libavutil/pixdesc.c

@@ -2344,6 +2344,10 @@ static const AVPixFmtDescriptor av_pix_fmt_descriptors[AV_PIX_FMT_NB] = {
         },
         .flags = AV_PIX_FMT_FLAG_PLANAR,
     },
+    [AV_PIX_FMT_VULKAN] = {
+        .name = "vulkan",
+        .flags = AV_PIX_FMT_FLAG_HWACCEL,
+    },
 };
 #if FF_API_PLUS1_MINUS1
 FF_ENABLE_DEPRECATION_WARNINGS

libavutil/pixfmt.h

@@ -348,6 +348,13 @@ enum AVPixelFormat {
     AV_PIX_FMT_NV24,      ///< planar YUV 4:4:4, 24bpp, 1 plane for Y and 1 plane for the UV components, which are interleaved (first byte U and the following byte V)
     AV_PIX_FMT_NV42,      ///< as above, but U and V bytes are swapped
 
+    /**
+     * Vulkan hardware images.
+     *
+     * data[0] points to an AVVkFrame
+     */
+    AV_PIX_FMT_VULKAN,
+
     AV_PIX_FMT_NB         ///< number of pixel formats, DO NOT USE THIS if you want to link with shared libav* because the number of formats might differ between versions
 };