Merge pull request #10154 from alalek:ocl_cleanup_obsolete_cache

modules/core/include/opencv2/core/utils/filesystem.hpp

@@ -11,9 +11,42 @@ namespace cv { namespace utils { namespace fs {
 CV_EXPORTS bool exists(const cv::String& path);
 CV_EXPORTS bool isDirectory(const cv::String& path);
 
+CV_EXPORTS void remove_all(const cv::String& path);
+
 
 CV_EXPORTS cv::String getcwd();
 
+/** Join path components */
+CV_EXPORTS cv::String join(const cv::String& base, const cv::String& path);
+
+/**
+ * Generate a list of all files that match the globbing pattern.
+ *
+ * Result entries are prefixed by base directory path.
+ *
+ * @param directory base directory
+ * @param pattern filter pattern (based on '*'/'?' symbols). Use empty string to disable filtering and return all results
+ * @param[out] result result of globing.
+ * @param recursive scan nested directories too
+ * @param includeDirectories include directories into results list
+ */
+CV_EXPORTS void glob(const cv::String& directory, const cv::String& pattern,
+        CV_OUT std::vector<cv::String>& result,
+        bool recursive = false, bool includeDirectories = false);
+
+/**
+ * Generate a list of all files that match the globbing pattern.
+ *
+ * @param directory base directory
+ * @param pattern filter pattern (based on '*'/'?' symbols). Use empty string to disable filtering and return all results
+ * @param[out] result globbing result with relative paths from base directory
+ * @param recursive scan nested directories too
+ * @param includeDirectories include directories into results list
+ */
+CV_EXPORTS void glob_relative(const cv::String& directory, const cv::String& pattern,
+        CV_OUT std::vector<cv::String>& result,
+        bool recursive = false, bool includeDirectories = false);
+
 
 CV_EXPORTS bool createDirectory(const cv::String& path);
 CV_EXPORTS bool createDirectories(const cv::String& path);

modules/core/include/opencv2/core/utils/logger.hpp

@@ -58,9 +58,9 @@ enum LogLevel {
 #endif
 
 
-#define CV_LOG_FATAL(tag, ...)   for(;;) { std::stringstream ss; ss << "[FATAL:" << cv::utils::getThreadID() << "] " << __VA_ARGS__ << std::endl; std::cerr << ss.str(); break; }
-#define CV_LOG_ERROR(tag, ...)   for(;;) { std::stringstream ss; ss << "[ERROR:" << cv::utils::getThreadID() << "] " << __VA_ARGS__ << std::endl; std::cerr << ss.str(); break; }
-#define CV_LOG_WARNING(tag, ...) for(;;) { std::stringstream ss; ss << "[ WARN:" << cv::utils::getThreadID() << "] " << __VA_ARGS__ << std::endl; std::cout << ss.str(); break; }
+#define CV_LOG_FATAL(tag, ...)   for(;;) { std::stringstream ss; ss << "[FATAL:" << cv::utils::getThreadID() << "] " << __VA_ARGS__ << std::endl; std::cerr << ss.str() << std::flush; break; }
+#define CV_LOG_ERROR(tag, ...)   for(;;) { std::stringstream ss; ss << "[ERROR:" << cv::utils::getThreadID() << "] " << __VA_ARGS__ << std::endl; std::cerr << ss.str() << std::flush; break; }
+#define CV_LOG_WARNING(tag, ...) for(;;) { std::stringstream ss; ss << "[ WARN:" << cv::utils::getThreadID() << "] " << __VA_ARGS__ << std::endl; std::cout << ss.str() << std::flush; break; }
 #if CV_LOG_STRIP_LEVEL <= CV_LOG_LEVEL_INFO
 #define CV_LOG_INFO(tag, ...)
 #else

modules/core/src/glob.cpp

@@ -47,7 +47,6 @@
 #if defined _WIN32 || defined WINCE
 # include <windows.h>
 const char dir_separators[] = "/\\";
-const char native_separator = '\\';
 
 namespace
 {
@@ -136,7 +135,6 @@ namespace
 # include <dirent.h>
 # include <sys/stat.h>
 const char dir_separators[] = "/";
-const char native_separator = '/';
 #endif
 
 static bool isDir(const cv::String& path, DIR* dir)
@@ -225,34 +223,36 @@ static bool wildcmp(const char *string, const char *wild)
     return *wild == 0;
 }
 
-static void glob_rec(const cv::String& directory, const cv::String& wildchart, std::vector<cv::String>& result, bool recursive)
+static void glob_rec(const cv::String& directory, const cv::String& wildchart, std::vector<cv::String>& result,
+        bool recursive, bool includeDirectories, const cv::String& pathPrefix)
 {
     DIR *dir;
-    struct dirent *ent;
 
     if ((dir = opendir (directory.c_str())) != 0)
     {
         /* find all the files and directories within directory */
         try
         {
+            struct dirent *ent;
             while ((ent = readdir (dir)) != 0)
             {
                 const char* name = ent->d_name;
                 if((name[0] == 0) || (name[0] == '.' && name[1] == 0) || (name[0] == '.' && name[1] == '.' && name[2] == 0))
                     continue;
 
-                cv::String path = directory + native_separator + name;
+                cv::String path = cv::utils::fs::join(directory, name);
+                cv::String entry = cv::utils::fs::join(pathPrefix, name);
 
                 if (isDir(path, dir))
                 {
                     if (recursive)
-                        glob_rec(path, wildchart, result, recursive);
+                        glob_rec(path, wildchart, result, recursive, includeDirectories, entry);
+                    if (!includeDirectories)
+                        continue;
                 }
-                else
-                {
+
                 if (wildchart.empty() || wildcmp(name, wildchart.c_str()))
-                        result.push_back(path);
-                }
+                    result.push_back(entry);
             }
         }
         catch (...)
@@ -262,7 +262,10 @@ static void glob_rec(const cv::String& directory, const cv::String& wildchart, s
         }
         closedir(dir);
     }
-    else CV_Error(CV_StsObjectNotFound, cv::format("could not open directory: %s", directory.c_str()));
+    else
+    {
+        CV_Error_(CV_StsObjectNotFound, ("could not open directory: %s", directory.c_str()));
+    }
 }
 
 void cv::glob(String pattern, std::vector<String>& result, bool recursive)
@@ -298,6 +301,22 @@ void cv::glob(String pattern, std::vector<String>& result, bool recursive)
         }
     }
 
-    glob_rec(path, wildchart, result, recursive);
+    glob_rec(path, wildchart, result, recursive, false, path);
+    std::sort(result.begin(), result.end());
+}
+
+void cv::utils::fs::glob(const cv::String& directory, const cv::String& pattern,
+        std::vector<cv::String>& result,
+        bool recursive, bool includeDirectories)
+{
+    glob_rec(directory, pattern, result, recursive, includeDirectories, directory);
+    std::sort(result.begin(), result.end());
+}
+
+void cv::utils::fs::glob_relative(const cv::String& directory, const cv::String& pattern,
+        std::vector<cv::String>& result,
+        bool recursive, bool includeDirectories)
+{
+    glob_rec(directory, pattern, result, recursive, includeDirectories, cv::String());
     std::sort(result.begin(), result.end());
 }

modules/core/src/ocl.cpp

@@ -180,6 +180,7 @@ void traceOpenCLCheck(cl_int status, const char* message)
 static const bool CV_OPENCL_CACHE_ENABLE = utils::getConfigurationParameterBool("OPENCV_OPENCL_CACHE_ENABLE", true);
 static const bool CV_OPENCL_CACHE_WRITE = utils::getConfigurationParameterBool("OPENCV_OPENCL_CACHE_WRITE", true);
 static const bool CV_OPENCL_CACHE_LOCK_ENABLE = utils::getConfigurationParameterBool("OPENCV_OPENCL_CACHE_LOCK_ENABLE", true);
+static const bool CV_OPENCL_CACHE_CLEANUP = utils::getConfigurationParameterBool("OPENCV_OPENCL_CACHE_CLEANUP", true);
 
 #if CV_OPENCL_VALIDATE_BINARY_PROGRAMS
 static const bool CV_OPENCL_VALIDATE_BINARY_PROGRAMS_VALUE = utils::getConfigurationParameterBool("OPENCV_OPENCL_VALIDATE_BINARY_PROGRAMS", false);
@@ -254,6 +255,7 @@ struct OpenCLBinaryCacheConfigurator
 
     typedef std::map<std::string, std::string> ContextCacheType;
     ContextCacheType prepared_contexts_;
+    Mutex mutex_prepared_contexts_;
 
     OpenCLBinaryCacheConfigurator()
     {
@@ -355,14 +357,17 @@ struct OpenCLBinaryCacheConfigurator
         cache_lock_.release();
     }
 
-    std::string prepareCacheDirectoryForContext(const std::string& ctx_prefix)
+    std::string prepareCacheDirectoryForContext(const std::string& ctx_prefix,
+            const std::string& cleanup_prefix)
     {
         if (cache_path_.empty())
             return std::string();
 
-        ContextCacheType::iterator i = prepared_contexts_.find(ctx_prefix);
-        if (i != prepared_contexts_.end())
-            return i->second;
+        AutoLock lock(mutex_prepared_contexts_);
+
+        ContextCacheType::iterator found_it = prepared_contexts_.find(ctx_prefix);
+        if (found_it != prepared_contexts_.end())
+            return found_it->second;
 
         CV_LOG_INFO(NULL, "Preparing OpenCL cache configuration for context: " << ctx_prefix);
 
@@ -390,8 +395,59 @@ struct OpenCLBinaryCacheConfigurator
         target_directory = result ? target_directory : std::string();
         prepared_contexts_.insert(std::pair<std::string, std::string>(ctx_prefix, target_directory));
 
-        CV_LOG_VERBOSE(NULL, 1, "  Result: " << (target_directory.empty() ? std::string("Failed") : target_directory));
+        if (result && CV_OPENCL_CACHE_CLEANUP && CV_OPENCL_CACHE_WRITE && !cleanup_prefix.empty())
+        {
+            try
+            {
+                std::vector<String> entries;
+                utils::fs::glob_relative(cache_path_, cleanup_prefix + "*", entries, false, true);
+                std::vector<String> remove_entries;
+                for (size_t i = 0; i < entries.size(); i++)
+                {
+                    const String& name = entries[i];
+                    if (0 == name.find(cleanup_prefix))
+                    {
+                        if (0 == name.find(ctx_prefix))
+                            continue; // skip current
+                        remove_entries.push_back(name);
+                    }
+                }
+                if (!remove_entries.empty())
+                {
+                    CV_LOG_WARNING(NULL, (remove_entries.size() == 1
+                            ? "Detected OpenCL cache directory for other version of OpenCL device."
+                            : "Detected OpenCL cache directories for other versions of OpenCL device.")
+                            << " We assume that these directories are obsolete after OpenCL runtime/drivers upgrade.");
+                    CV_LOG_WARNING(NULL, "Trying to remove these directories...");
+                    for (size_t i = 0; i < remove_entries.size(); i++)
+                    {
+                        CV_LOG_WARNING(NULL, "- " << remove_entries[i]);
+                    }
+                    CV_LOG_WARNING(NULL,"Note: You can disable this behavior via this option: CV_OPENCL_CACHE_CLEANUP=0");
 
+                    for (size_t i = 0; i < remove_entries.size(); i++)
+                    {
+                        const String& name = remove_entries[i];
+                        cv::String path = utils::fs::join(cache_path_, name);
+                        try
+                        {
+                            utils::fs::remove_all(path);
+                            CV_LOG_WARNING(NULL, "Removed: " << path);
+                        }
+                        catch (const cv::Exception& e)
+                        {
+                            CV_LOG_ERROR(NULL, "Exception during removal of obsolete OpenCL cache directory: " << path << std::endl << e.what());
+                        }
+                    }
+                }
+            }
+            catch (...)
+            {
+                CV_LOG_WARNING(NULL, "Can't check for obsolete OpenCL cache directories");
+            }
+        }
+
+        CV_LOG_VERBOSE(NULL, 1, "  Result: " << (target_directory.empty() ? std::string("Failed") : target_directory));
         return target_directory;
     }
 
@@ -1969,7 +2025,7 @@ struct Context::Impl
         }
     }
 
-    std::string getPrefixString()
+    std::string& getPrefixString()
     {
         if (prefix.empty())
         {
@@ -1988,12 +2044,32 @@ struct Context::Impl
         return prefix;
     }
 
+    std::string& getPrefixBase()
+    {
+        if (prefix_base.empty())
+        {
+            const Device& d = devices[0];
+            prefix_base = d.vendorName() + "--" + d.name() + "--";
+            // sanitize chars
+            for (size_t i = 0; i < prefix_base.size(); i++)
+            {
+                char c = prefix_base[i];
+                if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || c == '-'))
+                {
+                    prefix_base[i] = '_';
+                }
+            }
+        }
+        return prefix_base;
+    }
+
     IMPLEMENT_REFCOUNTABLE();
 
     cl_context handle;
     std::vector<Device> devices;
 
     std::string prefix;
+    std::string prefix_base;
 
     cv::Mutex program_cache_mutex;
     typedef std::map<std::string, Program> phash_t;
@@ -3233,7 +3309,10 @@ struct Program::Impl
     {
 #if OPENCV_HAVE_FILESYSTEM_SUPPORT
         OpenCLBinaryCacheConfigurator& config = OpenCLBinaryCacheConfigurator::getSingletonInstance();
-        const std::string base_dir = config.prepareCacheDirectoryForContext(ctx.getImpl()->getPrefixString());
+        const std::string base_dir = config.prepareCacheDirectoryForContext(
+                ctx.getImpl()->getPrefixString(),
+                ctx.getImpl()->getPrefixBase()
+        );
         const std::string fname = base_dir.empty() ? std::string() :
                 std::string(base_dir + src.getImpl()->module_.c_str() + "--" + src.getImpl()->name_ + "_" + src.getImpl()->codeHash_ + ".bin");
         const cv::Ptr<utils::fs::FileLock> fileLock = config.cache_lock_; // can be empty

modules/core/src/utils/filesystem.cpp

@@ -31,6 +31,8 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <errno.h>
+#include <io.h>
+#include <stdio.h>
 #elif defined __linux__ || defined __APPLE__
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -41,12 +43,43 @@
 
 namespace cv { namespace utils { namespace fs {
 
+#ifdef _WIN32
+static const char native_separator = '\\';
+#else
+static const char native_separator = '/';
+#endif
+
 static inline
 bool isPathSeparator(char c)
 {
     return c == '/' || c == '\\';
 }
 
+cv::String join(const cv::String& base, const cv::String& path)
+{
+    if (base.empty())
+        return path;
+    if (path.empty())
+        return base;
+
+    bool baseSep = isPathSeparator(base[base.size() - 1]);
+    bool pathSep = isPathSeparator(path[0]);
+    String result;
+    if (baseSep && pathSep)
+    {
+        result = base + path.substr(1);
+    }
+    else if (!baseSep && !pathSep)
+    {
+        result = base + native_separator + path;
+    }
+    else
+    {
+        result = base + path;
+    }
+    return result;
+}
+
 bool exists(const cv::String& path)
 {
     CV_INSTRUMENT_REGION()
@@ -72,6 +105,44 @@ bool exists(const cv::String& path)
 #endif
 }
 
+CV_EXPORTS void remove_all(const cv::String& path)
+{
+    if (!exists(path))
+        return;
+    if (isDirectory(path))
+    {
+        std::vector<String> entries;
+        utils::fs::glob(path, cv::String(), entries, false, true);
+        for (size_t i = 0; i < entries.size(); i++)
+        {
+            const String& e = entries[i];
+            remove_all(e);
+        }
+#ifdef _MSC_VER
+        bool result = _rmdir(path.c_str()) == 0;
+#else
+        bool result = rmdir(path.c_str()) == 0;
+#endif
+        if (!result)
+        {
+            CV_LOG_ERROR(NULL, "Can't remove directory: " << path);
+        }
+    }
+    else
+    {
+#ifdef _MSC_VER
+        bool result = _unlink(path.c_str()) == 0;
+#else
+        bool result = unlink(path.c_str()) == 0;
+#endif
+        if (!result)
+        {
+            CV_LOG_ERROR(NULL, "Can't remove file: " << path);
+        }
+    }
+}
+
+
 cv::String getcwd()
 {
     CV_INSTRUMENT_REGION()
@@ -138,7 +209,7 @@ bool createDirectories(const cv::String& path_)
     for (;;)
     {
         char last_char = path.empty() ? 0 : path[path.length() - 1];
-        if (last_char == '/' || last_char == '\\')
+        if (isPathSeparator(last_char))
         {
             path = path.substr(0, path.length() - 1);
             continue;
@@ -364,7 +435,7 @@ cv::String getCacheDirectory(const char* sub_directory_name, const char* configu
             if (home_env && home_env[0] && utils::fs::isDirectory(home_env))
             {
                 cv::String home_path = home_env;
-                cv::String home_cache_path = home_path + "/.cache/";
+                cv::String home_cache_path = utils::fs::join(home_path, ".cache/");
                 if (utils::fs::isDirectory(home_cache_path))
                 {
                     default_cache_path = home_cache_path;
@@ -393,9 +464,9 @@ cv::String getCacheDirectory(const char* sub_directory_name, const char* configu
         {
             if (utils::fs::isDirectory(default_cache_path))
             {
-                default_cache_path += "/opencv/" CV_VERSION "/";
+                default_cache_path = utils::fs::join(default_cache_path, utils::fs::join("opencv", CV_VERSION));
                 if (sub_directory_name && sub_directory_name[0] != '\0')
-                    default_cache_path += cv::String(sub_directory_name) + "/";
+                    default_cache_path = utils::fs::join(default_cache_path, cv::String(sub_directory_name) + native_separator);
                 if (!utils::fs::createDirectories(default_cache_path))
                 {
                     CV_LOG_DEBUG(NULL, "Can't create OpenCV cache sub-directory: " << default_cache_path);
@@ -434,7 +505,7 @@ cv::String getCacheDirectory(const char* sub_directory_name, const char* configu
     {
         if (!isPathSeparator(cache_path[cache_path.size() - 1]))
         {
-            cache_path += '/';
+            cache_path += native_separator;
         }
     }
     return cache_path;