brisque_eval_tid2008.cpp 7.86 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243
#include <fstream>

#include "opencv2/quality.hpp"
#include "opencv2/imgproc.hpp"
#include "opencv2/imgcodecs.hpp"
#include "opencv2/ml.hpp"

/*
BRISQUE evaluator using TID2008

TID2008:
http://www.ponomarenko.info/tid2008.htm

[1] N. Ponomarenko, V. Lukin, A. Zelensky, K. Egiazarian, M. Carli,
F. Battisti, "TID2008 - A Database for Evaluation of Full-Reference
Visual Quality Assessment Metrics", Advances of Modern
Radioelectronics, Vol. 10, pp. 30-45, 2009.

[2] N. Ponomarenko, F. Battisti, K. Egiazarian, J. Astola,  V. Lukin
"Metrics performance comparison for color image database", Fourth
international workshop on video processing and quality metrics
for consumer electronics, Scottsdale, Arizona, USA. Jan. 14-16, 2009, 6 p.

*/

namespace {

    // get ordinal ranks of data, fractional ranks assigned for ties.  O(n^2) time complexity
    //  optional binary predicate used for rank ordering of data elements, equality evaluation
    template <typename T, typename PrEqual = std::equal_to<T>, typename PrLess = std::less<T>>
    std::vector<float> rank_ordinal(const T* data, std::size_t sz, PrEqual&& eq = {}, PrLess&& lt = {})
    {
        std::vector<float> result{};
        result.resize(sz, -1);// set all ranks to -1, indicating not yet done

        int rank = 0;
        while (rank < (int)sz)
        {
            std::vector<int> els = {};

            for (int i = 0; i < (int)sz; ++i)
            {
                if (result[i] < 0)//not yet done
                {
                    if (!els.empty())// already found something
                    {
                        if (lt(data[i], data[els[0]]))//found a smaller item, replace existing
                        {
                            els.clear();
                            els.emplace_back(i);
                        }
                        else if (eq(data[i], data[els[0]]))// found a tie, add to vector
                            els.emplace_back(i);
                    }
                    else//els.empty==no current item, add it
                        els.emplace_back(i);
                }
            }

            CV_Assert(!els.empty());

            // compute, assign arithmetic mean
            const auto assigned_rank = (double)rank + (double)(els.size() - 1) / 2.;
            for (auto el : els)
                result[el] = (float)assigned_rank;

            rank += (int)els.size();
        }

        return result;
    }

    template <typename T>
    double pearson(const T* x, const T* y, std::size_t sz)
    {
        // based on https://www.geeksforgeeks.org/program-spearmans-rank-correlation/

        double sigma_x = {}, sigma_y = {}, sigma_xy = {}, sigma_xsq = {}, sigma_ysq = {};
        for (unsigned i = 0; i < sz; ++i)
        {
            sigma_x += x[i];
            sigma_y += y[i];
            sigma_xy += x[i] * y[i];
            sigma_xsq += x[i] * x[i];
            sigma_ysq += y[i] * y[i];
        }

        const double
            num = (sz * sigma_xy - sigma_x * sigma_y)
            , den = std::sqrt(((double)sz*sigma_xsq - sigma_x * sigma_x) * ((double)sz*sigma_ysq - sigma_y * sigma_y))
            ;
        return num / den;
    }

    // https://en.wikipedia.org/wiki/Spearman%27s_rank_correlation_coefficient
    template <typename T>
    double spearman(const T* x, const T* y, std::size_t sz)
    {
        // convert x, y to ranked integral vectors
        const auto
            x_rank = rank_ordinal(x, sz)
            , y_rank = rank_ordinal(y, sz)
            ;

        return pearson(x_rank.data(), y_rank.data(), sz);
    }

    // returns cv::Mat of columns: { Distortion Type ID, MOS_Score, Brisque_Score }
    cv::Mat tid2008_eval(const std::string& root, cv::quality::QualityBRISQUE& alg)
    {
        const std::string
            mos_with_names_path = root + "mos_with_names.txt"
            , dist_imgs_root = root + "distorted_images/"
            ;

        cv::Mat result(0, 3, CV_32FC1);

        // distortion types we care about
        static const std::vector<int> distortion_types = {
            10 // jpeg compression
            , 11 // jp2k compression
            , 1 // additive gaussian noise
            , 8 // gaussian blur
        };

        static const int
            num_images = 25 // [I01_ - I25_], file names
            , num_distortions = 4 // num distortions per image
            ;

        // load mos_with_names.  format: { mos, fname }
        std::vector<std::pair<float, std::string>> mos_with_names = {};

        std::ifstream mos_file(mos_with_names_path, std::ios::in);
        while (true)
        {
            std::string line;
            std::getline(mos_file, line);
            if (!line.empty())
            {
                const auto space_pos = line.find(' ');
                CV_Assert(space_pos != line.npos);

                mos_with_names.emplace_back(std::make_pair(
                    (float)std::atof(line.substr(0, space_pos).c_str())
                    , line.substr(space_pos + 1)
                ));
            }

            if (mos_file.peek() == EOF)
                break;
        };

        // foreach image
        //  foreach distortion type
        //      foreach distortion level
        //          distortion type id, mos value, brisque value

        for (int i = 0; i < num_images; ++i)
        {
            for (int ty = 0; ty < (int)distortion_types.size(); ++ty)
            {
                for (int dist = 1; dist <= num_distortions; ++dist)
                {
                    float mos_val = 0.f;

                    const std::string img_name = std::string("i")
                        + (((i + 1) < 10) ? "0" : "")
                        + std::to_string(i + 1)
                        + "_"
                        + ((distortion_types[ty] < 10) ? "0" : "")
                        + std::to_string(distortion_types[ty])
                        + "_"
                        + std::to_string(dist)
                        + ".bmp";

                    // find mos
                    bool found = false;
                    for (const auto& val : mos_with_names)
                    {
                        if (val.second == img_name)
                        {
                            found = true;
                            mos_val = val.first;
                            break;
                        }

                    }

                    CV_Assert(found);

                    // do brisque
                    auto img = cv::imread(dist_imgs_root + img_name);

                    // typeid, mos, brisque
                    cv::Mat row(1, 3, CV_32FC1);
                    row.at<float>(0) = (float)distortion_types[ty];
                    row.at<float>(1) = mos_val;
                    row.at<float>(2) = (float)alg.compute(img)[0];
                    result.push_back(row);

                }// dist
            }//ty
        }//i

        return result;
    }
}

inline void printHelp()
{
    using namespace std;
    cout << "    Demo of comparing BRISQUE quality assessment model against TID2008 database." << endl;
    cout << "    A. Mittal, A. K. Moorthy and A. C. Bovik, 'No Reference Image Quality Assessment in the Spatial Domain'" << std::endl << std::endl;
    cout << "    Usage: program <tid2008_path> <brisque_model_path> <brisque_range_path>" << endl << endl;
}

int main(int argc, const char * argv[])
{
    using namespace cv::ml;

    if (argc != 4)
    {
        printHelp();
        exit(1);
    }

    std::cout << "Evaluating database at " << argv[1] << "..." << std::endl;

    const auto ptr = cv::quality::QualityBRISQUE::create(argv[2], argv[3]);

    const auto data = tid2008_eval( std::string( argv[1] ) + "/", *ptr );

    // create contiguous mats
    const auto mos = data.col(1).clone();
    const auto brisque = data.col(2).clone();

    // calc srocc
    const auto cc = spearman((const float*)mos.data, (const float*)brisque.data, data.rows);
    std::cout << "SROCC: " << cc << std::endl;

    return 0;
}