Added margin type, added tests with different scales of features.

Also fixed documentation, refactored sample.

modules/ml/include/opencv2/ml.hpp

@@ -1504,27 +1504,78 @@ public:
 /*!
 @brief Stochastic Gradient Descent SVM classifier
 
-SVMSGD provides a fast and easy-to-use implementation of the SVM classifier using the Stochastic Gradient Descent approach, as presented in @cite bottou2010large.
+SVMSGD provides a fast and easy-to-use implementation of the SVM classifier using the Stochastic Gradient Descent approach,
+as presented in @cite bottou2010large.
 The gradient descent show amazing performance for large-scale problems, reducing the computing time.
 
+The classifier has 5 parameters. These are
+- model type,
+- margin type,
+- \f$\lambda\f$ (strength of restrictions on outliers),
+- \f$\gamma_0\f$ (initial step size),
+- \f$c\f$ (power coefficient for decreasing of step size),
+- and termination criteria.
 
+The model type may have one of the following values: \ref SGD and \ref ASGD.
 
-First, create the SVMSGD object. Set parametrs of model (type, lambda, gamma0, c) using the functions setType, setLambda, setGamma0 and setC or the function setOptimalParametrs.
-Recommended model type is ASGD.
+- \ref SGD is the classic version of SVMSGD classifier: every next step is calculated by the formula
+  \f[w_{t+1} = w_t - \gamma(t) \frac{dQ_i}{dw} |_{w = w_t}\f]
+  where
+  - \f$w_t\f$ is the weights vector for decision function at step \f$t\f$,
+  - \f$\gamma(t)\f$ is the step size of model parameters at the iteration \f$t\f$, it is decreased on each step by the formula
+    \f$\gamma(t) = \gamma_0  (1 + \lambda  \gamma_0 t) ^ {-c}\f$
+  - \f$Q_i\f$ is the target functional from SVM task for sample with number \f$i\f$, this sample is chosen stochastically on each step of the algorithm.
 
-Then the SVM model can be trained using the train features and the correspondent labels.
+- \ref ASGD is Average Stochastic Gradient Descent SVM Classifier. ASGD classifier averages weights vector on each step of algorithm by the formula
+\f$\widehat{w}_{t+1} = \frac{t}{1+t}\widehat{w}_{t} + \frac{1}{1+t}w_{t+1}\f$
 
-After that, the label of a new feature vector can be predicted using the predict function.
+The recommended model type is ASGD (following @cite bottou2010large).
+
+The margin type may have one of the following values: \ref SOFT_MARGIN or \ref HARD_MARGIN.
+
+- You should use \ref HARD_MARGIN type, if you have linearly separable sets.
+- You should use \ref SOFT_MARGIN type, if you have non-linearly separable sets or sets with outliers.
+- In the general case (if you know nothing about linearly separability of your sets), use SOFT_MARGIN.
+
+The other parameters may be described as follows:
+- \f$\lambda\f$ parameter is responsible for weights decreasing at each step and for the strength of restrictions on outliers
+  (the less the parameter, the less probability that an outlier will be ignored).
+  Recommended value for SGD model is 0.0001, for ASGD model is 0.00001.
+
+- \f$\gamma_0\f$ parameter is the initial value for the step size \f$\gamma(t)\f$.
+  You will have to find the best \f$\gamma_0\f$ for your problem.
+
+- \f$c\f$ is the power parameter for \f$\gamma(t)\f$ decreasing by the formula, mentioned above.
+  Recommended value for SGD model is 1, for ASGD model is 0.75.
+
+- Termination criteria can be TermCriteria::COUNT, TermCriteria::EPS or TermCriteria::COUNT + TermCriteria::EPS.
+  You will have to find the best termination criteria for your problem.
+
+Note that the parameters \f$\lambda\f$, \f$\gamma_0\f$, and \f$c\f$ should be positive.
+
+To use SVMSGD algorithm do as follows:
+
+- first, create the SVMSGD object.
+
+- then set parameters  (model type, margin type, \f$\lambda\f$, \f$\gamma_0\f$, \f$c\f$) using the functions
+  setSvmsgdType(), setMarginType(), setLambda(), setGamma0(), and setC(), or the function setOptimalParameters().
+
+- then the SVM model can be trained using the train features and the correspondent labels by the method train().
+
+- after that, the label of a new feature vector can be predicted using the method predict().
 
 @code
-// Initialize object
-SVMSGD SvmSgd;
+// Create empty object
+cv::Ptr<SVMSGD> svmsgd = SVMSGD::create();
+
+// Set parameters
+svmsgd->setOptimalParameters();
 
 // Train the Stochastic Gradient Descent SVM
-SvmSgd.train(trainFeatures, labels);
+SvmSgd->train(trainData);
 
-// Predict label for the new feature vector (1xM)
-predictedLabel = SvmSgd.predict(newFeatureVector);
+// Predict labels for the new samples
+svmsgd->predict(samples, responses);
 @endcode
 
 */
@@ -1537,9 +1588,17 @@ public:
     ASGD is often the preferable choice. */
     enum SvmsgdType
     {
-        ILLEGAL_VALUE,
-        SGD, //!Stochastic Gradient Descent
-        ASGD //!Average Stochastic Gradient Descent
+        ILLEGAL_SVMSGD_TYPE,
+        SGD, //!< Stochastic Gradient Descent
+        ASGD //!< Average Stochastic Gradient Descent
+    };
+
+    /** Margin type.*/
+    enum MarginType
+    {
+        ILLEGAL_MARGIN_TYPE,
+        SOFT_MARGIN, //!< General case, suits to the case of non-linearly separable sets, allows outliers.
+        HARD_MARGIN  //!< More accurate for the case of linearly separable sets.
     };
 
     /**
@@ -1553,49 +1612,59 @@ public:
     CV_WRAP virtual float getShift() = 0;
 
 
-    /** Creates empty model.
+    /** @brief Creates empty model.
         Use StatModel::train to train the model. Since %SVMSGD has several parameters, you may want to
         find the best parameters for your problem or use setOptimalParameters() to set some default parameters.
     */
     CV_WRAP static Ptr<SVMSGD> create();
 
-    /** Function sets optimal parameters values for chosen SVM SGD model.
+    /** @brief Function sets optimal parameters values for chosen SVM SGD model.
      * If chosen type is ASGD, function sets the following values for parameters of model:
-     * lambda = 0.00001;
-     * gamma0 = 0.05;
-     * c = 0.75;
+     * \f$\lambda = 0.00001\f$;
+     * \f$\gamma_0 = 0.05\f$;
+     * \f$c = 0.75\f$;
      * termCrit.maxCount = 100000;
      * termCrit.epsilon = 0.00001;
      *
      * If SGD:
-     * lambda = 0.0001;
-     * gamma0 = 0.05;
-     * c = 1;
+     * \f$\lambda = 0.0001\f$;
+     * \f$\gamma_0 = 0.05\f$;
+     * \f$c = 1\f$;
      * termCrit.maxCount = 100000;
      * termCrit.epsilon = 0.00001;
-     * @param type is the type of SVMSGD classifier. Legal values are SvmsgdType::SGD and SvmsgdType::ASGD.
+     * @param svmsgdType is the type of SVMSGD classifier. Legal values are SvmsgdType::SGD and SvmsgdType::ASGD.
      * Recommended value is SvmsgdType::ASGD (by default).
+     * @param marginType is the type of margin constraint. Legal values are MarginType::SOFT_MARGIN and MarginType::HARD_MARGIN.
+     * Default value is MarginType::SOFT_MARGIN.
     */
-    CV_WRAP virtual void setOptimalParameters(int type = ASGD) = 0;
+    CV_WRAP virtual void setOptimalParameters(int svmsgdType = ASGD, int marginType = SOFT_MARGIN) = 0;
 
-    /** %Algorithm type, one of SVMSGD::SvmsgdType. */
+    /** @brief %Algorithm type, one of SVMSGD::SvmsgdType. */
     /** @see setAlgorithmType */
-    CV_WRAP virtual int getType() const = 0;
+    CV_WRAP virtual int getSvmsgdType() const = 0;
     /** @copybrief getAlgorithmType @see getAlgorithmType */
-    CV_WRAP virtual void setType(int type) = 0;
+    CV_WRAP virtual void setSvmsgdType(int svmsgdType) = 0;
+
+    /** @brief %Margin type, one of SVMSGD::MarginType. */
+    /** @see setMarginType */
+    CV_WRAP virtual int getMarginType() const = 0;
+    /** @copybrief getMarginType @see getMarginType */
+    CV_WRAP virtual void setMarginType(int marginType) = 0;
+
 
-    /** Parameter _Lambda_ of a %SVMSGD optimization problem. Default value is 0. */
+    /** @brief Parameter \f$\lambda\f$ of a %SVMSGD optimization problem. Default value is 0. */
     /** @see setLambda */
     CV_WRAP virtual float getLambda() const = 0;
     /** @copybrief getLambda @see getLambda */
     CV_WRAP virtual void setLambda(float lambda) = 0;
 
-    /** Parameter _Gamma0_ of a %SVMSGD optimization problem. Default value is 0. */
+    /** @brief Parameter \f$\gamma_0\f$ of a %SVMSGD optimization problem. Default value is 0. */
     /** @see setGamma0 */
     CV_WRAP virtual float getGamma0() const = 0;
+    /** @copybrief getGamma0 @see getGamma0 */
     CV_WRAP virtual void setGamma0(float gamma0) = 0;
 
-    /** Parameter _C_ of a %SVMSGD optimization problem. Default value is 0. */
+    /** @brief Parameter \f$c\f$ of a %SVMSGD optimization problem. Default value is 0. */
     /** @see setC */
     CV_WRAP virtual float getC() const = 0;
     /** @copybrief getC @see getC */

modules/ml/test/test_mltests2.cpp

@@ -199,10 +199,19 @@ int str_to_svmsgd_type( String& str )
         return SVMSGD::SGD;
     if ( !str.compare("ASGD") )
         return SVMSGD::ASGD;
-    CV_Error( CV_StsBadArg, "incorrect boost type string" );
+    CV_Error( CV_StsBadArg, "incorrect svmsgd type string" );
     return -1;
 }
 
+int str_to_margin_type( String& str )
+{
+    if ( !str.compare("SOFT_MARGIN") )
+        return SVMSGD::SOFT_MARGIN;
+    if ( !str.compare("HARD_MARGIN") )
+        return SVMSGD::HARD_MARGIN;
+    CV_Error( CV_StsBadArg, "incorrect svmsgd margin type string" );
+    return -1;
+}
 // ---------------------------------- MLBaseTest ---------------------------------------------------
 
 CV_MLBaseTest::CV_MLBaseTest(const char* _modelName)
@@ -452,10 +461,16 @@ int CV_MLBaseTest::train( int testCaseIdx )
     {
         String svmsgdTypeStr;
         modelParamsNode["svmsgdType"] >> svmsgdTypeStr;
-        Ptr<SVMSGD> m = SVMSGD::create();      
-        int type = str_to_svmsgd_type( svmsgdTypeStr );
-        m->setType(type);
-        //m->setType(str_to_svmsgd_type( svmsgdTypeStr ));
+
+        Ptr<SVMSGD> m = SVMSGD::create();
+        int svmsgdType = str_to_svmsgd_type( svmsgdTypeStr );
+        m->setSvmsgdType(svmsgdType);
+
+        String marginTypeStr;
+        modelParamsNode["marginType"] >> marginTypeStr;
+        int marginType = str_to_margin_type( marginTypeStr );
+        m->setMarginType(marginType);
+
         m->setLambda(modelParamsNode["lambda"]);
         m->setGamma0(modelParamsNode["gamma0"]);
         m->setC(modelParamsNode["c"]);

modules/ml/test/test_save_load.cpp

@@ -270,11 +270,7 @@ TEST(ML_DTree, legacy_load) { CV_LegacyTest test(CV_DTREE, "_abalone.xml;_mushro
 TEST(ML_NBayes, legacy_load) { CV_LegacyTest test(CV_NBAYES, "_waveform.xml"); test.safe_run(); }
 TEST(ML_SVM, legacy_load) { CV_LegacyTest test(CV_SVM, "_poletelecomm.xml;_waveform.xml"); test.safe_run(); }
 TEST(ML_RTrees, legacy_load) { CV_LegacyTest test(CV_RTREES, "_waveform.xml"); test.safe_run(); }
-TEST(ML_SVMSGD, legacy_load)
-{
-    CV_LegacyTest test(CV_SVMSGD, "_waveform.xml");
-    test.safe_run();
-}
+TEST(ML_SVMSGD, legacy_load) { CV_LegacyTest test(CV_SVMSGD, "_waveform.xml"); test.safe_run(); }
 
 /*TEST(ML_SVM, throw_exception_when_save_untrained_model)
 {

modules/ml/test/test_svmsgd.cpp

@@ -52,45 +52,99 @@ using cv::ml::TrainData;
 class CV_SVMSGDTrainTest : public cvtest::BaseTest
 {
 public:
-    CV_SVMSGDTrainTest(Mat _weights, float shift);
+    enum TrainDataType
+    {
+        UNIFORM_SAME_SCALE,
+        UNIFORM_DIFFERENT_SCALES
+    };
+
+    CV_SVMSGDTrainTest(Mat _weights, float shift, TrainDataType type, double precision = 0.01);
 private:
     virtual void run( int start_from );
-    float decisionFunction(Mat sample, Mat weights, float shift);
-
+    static float decisionFunction(const Mat &sample, const Mat &weights, float shift);
+    void makeTrainData(Mat weights, float shift);
+    void makeTestData(Mat weights, float shift);
+    void generateSameScaleData(Mat &samples);
+    void generateDifferentScalesData(Mat &samples, float shift);
+
+    TrainDataType type;
+    double precision;
     cv::Ptr<TrainData> data;
     cv::Mat testSamples;
     cv::Mat testResponses;
     static const int TEST_VALUE_LIMIT = 500;
 };
 
-CV_SVMSGDTrainTest::CV_SVMSGDTrainTest(Mat weights, float shift)
+void CV_SVMSGDTrainTest::generateSameScaleData(Mat &samples)
 {
-    int datasize = 100000;
-    int varCount = weights.cols;
-    cv::Mat samples = cv::Mat::zeros( datasize, varCount, CV_32FC1 );
-    cv::Mat responses = cv::Mat::zeros( datasize, 1, CV_32FC1 );
+    float lowerLimit = -TEST_VALUE_LIMIT;
+    float upperLimit = TEST_VALUE_LIMIT;
     cv::RNG rng(0);
+    rng.fill(samples, RNG::UNIFORM, lowerLimit, upperLimit);
+}
+
+void CV_SVMSGDTrainTest::generateDifferentScalesData(Mat &samples, float shift)
+{
+    int featureCount = samples.cols;
 
     float lowerLimit = -TEST_VALUE_LIMIT;
     float upperLimit = TEST_VALUE_LIMIT;
+    cv::RNG rng(10);
 
 
-    rng.fill(samples, RNG::UNIFORM, lowerLimit, upperLimit);
-    for (int sampleIndex = 0; sampleIndex < datasize; sampleIndex++)
+    for (int featureIndex = 0; featureIndex < featureCount; featureIndex++)
     {
-        responses.at<float>( sampleIndex ) = decisionFunction(samples.row(sampleIndex), weights, shift) > 0 ? 1 : -1;
+        int crit = rng.uniform(0, 2);
+
+        if (crit > 0)
+        {
+            rng.fill(samples.col(featureIndex), RNG::UNIFORM, lowerLimit - shift, upperLimit - shift);
+        }
+        else
+        {
+            rng.fill(samples.col(featureIndex), RNG::UNIFORM, lowerLimit/10, upperLimit/10);
+        }
     }
+}
 
+void CV_SVMSGDTrainTest::makeTrainData(Mat weights, float shift)
+{
+    int datasize = 100000;
+    int featureCount = weights.cols;
+    cv::Mat samples = cv::Mat::zeros(datasize, featureCount, CV_32FC1);
+    cv::Mat responses = cv::Mat::zeros(datasize, 1, CV_32FC1);
 
+    switch(type)
+    {
+        case UNIFORM_SAME_SCALE:
+            generateSameScaleData(samples);
+            break;
+        case UNIFORM_DIFFERENT_SCALES:
+            generateDifferentScalesData(samples, shift);
+            break;
+        default:
+            CV_Error(CV_StsBadArg, "Unknown train data type");
+    }
 
-    std::cout << "real weights\n" << weights/norm(weights) << "\n" << std::endl;
-    std::cout << "real shift \n" << shift/norm(weights) << "\n" << std::endl;
+    for (int sampleIndex = 0; sampleIndex < datasize; sampleIndex++)
+    {
+        responses.at<float>(sampleIndex) = decisionFunction(samples.row(sampleIndex), weights, shift) > 0 ? 1 : -1;
+    }
 
-    data = TrainData::create( samples, cv::ml::ROW_SAMPLE, responses );
+    data = TrainData::create(samples, cv::ml::ROW_SAMPLE, responses);
+}
 
+void CV_SVMSGDTrainTest::makeTestData(Mat weights, float shift)
+{
     int testSamplesCount = 100000;
+    int featureCount = weights.cols;
+
+    float lowerLimit = -TEST_VALUE_LIMIT;
+    float upperLimit = TEST_VALUE_LIMIT;
+
+    cv::RNG rng(0);
 
-    testSamples.create(testSamplesCount, varCount, CV_32FC1);
+    testSamples.create(testSamplesCount, featureCount, CV_32FC1);
     rng.fill(testSamples, RNG::UNIFORM, lowerLimit, upperLimit);
     testResponses.create(testSamplesCount, 1, CV_32FC1);
 
@@ -100,12 +154,24 @@ CV_SVMSGDTrainTest::CV_SVMSGDTrainTest(Mat weights, float shift)
     }
 }
 
+CV_SVMSGDTrainTest::CV_SVMSGDTrainTest(Mat weights, float shift, TrainDataType _type, double _precision)
+{
+    type = _type;
+    precision = _precision;
+    makeTrainData(weights, shift);
+    makeTestData(weights, shift);
+}
+
+float CV_SVMSGDTrainTest::decisionFunction(const Mat &sample, const Mat &weights, float shift)
+{
+    return sample.dot(weights) + shift;
+}
+
 void CV_SVMSGDTrainTest::run( int /*start_from*/ )
 {
     cv::Ptr<SVMSGD> svmsgd = SVMSGD::create();
 
-    svmsgd->setOptimalParameters(SVMSGD::ASGD);
-    svmsgd->setTermCriteria(TermCriteria(TermCriteria::EPS, 0, 0.00005));
+    svmsgd->setOptimalParameters();
 
     svmsgd->train(data);
 
@@ -118,77 +184,106 @@ void CV_SVMSGDTrainTest::run( int /*start_from*/ )
 
     for (int i = 0; i < testSamplesCount; i++)
     {
-        if (responses.at<float>(i) * testResponses.at<float>(i) < 0 )
+        if (responses.at<float>(i) * testResponses.at<float>(i) < 0)
             errCount++;
     }
 
-
-    float normW = norm(svmsgd->getWeights());
-
-    std::cout << "found weights\n" << svmsgd->getWeights()/normW << "\n" << std::endl;
-    std::cout << "found shift \n" << svmsgd->getShift()/normW << "\n" << std::endl;
-
     float err = (float)errCount / testSamplesCount;
     std::cout << "err " << err << std::endl;
 
-    if ( err > 0.01 )
+    if ( err > precision )
     {
-        ts->set_failed_test_info( cvtest::TS::FAIL_BAD_ACCURACY );
+        ts->set_failed_test_info(cvtest::TS::FAIL_BAD_ACCURACY);
     }
 }
 
-float CV_SVMSGDTrainTest::decisionFunction(Mat sample, Mat weights, float shift)
+
+void makeWeightsAndShift(int featureCount, Mat &weights, float &shift)
 {
-    return sample.dot(weights) + shift;
+    weights.create(1, featureCount, CV_32FC1);
+    cv::RNG rng(0);
+    double lowerLimit = -1;
+    double upperLimit = 1;
+
+    rng.fill(weights, RNG::UNIFORM, lowerLimit, upperLimit);
+    shift = rng.uniform(-featureCount, featureCount);
 }
 
-TEST(ML_SVMSGD, train0)
+
+TEST(ML_SVMSGD, trainSameScale2)
 {
-    int varCount = 2;
+    int featureCount = 2;
 
     Mat weights;
-    weights.create(1, varCount, CV_32FC1);
-    weights.at<float>(0) = 1;
-    weights.at<float>(1) = 0;
-    cv::RNG rng(1);
-    float shift = rng.uniform(-varCount, varCount);
 
-    CV_SVMSGDTrainTest test(weights, shift);
+    float shift = 0;
+    makeWeightsAndShift(featureCount, weights, shift);
+
+    CV_SVMSGDTrainTest test(weights, shift, CV_SVMSGDTrainTest::UNIFORM_SAME_SCALE);
     test.safe_run();
 }
 
-TEST(ML_SVMSGD, train1)
+TEST(ML_SVMSGD, trainSameScale5)
 {
-    int varCount = 5;
+    int featureCount = 5;
 
     Mat weights;
-    weights.create(1, varCount, CV_32FC1);
 
-    float lowerLimit = -1;
-    float upperLimit = 1;
-    cv::RNG rng(0);
-    rng.fill(weights, RNG::UNIFORM, lowerLimit, upperLimit);
+    float shift = 0;
+    makeWeightsAndShift(featureCount, weights, shift);
+
+    CV_SVMSGDTrainTest test(weights, shift, CV_SVMSGDTrainTest::UNIFORM_SAME_SCALE);
+    test.safe_run();
+}
+
+TEST(ML_SVMSGD, trainSameScale100)
+{
+    int featureCount = 100;
+
+    Mat weights;
 
-    float shift = rng.uniform(-varCount, varCount);
+    float shift = 0;
+    makeWeightsAndShift(featureCount, weights, shift);
 
-    CV_SVMSGDTrainTest test(weights, shift);
+    CV_SVMSGDTrainTest test(weights, shift, CV_SVMSGDTrainTest::UNIFORM_SAME_SCALE);
     test.safe_run();
 }
 
-TEST(ML_SVMSGD, train2)
+TEST(ML_SVMSGD, trainDifferentScales2)
 {
-    int varCount = 100;
+    int featureCount = 2;
 
     Mat weights;
-    weights.create(1, varCount, CV_32FC1);
 
-    float lowerLimit = -1;
-    float upperLimit = 1;
-    cv::RNG rng(0);
-    rng.fill(weights, RNG::UNIFORM, lowerLimit, upperLimit);
+    float shift = 0;
+    makeWeightsAndShift(featureCount, weights, shift);
+
+    CV_SVMSGDTrainTest test(weights, shift, CV_SVMSGDTrainTest::UNIFORM_DIFFERENT_SCALES, 0.01);
+    test.safe_run();
+}
+
+TEST(ML_SVMSGD, trainDifferentScales5)
+{
+    int featureCount = 5;
+
+    Mat weights;
+
+    float shift = 0;
+    makeWeightsAndShift(featureCount, weights, shift);
+
+    CV_SVMSGDTrainTest test(weights, shift, CV_SVMSGDTrainTest::UNIFORM_DIFFERENT_SCALES, 0.05);
+    test.safe_run();
+}
+
+TEST(ML_SVMSGD, trainDifferentScales100)
+{
+    int featureCount = 100;
+
+    Mat weights;
 
-    float shift = rng.uniform(-varCount, varCount);
+    float shift = 0;
+    makeWeightsAndShift(featureCount, weights, shift);
 
-    CV_SVMSGDTrainTest test(weights,shift);
+    CV_SVMSGDTrainTest test(weights, shift, CV_SVMSGDTrainTest::UNIFORM_DIFFERENT_SCALES, 0.10);
     test.safe_run();
 }

samples/cpp/train_svmsgd.cpp

@@ -40,16 +40,13 @@ void fillSegments(std::vector<std::pair<Point,Point> > &segments, int width, int
 void redraw(Data data, const Point points[2]);
 
 //add point in train set, train SVMSGD algorithm and draw results on image
-void addPointRetrainAndRedraw(Data &data, int x, int y);
+void addPointRetrainAndRedraw(Data &data, int x, int y, int response);
 
 
 bool doTrain( const Mat samples, const Mat responses, Mat &weights, float &shift)
 {
     cv::Ptr<SVMSGD> svmsgd = SVMSGD::create();
-    svmsgd->setOptimalParameters(SVMSGD::ASGD);
-    svmsgd->setTermCriteria(TermCriteria(TermCriteria::EPS, 0, 0.00000001));
-    svmsgd->setLambda(0.00000001);
-
+    svmsgd->setOptimalParameters();
 
     cv::Ptr<TrainData> train_data = TrainData::create(samples, cv::ml::ROW_SAMPLE, responses);
     svmsgd->train( train_data );
@@ -64,6 +61,27 @@ bool doTrain( const Mat samples, const Mat responses, Mat &weights, float &shift
     return false;
 }
 
+void fillSegments(std::vector<std::pair<Point,Point> > &segments, int width, int height)
+{
+    std::pair<Point,Point> currentSegment;
+
+    currentSegment.first = Point(width, 0);
+    currentSegment.second = Point(width, height);
+    segments.push_back(currentSegment);
+
+    currentSegment.first = Point(0, height);
+    currentSegment.second = Point(width, height);
+    segments.push_back(currentSegment);
+
+    currentSegment.first = Point(0, 0);
+    currentSegment.second = Point(width, 0);
+    segments.push_back(currentSegment);
+
+    currentSegment.first = Point(0, 0);
+    currentSegment.second = Point(0, height);
+    segments.push_back(currentSegment);
+}
+
 
 bool findCrossPointWithBorders(const Mat &weights, float shift, const std::pair<Point,Point> &segment, Point &crossPoint)
 {
@@ -123,27 +141,6 @@ bool findPointsForLine(const Mat &weights, float shift, Point (&points)[2], int
     return true;
 }
 
-void fillSegments(std::vector<std::pair<Point,Point> > &segments, int width, int height)
-{
-    std::pair<Point,Point> currentSegment;
-
-    currentSegment.first = Point(width, 0);
-    currentSegment.second = Point(width, height);
-    segments.push_back(currentSegment);
-
-    currentSegment.first = Point(0, height);
-    currentSegment.second = Point(width, height);
-    segments.push_back(currentSegment);
-
-    currentSegment.first = Point(0, 0);
-    currentSegment.second = Point(width, 0);
-    segments.push_back(currentSegment);
-
-    currentSegment.first = Point(0, 0);
-    currentSegment.second = Point(0, height);
-    segments.push_back(currentSegment);
-}
-
 void redraw(Data data, const Point points[2])
 {
     data.img.setTo(0);
@@ -162,19 +159,20 @@ void redraw(Data data, const Point points[2])
     imshow("Train svmsgd", data.img);
 }
 
-void addPointRetrainAndRedraw(Data &data, int x, int y)
+void addPointRetrainAndRedraw(Data &data, int x, int y, int response)
 {
     Mat currentSample(1, 2, CV_32F);
 
     currentSample.at<float>(0,0) = x;
     currentSample.at<float>(0,1) = y;
     data.samples.push_back(currentSample);
+    data.responses.push_back(response);
 
     Mat weights(1, 2, CV_32F);
     float shift = 0;
 
     if (doTrain(data.samples, data.responses, weights, shift))
-   {
+    {
         Point points[2];
         findPointsForLine(weights, shift, points, data.img.cols, data.img.rows);
 
@@ -189,15 +187,12 @@ static void onMouse( int event, int x, int y, int, void* pData)
 
     switch( event )
     {
-    case CV_EVENT_LBUTTONUP:
-        data.responses.push_back(1);
-        addPointRetrainAndRedraw(data, x, y);
-
+    case CV_EVENT_LBUTTONUP:        
+        addPointRetrainAndRedraw(data, x, y, 1);
         break;
 
     case CV_EVENT_RBUTTONDOWN:
-        data.responses.push_back(-1);
-        addPointRetrainAndRedraw(data, x, y);
+        addPointRetrainAndRedraw(data, x, y, -1);
         break;
     }