Initial commit

.gitignore

+/build
+/test_video
+/.vscode
+*.mp4
+*.md~
\ No newline at end of file

CBM_model.h

+#ifndef _CBM_MODEL_H
+#define _CBM_MODEL_H
+
+#include "VideoDetails.h"
+#include "myUtility.h"
+#include "myDIP.h"
+#include "myGMM.h"
+#include <time.h>
+#include <vector>
+#include <stdio.h>
+#include <stdlib.h>
+#include <list>
+//#include <Windows.h>
+
+#include "connected.h"
+#include <functional>
+
+#include "parameter.h"
+
+#include "opencv2/objdetect/objdetect.hpp"
+
+
+#define Bin_num  9
+#define Block_size 16
+#define Block_stride 8
+#define Cell_size 8
+#define Win_width 64
+#define Win_height 128
+
+using namespace std;
+using namespace cv;
+struct Obj_info
+{
+public:	
+	int x;
+	int y;
+	int width;
+	int height;
+	int label;
+	float distance;
+	bool tracked;
+	double Owner_R[10];
+	double Owner_G[10];
+	double Owner_B[10];
+	int traj_label;
+	float traj_dist;
+
+	Obj_info()
+	{
+		label = 0;
+		distance = 2147483647.0;
+		tracked = false;
+		traj_dist = 2147483647.0;
+		for (int i = 0; i < 10; i++)
+		{
+			Owner_R[i] = 0.0; Owner_G[i] = 0.0; Owner_B[i] = 0.0;//for validation method 3 
+		}
+	}
+};
+
+struct pixelFSM
+{
+	//short  long   state
+	//  0    0        0
+	//  0    1        1
+	//  1    0        2
+	//  1    1        3
+public:
+	int state_now;
+	int state_pre;
+	int static_count;
+	bool staticFG_candidate;
+	bool staticFG_stable;
+
+	pixelFSM()
+	{
+		state_now = 0;
+		state_pre = 0;
+		static_count = 0;
+		staticFG_candidate = false;
+		staticFG_stable = false;
+	}
+};
+
+class CBM_model
+{
+public:
+
+	int MOG_LEARN_FRAMES;
+	int MIN_AREA;
+	int TEMPORAL_RULE;
+
+	//VideoDetails *_video;//input video
+	cv::VideoWriter _writer6, _writer7, _writer8, _writer9, _writer10;
+	CBM_model(myImage * input, int set_MOG_LearnFrame, int set_min_area, int set_buffer_len, float set_resize, myImage * mask);
+	~CBM_model();
+	void Initialize();
+	void Uninitialize();
+	bool Motion_Detection(myImage *img);
+
+	bool myClustering2( myImage *img, int option);//option: 0 for moving obj, 1 for static obj
+	int GetLabeling2( myImage *pImg1, int areaThreshold, int option); //option: 0 for moving obj, 1 for static obj
+
+	void myFSM(myImage *short_term, myImage *long_term, pixelFSM ** imageFSM, bool *** Previous_FG);
+	void myConvert2Img(bool **Array, myImage *output);
+	void myConvertFSM2Img(pixelFSM **Array, myImage *Candidate_Fg, myImage *Static_Fg);
+
+	int check_foreground2( myImage * img);
+
+	int frame_count;
+	int sampling_idx;
+
+	//********RESIZE**************//
+	int new_width;
+	int new_height;
+	float RESIZE_RATE;
+	//********detected result**************//
+	vector<Obj_info*> detected_result;//information of the moving objects
+	vector<Obj_info*> static_object_result;//information of the static foreground objects
+	vector<Obj_info*> GetDetectResult();//get the information of the moving objects
+	vector<Obj_info*> GetStaticForegourdResult();//get the information of the static foreground objects 
+
+	bool ** GetPrevious_nForeground( int n);
+	void DetectPrevious_nForeground_DPM2( int n);
+	void DetectPrevious_nForeground_DPM( int n);
+	void DetectPrevious_nForeground_HOG( int n);
+	//CvLatentSvmDetector * detector;
+	// cv::ml::SVM *  detector;
+	// cv::HOGDescriptor hog;//�ŧihog detector
+	vector<cv::Rect>  found;
+
+	myImage * _GetStaticForeground();
+	myImage * _GetPreviousFrame();
+	myImage * _GetObjLfetFrame();
+	myImage * _GetPrevious_nFrame( int n);
+
+	void System_Reset();
+
+	bool ***Previous_FG;
+
+	myImage ** _Previous_Img;
+	myImage * my_imgStatic, * my_imgStatic2;
+	myImage * maskROI;
+	myImage * input_temp;
+private:
+	cv::Mat mog_fg;//long term
+	cv::Mat mog_fg2;//short term
+	cv::Mat imgStatic;
+
+
+	myImage * my_mog_fg;//long term
+	myImage * my_mog_fg2;//short term
+	myImage * my_imgCandiStatic; 
+
+	VideoDetails *video;
+	
+	myGMM * _myGMM;//long term
+	myGMM * _myGMM2;//short term
+
+	float MoG_LearningRate;// long term
+	float MoG_LearningRate2;// short term
+	int FG_count;
+
+	pixelFSM **imageFSM;
+
+	int staticFG_pixel_num_now;
+	int staticFG_pixel_num_pre;
+	int staticFG_pixel_num_pre2;
+
+	myImage * dpm_gray;//for dpm
+
+};
+
+#endif

CMakeLists.txt

+cmake_minimum_required(VERSION 2.8)
+project(ObjLeft)
+find_package(OpenCV  REQUIRED)
+link_directories(${MYPROJECT_DIR})
+
+
+#set(CMAKE_BUILD_TYPE "Debug")
+set(CMAKE_BUILD_TYPE "Release")
+set (CMAKE_CXX_FLAGS "-O0 -march=native")
+set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -Wno-write-strings")
+set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-rpath -Wl,$ORIGIN") 
+
+add_subdirectory(yolov3)
+
+# add_executable(ObjLeft main CBM_model myDIP myGMM myUtility ObjLeftDetect VideoDetails)
+add_executable(ObjLeft main DetectAnomaly CBM_model myDIP myGMM myUtility ObjLeftDetect VideoDetails)
+target_link_libraries( ObjLeft ${OpenCV_LIBS} gpu)

DetectAnomaly.cpp

+// Copyright 2020 WuLIAO
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "DetectAnomaly.h"
+#include "parameter.h"
+
+int OWNER_SEARCH_ROI;
+int GMM_LEARN_FRAME; //frame number for GMM initialization
+int MAX_SFG;
+int MIN_SFG;
+int MAX_FG;
+int MIN_FG;
+int BUFFER_LENGTH;
+double GMM_LONG_LEARN_RATE;
+double GMM_SHORT_LEARN_RATE;
+double INPUT_RESIZE;
+
+bool DetectAnomaly::detectAnomaly(cv::Mat img, vector<vector<int>>* resultLocation)
+{   
+    yolo_test = img.clone();
+    cv::resize(img, img,cv::Size(2448*0.25, 2048*0.25), cv::INTER_CUBIC);
+    img.copyTo(myimg);
+    medianBlur(mat_myimg, mat_myimg, 3);
+    opencv_2_myImage(myimg, myimg1); //transfer opencv data to myimage data
+
+    /************************************************************************/
+    /* abandoned object detection algorithm                                 */
+    /************************************************************************/
+    obj_left = _objleft.process(myimg1,yolo_test,resultLocation);
+#ifdef DEBUG
+    // if (obj_left == true)
+    // {
+    //     printf("alram!!\n");
+    // }
+    
+    // Mat _qImg(qImg);
+    // putText(_qImg, "fps:" + WorkFps(), Point(5, 20), FONT_HERSHEY_SIMPLEX, 0.9, Scalar(255, 100, 0), 2);
+    // cvWriteFrame(_writer, qImg);
+    // cvWaitKey(1);
+#endif
+    return obj_left;
+}
+
+DetectAnomaly::DetectAnomaly()
+{
+    // cfgAnomaly.input_resize = 0.25;
+    // cfgAnomaly.owner_search_roi = 10; //100->10
+    // cfgAnomaly.gmm_learn_frame  = 100;
+    // cfgAnomaly.max_sfg          = 50000;
+    // cfgAnomaly.min_sfg          = 100;
+    // cfgAnomaly.max_fg           = 100000;
+    // cfgAnomaly.min_fg           = 200;
+    // cfgAnomaly.buffer_length    = 100; //300->100 
+    // cfgAnomaly.image_height     = 2048 * cfgAnomaly.input_resize;
+    // cfgAnomaly.image_width      = 2448 * cfgAnomaly.input_resize;
+
+    // cfgAnomaly.gmm_long_learn_rate = 0.0001;
+    // cfgAnomaly.gmm_short_learn_rate = 0.002;
+
+
+    // // set gloabl var
+    // INPUT_RESIZE = 1.0;
+    // OWNER_SEARCH_ROI = 10;
+    // GMM_LEARN_FRAME = 100; //frame number for GMM initialization
+    // MAX_SFG = 50000;
+    // MIN_SFG = 100;
+    // MAX_FG = 100000;
+    // MIN_FG = 200;
+    // BUFFER_LENGTH = 100;
+    // GMM_LONG_LEARN_RATE = 0.0001;
+    // GMM_SHORT_LEARN_RATE = 0.002;
+    // // INPUT_RESIZE = 0.25;
+    cfgAnomaly.input_resize = 0.25;
+    cfgAnomaly.owner_search_roi = 10;
+    cfgAnomaly.gmm_learn_frame  = 100;
+    cfgAnomaly.max_sfg          = 15625;
+    cfgAnomaly.min_sfg          = 50;
+    cfgAnomaly.max_fg           = 75000;
+    cfgAnomaly.min_fg           = 100; 
+    cfgAnomaly.buffer_length    = 100; 
+    cfgAnomaly.image_height     = 2048 * cfgAnomaly.input_resize;
+    cfgAnomaly.image_width      = 2448 * cfgAnomaly.input_resize;
+
+    cfgAnomaly.gmm_long_learn_rate = 0.0001;
+    cfgAnomaly.gmm_short_learn_rate = 0.002;
+
+
+    // set gloabl var
+    INPUT_RESIZE = 1.0;
+    OWNER_SEARCH_ROI = 10;
+    GMM_LEARN_FRAME = 100; //frame number for GMM initialization
+    MAX_SFG = 15625;
+    MIN_SFG = 50;
+    MAX_FG = 75000;
+    MIN_FG = 100;
+    BUFFER_LENGTH = 100;
+    GMM_LONG_LEARN_RATE = 0.0001;
+    GMM_SHORT_LEARN_RATE = 0.002;
+    // INPUT_RESIZE = 0.25;
+
+    cfgAnomaly.top_left_x = 1080 * cfgAnomaly.input_resize;
+    cfgAnomaly.top_left_y = 540 * cfgAnomaly.input_resize;
+    cfgAnomaly.top_right_x = 1680 * cfgAnomaly.input_resize;
+    cfgAnomaly.top_right_y = 820 * cfgAnomaly.input_resize;
+    cfgAnomaly.bottom_left_x = 847 * cfgAnomaly.input_resize;
+    cfgAnomaly.bottom_left_y = 1015 * cfgAnomaly.input_resize;
+    cfgAnomaly.bottom_right_x = 1489 * cfgAnomaly.input_resize;
+    cfgAnomaly.bottom_right_y = 1255 * cfgAnomaly.input_resize;
+    obj_left = false;
+    arr[0] = 4;
+
+    mymask = myCreateImage(cfgAnomaly.image_width, cfgAnomaly.image_height, 3);
+    myInverse(mymask, mymask);
+    myimg1 = myCreateImage(cfgAnomaly.image_width, cfgAnomaly.image_height, 3);
+    myimg = cv::Mat(cv::Size(cfgAnomaly.image_width, cfgAnomaly.image_height), CV_8UC3);
+    myimg.copyTo(mat_myimg);
+    // mat_myimg(myimg, 0);
+
+    // set roi
+    // PointArray[0][0] = cv::Point(567, 635);
+    // PointArray[0][1] = cv::Point(1009, 625);
+    // PointArray[1][1] = cv::Point(9, 859);
+    // PointArray[1][0] = cv::Point(1427, 1069);
+
+    PointArray[0][0] = cv::Point(cfgAnomaly.top_left_x, cfgAnomaly.top_left_y);
+    PointArray[0][1] = cv::Point(cfgAnomaly.top_right_x, cfgAnomaly.top_right_y);
+    PointArray[1][1] = cv::Point(cfgAnomaly.bottom_left_x, cfgAnomaly.bottom_left_y);
+    PointArray[1][0] = cv::Point(cfgAnomaly.bottom_right_x, cfgAnomaly.bottom_right_y);
+
+    const Point* ppt[2] = {PointArray[0], PointArray[1]};
+
+    cv::Mat Image1 = cv::Mat(cvSize(cfgAnomaly.image_width, cfgAnomaly.image_height), CV_8UC1);
+    cv::polylines(Image1, PointArray, arr, 1,true, cv::Scalar(255, 255, 255),5,8,0);
+    cv::fillPoly(Image1, ppt, arr, 1, cv::Scalar(255, 255, 255),0);
+    cv::threshold(Image1, Image1, 254, 255,THRESH_BINARY);
+    mask = Image1.clone();
+
+    opencv_2_myImage(mask, mymask);
+#ifdef DEBUG
+    //cv::imwrite("mask.jpg", Image1);
+    cv::waitKey(1);
+    //cvReleaseImage(&Image1);
+#endif
+    _objleft.ObjLeftSet(myimg1, cfgAnomaly.gmm_learn_frame, cfgAnomaly.min_fg, cfgAnomaly.buffer_length, mymask);
+#ifdef DEBUG
+    printf("Finished initialization.\n");
+#endif
+}
+
+DetectAnomaly::DetectAnomaly(config cfg) : cfgAnomaly(cfg)
+{
+    // set params
+    cfgAnomaly.owner_search_roi = cfg.owner_search_roi;
+    cfgAnomaly.gmm_learn_frame  = cfg.gmm_learn_frame;
+    cfgAnomaly.max_sfg          = cfg.max_sfg;
+    cfgAnomaly.min_sfg          = cfg.min_sfg;
+    cfgAnomaly.max_fg           = cfg.max_fg;
+    cfgAnomaly.min_fg           = cfg.min_fg;
+    cfgAnomaly.buffer_length    = cfg.buffer_length;
+    cfgAnomaly.image_height     = cfg.image_height;
+    cfgAnomaly.image_width      = cfg.image_width;
+
+    cfgAnomaly.gmm_long_learn_rate  = cfg.gmm_long_learn_rate;
+    cfgAnomaly.gmm_short_learn_rate = cfg.gmm_short_learn_rate;
+    cfgAnomaly.input_resize         = cfg.input_resize;
+
+    // set gloabl var
+    OWNER_SEARCH_ROI = cfg.owner_search_roi;
+    GMM_LEARN_FRAME  = cfg.gmm_learn_frame;
+    MAX_SFG          = cfg.max_sfg;
+    MIN_SFG          = cfg.min_sfg;
+    MAX_FG           = cfg.max_fg;
+    MIN_FG           = cfg.min_fg;
+    BUFFER_LENGTH    = cfg.buffer_length;
+    GMM_LONG_LEARN_RATE  = cfg.gmm_long_learn_rate;
+    GMM_SHORT_LEARN_RATE = cfg.gmm_short_learn_rate;
+    INPUT_RESIZE = 1.0;
+
+    cfgAnomaly.top_left_x       = cfg.top_left_x;
+    cfgAnomaly.top_left_y       = cfg.top_left_y;
+    cfgAnomaly.top_right_x      = cfg.top_right_x;
+    cfgAnomaly.top_right_y      = cfg.top_right_y;
+    cfgAnomaly.bottom_left_x    = cfg.bottom_left_x;
+    cfgAnomaly.bottom_left_y    = cfg.bottom_left_y;
+    cfgAnomaly.bottom_right_x   = cfg.bottom_right_x;
+    cfgAnomaly.bottom_right_y   = cfg.bottom_left_y;
+    arr[0] = 4;
+    obj_left = false;
+
+    mymask = myCreateImage(cfgAnomaly.image_width, cfgAnomaly.image_height, 3);
+    myInverse(mymask, mymask);
+    myimg1 = myCreateImage(cfgAnomaly.image_width, cfgAnomaly.image_height, 3);
+    myimg = cv::Mat(cv::Size(cfgAnomaly.image_width, cfgAnomaly.image_height), CV_8UC3);
+    myimg.copyTo(mat_myimg);
+    // mat_myimg(myimg, 0);
+
+    // set roi
+    // PointArray[0][0] = cv::Point(567, 635);
+    // PointArray[0][1] = cv::Point(1009, 625);
+    // PointArray[1][1] = cv::Point(9, 859);
+    // PointArray[1][0] = cv::Point(1427, 1069);
+    PointArray[0][0] = cv::Point(1080, 540);
+    PointArray[0][1] = cv::Point(1680, 820);
+    PointArray[1][1] = cv::Point(847, 1015);
+    PointArray[1][0] = cv::Point(1489, 1255);
+
+    const Point* ppt[2] = {PointArray[0], PointArray[1]};
+    
+
+    cv::Mat Image1 = cv::Mat(cv::Size(cfgAnomaly.image_width, cfgAnomaly.image_height), CV_8UC1);
+    cv::polylines(Image1, PointArray, arr, 1,true, cv::Scalar(255, 255, 255),5,8,0);
+    cv::fillPoly(Image1, ppt, arr, 1, cv::Scalar(255, 255, 255),0);
+    cv::threshold(Image1, Image1, 254, 255, THRESH_BINARY);
+    mask = Image1;
+
+    opencv_2_myImage(mask, mymask);
+#ifdef DEBUG 
+    cv::imwrite("mask.jpg", Image1);
+    cv::waitKey(1);
+    //cvReleaseImage(&Image1);
+#endif
+    _objleft.ObjLeftSet(myimg1, cfgAnomaly.gmm_learn_frame, cfgAnomaly.min_fg, cfgAnomaly.buffer_length, mymask);
+#ifdef DEBUG 
+    printf("Finished initialization.\n");
+#endif
+}
+
+DetectAnomaly::~DetectAnomaly()
+{
+    myimg1->myReleaseImage();
+    mymask->myReleaseImage();
+    // cvReleaseImage(&myimg);
+    // cvReleaseImage(&mask);
+    // delete _objleft;
+    // ObjLeftDetect::~ObjLeftDetect();
+}
\ No newline at end of file

DetectAnomaly.h

+#ifndef _DETECTANOMALY_H
+#define _DETECTANOMALY_H
+
+#define DEBUG
+
+#include "ObjLeftDetect.h"
+#include "VideoDetails.h"
+// #include "parameter.h"
+#include <iostream>
+
+typedef struct ConfigAnomaly
+{
+    int owner_search_roi;
+    int gmm_learn_frame; //frame number for GMM initialization
+    int max_sfg;
+    int min_sfg;
+    int max_fg;
+    int min_fg;
+    int buffer_length;
+    int image_height;
+    int image_width;
+    double gmm_long_learn_rate;
+    double gmm_short_learn_rate;
+    double input_resize;
+
+    int top_left_x;
+    int top_left_y;
+    int top_right_x;
+    int top_right_y;
+    int bottom_left_x;
+    int bottom_left_y;
+    int bottom_right_x;
+    int bottom_right_y;
+
+} config;
+
+class DetectAnomaly
+{
+private:
+    // config
+    config cfgAnomaly;
+
+    // roi point
+    cv::Point PointArray1[4];
+    cv::Point *PointArray[2] = {&PointArray1[0], &PointArray1[2]};
+
+    cv::Mat mask;
+    myImage *mymask;
+
+    cv::Mat myimg;
+    myImage *myimg1;
+
+    cv::Mat mat_myimg;
+
+    ObjLeftDetect _objleft;
+    bool obj_left;
+
+    int arr[1];
+
+public:
+    bool detectAnomaly(cv::Mat img, vector<vector<int>>* resultLocation);
+    DetectAnomaly();
+    DetectAnomaly(config cfg);
+
+    cv::Mat yolo_test;
+
+    ~DetectAnomaly();
+};
+
+
+
+#endif
\ No newline at end of file

ObjLeftDetect.h

+#ifndef _OBJLEFTDETECT_H_
+#define _OBJLEFTDETECT_H_
+#include "myDIP.h"
+#include "CBM_model.h"
+#include "ObjectDetail.h"
+#include "myUtility.h"
+#include <vector>
+#include <iostream>
+#include <sstream>
+#include <fstream>
+#include <limits.h>
+#include <string.h>
+#include <time.h>
+//#include <windows.h>
+//#include <process.h>
+//#include <io.h>
+#include <math.h>
+#include <string>
+
+#include "yolov3/extra/api.hpp"
+
+#define PI  3.1415926535897932384626433832795
+
+
+class ObjLeftDetect
+{
+public:
+	ObjLeftDetect();
+	ObjLeftDetect(myImage *input, int set_MOG_LearnFrame, int set_min_area, int set_buffer_len, myImage *mask);
+	~ObjLeftDetect();
+	bool process(myImage * input,cv::Mat show_pic, vector<vector<int>>* resultLocation);
+	CBM_model *_CBM_model;
+	//cv::VideoWriter _writer1, _writer2;
+	cv::Mat A, B;
+
+	int ** image;
+	myFloatColor * connect_colors;
+	void ObjLeftSet(myImage *input, int set_MOG_LearnFrame, int set_min_area, int set_buffer_len, myImage *mask);
+
+private:
+	//SYSTEMTIME st;
+	VideoDetails *_video, *_video2;
+	
+	char _statement[200]; 	
+	void initialize();
+	void do_detection( int validation_method);
+	bool soft_validation3( myImage * ImgSynopsis, vector<Obj_info*> obj_left);
+	float ComputeIoU(std::vector<int>& a, MyResult& b);
+	
+	//show result //
+	inline void WorkBegin();
+	inline void WorkEnd();
+	inline string WorkFps();
+
+
+	int new_width, new_height; 
+	vector<Obj_info *> alarmList;
+	myImage * myimg1, * myimg2;
+	cv::Mat ImgSynopsis;
+	myImage * _ImgSynopsis;
+
+	int Spatial_Temporal_Search( int ** Image, int i, int j, myFloatColor * colors, int time_stamp,int my_label);
+	int spatial_flood( bool ** foreground, int i, int j);
+
+	myColor color_rainbow(int total_time, int current_time);
+	bool object_detected;
+	vector<Obj_info*> ObjLocation;
+	vector<Obj_info*> LeftLocation;
+	bool set_alarm;
+	int pic_num = 0;
+
+};
+
+#endif

ObjectDetail.h

+#ifndef _OBJECTDETAIL_H
+#define _OBJECTDETAIL_H
+
+class ObjectDetail
+{
+public:
+	int GetCamID()			{return camID;};
+	int GetObjectID()		{return objectID;};
+	int GetTypeID()			{return typeID;};
+	int GetOriTypeID()		{return oritypeID;};
+	int GetArea()           {return area;};
+	int GetX()				{return x;};
+	int GetY()				{return y;};
+	int GetWidth()			{return width;};
+	int GetHeight()			{return height;};
+	int GetLength()			{return length;};
+	int GetSFrameID()		{return sFrameID;};
+	int GetDirection()      {return direction;};
+	long int GetFrameID()	{return frameID;};
+	long int GetStartFrame(){return startFrame;};
+	long int GetEndFrame()	{return endFrame;};
+
+	void SetCamID(int value)			{camID = value;};
+	void SetObjectID(int value)			{objectID = value;};
+	void SetTypeID(int value)           {typeID = value;};
+	void SetOriTypeID(int value)		{oritypeID = value;};
+	void SetArea(int value)				{area = value;};
+	void SetX(int value)				{x = value;};
+	void SetY(int value)				{y = value;};
+	void SetWidth(int value)			{width = value;};
+	void SetHeight(int value)			{height = value;};
+	void SetLength(int value)			{length = value;};
+	void SetSFrameID(int value)			{sFrameID = value;};
+	void SetDirection(int value)        {direction = value;};
+	void SetFrameID(long int value)		{frameID = value;};
+	void SetStartFrame(long int value)  {startFrame = value;};
+	void SetEndFrame(long int value)	{endFrame = value;};
+
+	int  offsetFrame;       
+	long int playFrame;    
+
+private:
+	int camID;
+	int objectID;
+	int typeID;
+	int oritypeID;
+	int area;
+	int x;
+	int y;
+	int width;
+	int height;
+	int length;
+	int sFrameID;    
+	int direction;
+	long int frameID;
+	long int startFrame;
+	long int endFrame;
+	
+};
+
+
+#endif

README.md

+_FYI: We use OpenCV only for video input and visualization. Most of the algorithms are implemented using C/C++. I am improving the readability of this code. Will update it soon!_
+
+---
+
+# ObjLeft
+
+Abandoned Object Detection via Temporal Consistency Modeling and Back-Tracing Verification for Visual Surveillance
+
+Created by Kevin Lin, Shen-Chi Chen, Chu-Song Chen, Daw-Tung Lin, Yi-Ping Hung at National Taiwan University.
+
+
+## Introduction
+
+This paper presents an effective approach for detecting abandoned luggage in surveillance videos. We combine short- and long-term background models to extract foreground objects, where each pixel in an input image is classified as a 2-bit code. Subsequently, we introduce a framework to identify static foreground regions based on the temporal transition of code patterns, and to determine whether the candidate regions contain abandoned objects by analyzing the back-traced trajectories of luggage owners. Experimental results show that our method performs more favorable against the others.
+
+The details can be found in the following [IEEE TIFS 2015 paper](http://www.csie.ntu.edu.tw/~r01944012/TIFS15_LIN.pdf)
+
+
+## Citing the detection works
+
+If you find our works useful in your research, please consider citing:
+
+    Abandoned Object Detection via Temporal Consistency Modeling and Back-Tracing Verification for Visual Surveillance
+    K. Lin, S.-C. Chen, C.-S. Chen, D.-T. Lin, and Y.-P. Hung
+    IEEE Transactions on Information Forensic and Security (TIFS), 2015  
+
+
+    Left-Luggage Detection from Finite-State-Machine Analysis in Static-Camera Videos
+    K. Lin, S.-C. Chen, C.-S. Chen, D.-T. Lin, and Y.-P. Hung
+    International Conference on Pattern Recognition (ICPR), 2014 
+
+
+## Prerequisites
+
+  0. OpenCV
+
+
+## Setup
+
+Simply run the following commands:
+
+    $ cmake .
+    $ make
+    $ ./download_video.sh
+
+
+## Demo
+
+This demo detect abandoned luggage in the video
+
+    $ ./ObjLeft
+
+Select the input source (1: video, 2: camera)
+
+    $ 1
+
+Input the filename if you choose video
+
+    $ pets2006_1.avi
+
+Click a rectangular Region of Interest (ROI) for detection
+
+<img src="https://www.csie.ntu.edu.tw/~r01944012/objleft_fig1.png" width="800">
+
+Double-press any key and start detection
+
+<img src="https://www.csie.ntu.edu.tw/~r01944012/objleft_fig2.jpg" width="800">
+
+This demo will detect the abandoned luggage and display the owner trajectory.
+
+
+## Contact
+
+Please feel free to leave suggestions or comments to Kevin Lin (kevinlin311.tw@iis.sinica.edu.tw)

VideoDetails.cpp

+#include "VideoDetails.h"
+using namespace std;
+VideoDetails::VideoDetails(char* filename)
+{
+	if (strcmp(filename, "")== 0)
+	{
+		_file.open(2);
+		_fps   =  30;
+	}
+	else{
+		_file.open(filename);
+		_fps   =  (int)_file.get(5);
+	}
+	_width =  (int)_file.get(3);
+	_height = (int)_file.get(4);
+	_frameNum = (int)_file.get(7);
+	//cvSetCaptureProperty( _file, CV_CAP_PROP_POS_FRAMES, 0 );/* Return to the beginning */
+	_frame = cv::Mat(cv::Size(_width,_height), CV_8UC3);//kv
+	// cout << "video fps = " << _fps << endl;
+	// cout << "video width = " << _width << endl;
+	// cout << "video height = " << _height << endl;
+}
+VideoDetails::~VideoDetails()
+{
+	//cvReleaseCapture(&_file);
+	// if (writer_create==true){
+	// 	writer_frame.release();
+	// }
+};
+
+// void VideoDetails::VideoWriter_Initial(cv::VideoWriter writer ,char* filename, int fps)
+// {
+// 	int AviForamt = 0;
+// 	//int fps = 60; // or 25 
+//     cv::Size AviSize = cv::Size( _width,_height);
+// 	int AviColor = 1; //0: binary  1:color
+// 	writer.open( filename,cv::VideoWriter::fourcc('X','V','I','D'),_fps,AviSize,AviColor);
+// 	writer_create = true;
+// }
\ No newline at end of file

VideoDetails.h

+#ifndef _VIDEO_DETAILS_H_
+#define _VIDEO_DETAILS_H_
+
+#include "opencvheader.h"
+
+class VideoDetails
+{
+public:
+	VideoDetails(char* filename);
+	~VideoDetails();
+
+	// void VideoWriter_Initial(cv::VideoWriter writer ,char* filename, int fps);
+	// cv::VideoWriter writer_frame; 
+	// cv::VideoWriter writer_GMM; 
+	// cv::VideoWriter writer_Diff3; 
+	// cv::VideoWriter writer_Foreground; 
+
+	bool writer_create;
+
+	int _currentFrame;  //record current frame
+	int _frameNum;      //record total frame number
+	int _fps;
+	cv::VideoCapture _file;
+	cv::Mat _frame; 
+	char _videoName[50];
+	double _width;
+	double _height;
+
+};
+#endif

connected.cc

+
+
+#include "connected.h"
+
+#define _TEST
+#ifdef _TEST
+
+#include <functional>
+#include <stdlib.h>
+#include <stdio.h>
+
+
+
+void test_inline_img()
+{
+    const  char *_img = {
+	"              "
+	"    *  0  *   "
+	"   **  0  *   "
+	"    *******   "
+	"      *       "
+	"   *          "
+	"  ***         "
+    };
+    const unsigned char *img = (const unsigned char *)_img;
+    int width = 14, height = 7;
+
+    unsigned char *out_uc = (unsigned char *)malloc(width*height);
+
+    ConnectedComponents cc(30);
+    cc.connected(img, out_uc, width, height,
+		 std::equal_to<unsigned char>(),
+		 false);
+
+    for(int r=0; r<height; ++r) {
+	for(int c=0; c<width; ++c)
+	    putchar('0'+out_uc[r*width+c]);
+	putchar('\n');
+    }
+
+    free(out_uc);
+}
+
+
+
+void test_raw_img(const char *infname, const char *outfname,
+		  int width, int height)
+{
+    unsigned char *img = (unsigned char *)malloc(width*height);
+
+    FILE *fin = fopen(infname, "r");
+    fread(img, 1, width*height, fin);
+
+    unsigned int *out = (unsigned int *)malloc(sizeof(*out)*width*height);
+
+    ConnectedComponents cc(30);
+    cc.connected(img, out, width, height,
+		 std::equal_to<unsigned char>(),
+		 constant<bool,true>());
+
+    unsigned char *out_uc = (unsigned char *)malloc(width*height);
+    std::copy(out, out+width*height, out_uc);
+    FILE *fout = fopen(outfname, "w");
+    fwrite(out_uc, 1, width*height, fout);
+
+    fclose(fout); fclose(fin);
+    free(img); free(out); free(out_uc);
+}
+
+
+// main()
+// {
+//     test_inline_img();
+//     test_raw_img("img.raw", "out.raw", 321, 241);
+// }
+
+#endif // _TEST

connected.h

+#ifndef _CONNECTED_H
+#define _CONNECTED_H
+
+
+#include <vector>
+#include <algorithm>
+
+template <class T, T V>
+struct constant
+{
+    operator T() const { return V; }
+};
+
+
+class ConnectedComponents
+{
+public:
+    ConnectedComponents(int soft_maxlabels) : labels(soft_maxlabels) {
+	clear();
+    }
+    void clear() {
+	std::fill(labels.begin(), labels.end(), Similarity());
+	highest_label = 0;
+    }
+    template<class Tin, class Tlabel, class Comparator, class Boolean>
+    int connected(const Tin *img, Tlabel *out,
+		     int width, int height, Comparator,
+		  Boolean K8_connectivity);
+
+
+private:
+    struct Similarity {
+	Similarity() : id(0), sameas(0) {}
+	Similarity(int _id, int _sameas) : id(_id), sameas(_sameas) {}
+	Similarity(int _id) : id(_id), sameas(_id) {}
+	int id, sameas, tag;
+    };
+
+    bool is_root_label(int id) {
+	return (labels[id].sameas == id);
+    }
+    int root_of(int id) {
+	while (!is_root_label(id)) {
+	    // link this MyNode to its parent's parent, just to shorten
+	    // the tree.
+	    labels[id].sameas = labels[labels[id].sameas].sameas;
+
+	    id = labels[id].sameas;
+	}
+	return id;
+    }
+    bool is_equivalent(int id, int as) {
+	return (root_of(id) == root_of(as));
+    }
+    bool merge(int id1, int id2) {
+	if(!is_equivalent(id1, id2)) {
+	    labels[root_of(id1)].sameas = root_of(id2);
+	    return false;
+	}
+	return true;
+    }
+    int new_label() {
+	if(highest_label+1 > labels.size())
+	    labels.reserve(highest_label*2);
+	labels.resize(highest_label+1);
+	labels[highest_label] = Similarity(highest_label);
+	return highest_label++;
+    }
+
+
+    template<class Tin, class Tlabel, class Comparator, class Boolean>
+    void label_image(const Tin *img, Tlabel *out,
+		     int width, int height, Comparator,
+		     Boolean K8_connectivity);
+    template<class Tlabel>
+    int relabel_image(Tlabel *out, int width, int height);
+
+
+    std::vector<Similarity> labels;
+    int highest_label;
+};
+
+template<class Tin, class Tlabel, class Comparator, class Boolean>
+int
+ConnectedComponents::connected(const Tin *img, Tlabel *labelimg,
+			       int width, int height, Comparator SAME,
+			       Boolean K8_connectivity)
+{
+    label_image(img,labelimg, width,height, SAME, K8_connectivity);
+    return relabel_image(labelimg, width,height);
+}
+
+
+
+
+template<class Tin, class Tlabel, class Comparator, class Boolean>
+void
+ConnectedComponents::label_image(const Tin *img, Tlabel *labelimg,
+				 int width, int height, Comparator SAME,
+				 const Boolean K8_CONNECTIVITY)
+{
+    const Tin *row = img;
+    const Tin *last_row = 0;
+    struct Label_handler {
+	Label_handler(const Tin *img, Tlabel *limg) :
+	    piximg(img), labelimg(limg) {}
+	Tlabel &operator()(const Tin *pixp) { return labelimg[pixp-piximg]; }
+	const Tin *piximg;
+	Tlabel *labelimg;
+    } label(img, labelimg);
+
+    clear();
+
+    label(&row[0]) = new_label();
+
+    // label the first row.
+    for(int c=1, r=0; c<width; ++c) {
+	if(SAME(row[c], row[c-1]))
+	    label(&row[c]) = label(&row[c-1]);
+	else
+	    label(&row[c]) = new_label();
+    }
+
+    // label subsequent rows.
+    for(int r=1; r<height; ++r)    {
+	// label the first pixel on this row.
+	last_row = row;
+	row = &img[width*r];
+
+	if(SAME(row[0], last_row[0]))
+	    label(&row[0]) = label(&last_row[0]);
+	else
+	    label(&row[0]) = new_label();
+
+	// label subsequent pixels on this row.
+	for(int c=1; c<width; ++c)	{
+	    int mylab = -1;
+
+	    // inherit label from pixel on the left if we're in the same blob.
+	    if(SAME(row[c],row[c-1]))
+		mylab = label(&row[c-1]);
+	    for(int d=(K8_CONNECTIVITY?-1:0); d<1; ++d) {
+		// if we're in the same blob, inherit value from above pixel.
+		// if we've already been assigned, merge its label with ours.
+		if(SAME(row[c], last_row[c+d])) {
+		    if(mylab>=0) merge(mylab, label(&last_row[c+d]));
+		    else mylab = label(&last_row[c+d]);
+		}
+	    }
+	    if(mylab>=0) label(&row[c]) = static_cast<Tlabel>(mylab);
+	    else label(&row[c]) = new_label();
+
+	    if(K8_CONNECTIVITY && SAME(row[c-1], last_row[c]))
+		merge(label(&row[c-1]), label(&last_row[c]));
+	}
+    }
+}
+
+template<class Tlabel>
+int
+ConnectedComponents::relabel_image(Tlabel *labelimg, int width, int height)
+{
+    int newtag = 0;
+    for(int id=0; id<labels.size(); ++id)
+	if(is_root_label(id))
+	    labels[id].tag = newtag++;
+
+    for(int i = 0; i<width*height; ++i)
+	labelimg[i] = labels[root_of(labelimg[i])].tag;
+
+    return newtag;
+}
+
+
+#endif // _CONNECTED_H

download_video.sh

+echo "Downloading video..."
+
+wget -O pets2006_1.avi "https://www.dropbox.com/s/hv994ztqdrmhlks/pets2006_1.avi?dl=1"
+
+
+

main.cpp

+#include "ObjLeftDetect.h"
+#include "VideoDetails.h"
+#include "DetectAnomaly.h"
+#include "parameter.h"
+#include <iostream>
+
+#include "yolov3/extra/api.hpp"
+
+using namespace std;
+
+
+// int64 work_begin;
+// double work_fps;
+
+// inline void WorkBegin()
+// {
+//     work_begin = getTickCount();
+// }
+
+// inline void WorkEnd()
+// {
+//     int64 delta = getTickCount() - work_begin;
+//     double freq = getTickFrequency();
+//     work_fps = freq / delta;
+// }
+
+// inline string WorkFps()
+// {
+//     stringstream ss;
+//     ss << work_fps;
+//     return ss.str();
+// }
+
+
+
+int main()
+{
+
+    //init
+    InitCnrt(false);
+    
+    int count = 0;
+    // char test_video[200] = "zhao.mp4";
+    char test_video[200] = "../test_video/temp-12252020164103.avi";
+    VideoDetails *_video;
+    _video = new VideoDetails(test_video);
+    //VideoCapture capture;
+    //capture.open("test_video/temp-12252020164103.avi");
+    // char test_video[200];
+    // sprintf(test_video, "");
+    cv::Mat qImg;
+    // config opt;
+    DetectAnomaly detect;
+#ifdef DEBUG
+    //cv::VideoWriter _writer;
+    //_writer.open("alarm.avi", cv::VideoWriter::fourcc('X', 'V', 'I', 'D'), 30, cv::Size(2448*0.25, 2048*0.25), 1);
+#endif
+    /************************************************************************/
+    /* main loop                                                       */
+    /************************************************************************/
+    bool obj_left = false;
+    while (_video->_file.read(qImg))
+    {
+        // cvResize(src_img,des_img,CV_INTER_LINEAR);
+        vector<vector<int>> resultLocation;
+
+        obj_left = detect.detectAnomaly(qImg.clone(),&resultLocation);
+
+
+        //yolo_infer
+        // vector<cv::Mat>* input_mats = new vector<cv::Mat>;
+        // input_mats->push_back(qImg.clone());
+        // std::map<int,Results*> results_map;
+        // Results* results = new Results;
+        // results_map.insert(std::make_pair(0,results));
+        // inference(&results_map,input_mats);
+        // delete results_map[0];
+        // delete input_mats;
+#ifdef DEBUG
+        if (obj_left == true)
+        {   
+            
+            printf("alram!!\n");
+
+            for(int i = 0; i < resultLocation.size();i++){
+                int x0 = resultLocation[i][0];
+                int y0 = resultLocation[i][1];
+                int x1 = resultLocation[i][2];
+                int y1 = resultLocation[i][3];
+
+                cv::Point p1(x0, y0);
+                cv::Point p2(x1, y1);
+                cv::rectangle(qImg, p1, p2, cv::Scalar(0, 255, 0), 2);
+            }
+            stringstream ss1;
+            string outFile;
+            ss1 << "yolov3_"
+                << "show"
+                << count
+                << ".jpg";
+            ss1 >> outFile;
+            cv::imwrite(outFile.c_str(), qImg);
+
+            count++;
+        }
+#endif
+    }
+#ifdef DEBUG
+    printf("Finished detec, then release ...\n");
+    
+#endif
+
+    //system("pause");
+    return 0;
+}

main_old.cpp

+#include "ObjLeftDetect.h"
+#include "VideoDetails.h"
+#include "parameter.h"
+#include <iostream>
+
+// #include "opencv/cv.h" // include standard OpenCV headers, same as before
+// #include "opencv/cxcore.h"
+// #include "opencv/highgui.h"
+// #include "opencv/cvaux.h"
+// #include "opencv/ml.h"
+
+using namespace std;
+// using namespace cv;
+int OWNER_SEARCH_ROI;			
+int GMM_LEARN_FRAME;//frame number for GMM initialization		
+int MAX_SFG;					
+int MIN_SFG;					
+int MAX_FG;						
+int MIN_FG;						
+int BUFFER_LENGTH;				
+double GMM_LONG_LEARN_RATE;		
+double GMM_SHORT_LEARN_RATE;	
+double INPUT_RESIZE;	
+
+
+
+int64 work_begin;
+double work_fps;
+
+int roipointnumber = 0;	
+int arr[1] = {4}; 
+CvPoint PointArray1[4];
+CvPoint *PointArray[2]= {&PointArray1[0],&PointArray1[2]};
+int imageheight;
+int imagewidth;
+IplImage *mask;
+myImage *mymask;
+void onMouse(int Event,int x,int y,int flags,void* param );
+
+void onMouse(int Event,int x,int y,int flags,void* param )
+{
+	if ( (Event == 1) && (roipointnumber <=3) )	
+	{
+		PointArray[roipointnumber/2][roipointnumber%2] = cvPoint(x,y);	
+		printf("( %d, %d)\n ",x,y);
+		printf("%d\n",roipointnumber);
+		roipointnumber = roipointnumber + 1;
+	}
+	if (  (Event == 1) && (roipointnumber >3))	
+	{
+		IplImage *Image1 = cvCreateImage(cvSize(imagewidth,imageheight), IPL_DEPTH_8U, 1);
+		cvPolyLine( Image1, PointArray, arr, 1, 5, CV_RGB(255,255,255));	
+		cvFillPoly(Image1,PointArray,arr,1,CV_RGB(255,255,255));	
+		cvThreshold(Image1,Image1,254,255,CV_THRESH_BINARY);	
+		mask = Image1;
+		opencv_2_myImage(mask,mymask);
+		cvSaveImage("mask.jpg",Image1);	
+		cvShowImage("SetROI",Image1);	
+		cvWaitKey(1);
+		cvReleaseImage(&Image1);
+	}
+}
+
+inline void WorkBegin() 
+{ 
+	work_begin = getTickCount(); 
+}
+
+inline void WorkEnd()
+{
+    int64 delta = getTickCount() - work_begin;
+    double freq = getTickFrequency();
+    work_fps = freq / delta;
+}
+
+inline string WorkFps()
+{
+    stringstream ss;
+    ss << work_fps;
+    return ss.str();
+}
+
+int main()
+{	
+	/************************************************************************/
+	/* parameter seeting                                                    */
+	/************************************************************************/
+	int count = 0;
+	double readData[10] = {0.0};
+	FILE *InputFile;  
+	InputFile = fopen( "parameter.txt","r");
+	if (InputFile == NULL)
+	{
+		printf("Loading parameter failed.\n");
+		system("pause");
+		return -1;
+	}
+	for (int i = 0; i < 10; i++ ){
+		fscanf( InputFile, "%lf", &readData[i]);
+	}
+	fclose(InputFile);
+	for (int i = 0; i < 10; i++)
+	{
+		switch(i)
+		{
+			case 0: OWNER_SEARCH_ROI = (int)readData[i]; break;
+			case 1: GMM_LEARN_FRAME = (int)readData[i]; break;
+			case 2: MAX_SFG = (int)readData[i]; break;
+			case 3: MIN_SFG = (int)readData[i]; break;
+			case 4: MAX_FG = (int)readData[i]; break;
+			case 5: MIN_FG = (int)readData[i]; break;
+			case 6: BUFFER_LENGTH = (int)readData[i]; break;
+			case 7: GMM_LONG_LEARN_RATE = readData[i]; break;
+			case 8: GMM_SHORT_LEARN_RATE = readData[i]; break;
+			case 9: INPUT_RESIZE = readData[i]; break;
+		}
+	}
+
+	/************************************************************************/
+	/* choose input channel                                                 */
+	/************************************************************************/
+	char test_video[200];
+	int method;
+	printf("1: video    2: camera\n");
+	scanf("%d",&method);
+	if (method==2){
+		sprintf(test_video, "");
+	}
+	else{
+		printf("please input video filename:  ***.avi or ***.mp4 \n");
+		scanf("%s",&test_video);
+	}
+
+	/************************************************************************/
+	/* Video input setting                                                   */
+	/************************************************************************/
+	VideoDetails *_video;
+	_video = new VideoDetails(test_video);
+	//_video = new VideoDetails("pets2006_1.avi");
+	IplImage *qImg, *myimg;
+	_video->_currentFrame = 0;
+	cvSetCaptureProperty(_video->_file, CV_CAP_PROP_POS_FRAMES, _video->_currentFrame);
+	myimg = cvCreateImage(cvSize(_video->_width,_video->_height),8,3);
+	cv::Mat mat_myimg(myimg,0);
+	myImage * myimg1 = myCreateImage(_video->_width,_video->_height,3);
+	CvVideoWriter *_writer;
+	_writer = cvCreateVideoWriter("video.avi",CV_FOURCC('X','V','I','D'),30,cvSize(_video->_width,_video->_height),1);
+
+	/************************************************************************/
+	/* ROI setting                                                          */
+	/************************************************************************/
+	imageheight = _video->_height*INPUT_RESIZE;
+	imagewidth = _video->_width*INPUT_RESIZE;
+	IplImage *setroi = cvQueryFrame(_video->_file);
+	IplImage *setroi2; 
+	setroi2 = cvCreateImage(cvSize(imagewidth,imageheight),8,3);
+	mymask = myCreateImage( imagewidth, imageheight, 3);
+	myInverse(mymask,mymask) ;
+	cvResize(setroi,setroi2);
+	cvShowImage("SetROI",setroi2);
+	cvSetMouseCallback("SetROI",onMouse,NULL);	
+	cvWaitKey(1);
+	cvDestroyWindow("SetROI");	
+	
+	/************************************************************************/
+	/* counstruct object left class                                         */
+	/************************************************************************/
+	ObjLeftDetect _objleft(myimg1,GMM_LEARN_FRAME,MIN_FG,BUFFER_LENGTH,mymask);
+	
+
+	/************************************************************************/
+	/* main loop                                                       */
+	/************************************************************************/
+	bool obj_left = false;
+	while(qImg = cvQueryFrame(_video->_file))
+	{		
+		WorkBegin();
+		cvCopy(qImg,myimg);
+		medianBlur( mat_myimg, mat_myimg, 3);
+		opencv_2_myImage(myimg,myimg1);//transfer opencv data to myimage data
+
+		/************************************************************************/
+		/* abandoned object detection algorithm                                 */
+		/************************************************************************/
+		obj_left = _objleft.process(myimg1);
+
+		if (obj_left==true)
+		{
+			printf("alram!!\n");
+		}
+		WorkEnd();
+		Mat _qImg(qImg);
+		putText(_qImg, "fps:" + WorkFps(), Point(5, 20), FONT_HERSHEY_SIMPLEX, 0.9, Scalar(255, 100, 0), 2);
+		cvShowImage( "video",qImg);
+		cvShowImage("video",qImg);
+		cvWriteFrame( _writer, qImg);	
+		cvWaitKey(1);
+	}
+	myimg1->myReleaseImage();
+	cvReleaseImage(&qImg);
+	cvReleaseImage(&myimg);
+	cvReleaseVideoWriter(&_writer);
+	system("pause");
+	return 0;
+}

main_useful.cpp

+#include "ObjLeftDetect.h"
+#include "VideoDetails.h"
+#include "parameter.h"
+#include <iostream>
+
+using namespace std;
+
+int OWNER_SEARCH_ROI;
+int GMM_LEARN_FRAME; //frame number for GMM initialization
+int MAX_SFG;
+int MIN_SFG;
+int MAX_FG;
+int MIN_FG;
+int BUFFER_LENGTH;
+double GMM_LONG_LEARN_RATE;
+double GMM_SHORT_LEARN_RATE;
+double INPUT_RESIZE;
+
+int64 work_begin;
+double work_fps;
+
+int roipointnumber = 0;
+int arr[1] = {4};
+CvPoint PointArray1[4];
+CvPoint *PointArray[2] = {&PointArray1[0], &PointArray1[2]};
+int imageheight;
+int imagewidth;
+IplImage *mask;
+myImage *mymask;
+
+void setRoi();
+void setRoi()
+{
+    // pets
+    // PointArray[0][0] = cvPoint(3, 91);
+    // PointArray[0][1] = cvPoint(657, 157);
+    // PointArray[1][1] = cvPoint(3, 565);
+    // PointArray[1][0] = cvPoint(705,565);
+
+    // zhaoshang
+    PointArray[0][0] = cvPoint(567, 635);
+    PointArray[0][1] = cvPoint(1009, 625);
+    PointArray[1][1] = cvPoint(9, 859);
+    PointArray[1][0] = cvPoint(1427, 1069);
+
+    IplImage *Image1 = cvCreateImage(cvSize(imagewidth, imageheight), IPL_DEPTH_8U, 1);
+    cvPolyLine(Image1, PointArray, arr, 1, 5, CV_RGB(255, 255, 255));
+    cvFillPoly(Image1, PointArray, arr, 1, CV_RGB(255, 255, 255));
+    cvThreshold(Image1, Image1, 254, 255, CV_THRESH_BINARY);
+    mask = Image1;
+    opencv_2_myImage(mask, mymask);
+    cvSaveImage("mask.jpg", Image1);
+    cvWaitKey(1);
+    cvReleaseImage(&Image1);
+}
+
+inline void WorkBegin()
+{
+    work_begin = getTickCount();
+}
+
+inline void WorkEnd()
+{
+    int64 delta = getTickCount() - work_begin;
+    double freq = getTickFrequency();
+    work_fps = freq / delta;
+}
+
+inline string WorkFps()
+{
+    stringstream ss;
+    ss << work_fps;
+    return ss.str();
+}
+
+int main()
+{
+    /************************************************************************/
+    /* parameter seeting                                                    */
+    /************************************************************************/
+    int count = 0;
+    double readData[10] = {0.0};
+    FILE *InputFile;
+    InputFile = fopen("parameter_zhaoshang.txt", "r");
+    if (InputFile == NULL)
+    {
+        printf("Loading parameter failed.\n");
+        system("pause");
+        return -1;
+    }
+    for (int i = 0; i < 10; i++)
+    {
+        fscanf(InputFile, "%lf", &readData[i]);
+    }
+    fclose(InputFile);
+    for (int i = 0; i < 10; i++)
+    {
+        switch (i)
+        {
+        case 0:
+            OWNER_SEARCH_ROI = (int)readData[i];
+            break;
+        case 1:
+            GMM_LEARN_FRAME = (int)readData[i];
+            break;
+        case 2:
+            MAX_SFG = (int)readData[i];
+            break;
+        case 3:
+            MIN_SFG = (int)readData[i];
+            break;
+        case 4:
+            MAX_FG = (int)readData[i];
+            break;
+        case 5:
+            MIN_FG = (int)readData[i];
+            break;
+        case 6:
+            BUFFER_LENGTH = (int)readData[i];
+            break;
+        case 7:
+            GMM_LONG_LEARN_RATE = readData[i];
+            break;
+        case 8:
+            GMM_SHORT_LEARN_RATE = readData[i];
+            break;
+        case 9:
+            INPUT_RESIZE = readData[i];
+            break;
+        }
+    }
+
+    /************************************************************************/
+    /* choose input channel                                                 */
+    /************************************************************************/
+    // char test_video[200] = "zhao.mp4";
+    char test_video[200] = "test_video/day-out.mp4";
+    int method;
+    printf("1: video    2: camera\n");
+    scanf("%d", &method);
+    if (method == 2)
+    {
+        sprintf(test_video, "");
+    }
+    else
+    {
+        printf("please input video filename:  ***.avi or ***.mp4 \n");
+        // scanf("%s", &test_video);
+    }
+
+    /************************************************************************/
+    /* Video input setting                                                   */
+    /************************************************************************/
+    VideoDetails *_video;
+    _video = new VideoDetails(test_video);
+    //_video = new VideoDetails("pets2006_1.avi");
+    IplImage *qImg, *myimg;
+    _video->_currentFrame = 0;
+    cvSetCaptureProperty(_video->_file, CV_CAP_PROP_POS_FRAMES, _video->_currentFrame);
+    myimg = cvCreateImage(cvSize(_video->_width, _video->_height), 8, 3);
+    cv::Mat mat_myimg(myimg, 0);
+    myImage *myimg1 = myCreateImage(_video->_width, _video->_height, 3);
+    CvVideoWriter *_writer;
+    _writer = cvCreateVideoWriter("video.avi", CV_FOURCC('X', 'V', 'I', 'D'), 30, cvSize(_video->_width, _video->_height), 1);
+    printf("222222222\n");
+    /************************************************************************/
+    /* ROI setting                                                          */
+    /************************************************************************/
+    imageheight = _video->_height * INPUT_RESIZE;
+    imagewidth = _video->_width * INPUT_RESIZE;
+    IplImage *setroi = cvQueryFrame(_video->_file);
+    IplImage *setroi2;
+    setroi2 = cvCreateImage(cvSize(imagewidth, imageheight), 8, 3);
+    mymask = myCreateImage(imagewidth, imageheight, 3);
+    myInverse(mymask, mymask);
+    // cvResize(setroi, setroi2);
+    // cvShowImage("SetROI", setroi2);
+    // cvSetMouseCallback("SetROI", onMouse, NULL);
+    setRoi();
+    cvWaitKey(1);
+    // cvDestroyWindow("SetROI");
+    printf("11111111111\n");
+
+    /************************************************************************/
+    /* counstruct object left class                                         */
+    /************************************************************************/
+    ObjLeftDetect _objleft(myimg1, GMM_LEARN_FRAME, MIN_FG, BUFFER_LENGTH, mymask);
+    printf("333333333333\n");
+    /************************************************************************/
+    /* main loop                                                       */
+    /************************************************************************/
+    bool obj_left = false;
+    while (qImg = cvQueryFrame(_video->_file))
+    {
+        WorkBegin();
+        cvCopy(qImg, myimg);
+        medianBlur(mat_myimg, mat_myimg, 3);
+        opencv_2_myImage(myimg, myimg1); //transfer opencv data to myimage data
+
+        /************************************************************************/
+        /* abandoned object detection algorithm                                 */
+        /************************************************************************/
+        obj_left = _objleft.process(myimg1);
+
+        if (obj_left == true)
+        {
+            printf("alram!!\n");
+        }
+        WorkEnd();
+        // printf("finished detect\n");
+        Mat _qImg(qImg);
+        putText(_qImg, "fps:" + WorkFps(), Point(5, 20), FONT_HERSHEY_SIMPLEX, 0.9, Scalar(255, 100, 0), 2);
+        // cvShowImage("video", qImg);
+        // cvShowImage("video", qImg);
+        cvWriteFrame(_writer, qImg);
+        cvWaitKey(1);
+    }
+    printf("Finished detec, then release ...\n");
+    myimg1->myReleaseImage();
+    cvReleaseImage(&qImg);
+    cvReleaseImage(&myimg);
+    cvReleaseVideoWriter(&_writer);
+    system("pause");
+    return 0;
+}

myDIP.h

+/************************************************************************/
+/* This code was created by Kevin (Ke-Yun) Lin at October, 2013.  
+   myDIP includes many basic image processing tools
+   feel free to contact me should you have any question!    
+   email: kevinlin311.tw[at]gmail.com                                   */
+/************************************************************************/
+  
+#ifndef _MY_DIP_H 
+#define _MY_DIP_H 
+  
+#include <stdio.h> 
+#include <vector> 
+#include <list> 
+#include <algorithm> 
+#include <math.h> 
+#include <iostream> 
+#include <time.h> 
+#include <limits.h> 
+#include <omp.h> 
+#include <stdexcept> 
+  
+#include <opencv2/opencv.hpp>
+#include <opencv2/highgui/highgui.hpp>
+
+void *** Memory_Allocate(int width, int height, int depth); 
+void Memory_Free(unsigned char ***pixel_index,int width, int height); 
+void * Memory_Allocate1D(int width, int height, int depth); 
+void Memory_Free1D(unsigned char *pixel_index,int width, int height); 
+  
+struct myImage 
+{ 
+    //unsigned char *** imageData; 
+    unsigned char * pixelData; 
+    int width; 
+    int height; 
+    int depth; 
+  
+    myImage(int set_width, int set_height, int set_depth){ 
+        width = set_width; 
+        height = set_height; 
+        depth = set_depth; 
+        pixelData = (unsigned char *)Memory_Allocate1D( width, height, depth); 
+        //imageData = (unsigned char***)Memory_Allocate( width, height, depth); 
+    } 
+  
+public: 
+    void myReleaseImage(){ 
+        Memory_Free1D(pixelData, width, height); 
+    } 
+}; 
+  
+struct myColor 
+{ 
+    unsigned char B; unsigned char G;  unsigned char R; 
+}; 
+  
+struct myFloatColor
+{
+	double B; double G;  double R;
+};
+void myInverse(myImage * input, myImage * output); 
+void myBinary(myImage * input, myImage * output, int threshold); 
+void myImageAND(myImage * input1, myImage * input2, myImage * output); 
+void myRGB2Gray(myImage  * input, myImage * output); 
+void myImageCopy(myImage * input, myImage * output); 
+void myImageZero(myImage * input); 
+myColor myGet2D(myImage * input, int x, int y); 
+void mySet2D(myImage * input, myColor colors, int x, int y); 
+  
+void opencv_2_myImage(cv::Mat input, myImage * output);//�u�O�ΨӼv����ƪ��ഫ 
+void myImage_2_opencv(myImage * input, cv::Mat output);//�u�O�ΨӼv����ƪ��ഫ 
+  
+void myResize( myImage * input, myImage * output); 
+void myErode( myImage * input, myImage * output);
+void myDilade( myImage * input, myImage * output);
+  
+myImage * myCreateImage(int width, int height, int depth); 
+int myCountNonZero(myImage * input);
+void myAbsDiff(myImage * input1, myImage * input2, myImage * output);
+  
+void myMedianBlur( myImage * input, myImage * output); 
+void mymedianFilter(unsigned char* corrupted, unsigned char* smooth, int width, int height); 
+
+void myredmask( myImage * input, myImage * mask );
+void mygreenmask( myImage * input, myImage * mask );
+void mybluemask( myImage * input, myImage * mask );
+
+void myErodeitself( myImage * input,int n);
+void myDiladeitself( myImage * input, int n);
+  
+#endif

myGMM.h

+/************************************************************************/
+/* Acknowledgements: This GMM code was slightly modified by Kevin (Ke-Yun) Lin
+   based on the source code from Gurpinder Singh Sandhu
+   The original version can be found here 
+   https://github.com/gpsinghsandhu/Background-Subtraction-using-GMM    */
+/************************************************************************/
+
+// /*IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
+// 
+// By downloading, copying, installing or using the software you agree to this license.
+// If you do not agree to this license, do not download, install,
+// copy or use the software.
+// 
+// 
+// License Agreement
+// 
+// Copyright (C) 2013, Gurpinder Singh Sandhu, all rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification,
+// are permitted provided that the following conditions are met:
+// 
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// 
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// 
+// * The name of the copyright holders may not be used to endorse or promote products
+// derived from this software without specific prior written permission.
+// 
+// This software is provided by the copyright holders and contributors "as is" and
+// any express or implied warranties, including, but not limited to, the implied
+// warranties of merchantability and fitness for a particular purpose are disclaimed.
+// In no event shall Gurpinder Singh Sandhu be liable for any direct,
+// indirect, incidental, special, exemplary, or consequential damages
+// (including, but not limited to, procurement of substitute goods or services;
+// loss of use, data, or profits; or business interruption) however caused
+// and on any theory of liability, whether in contract, strict liability,
+// or tort (including negligence or otherwise) arising in any way out of
+// the use of this software, even if advised of the possibility of such damage.*/
+
+
+
+#define USE_OPENCV  //LOOK LOOK LOOK HERE: If you don't want to use opencv library, just comment this declaration! "USE_OPENCV" 
+
+#ifndef _MYGMM_H_
+#define _MYGMM_H_
+
+#include "myDIP.h"
+//#include <conio.h>//for MS
+
+#ifdef USE_OPENCV
+	#include <opencv2/core/core.hpp>
+	#include <opencv2/imgproc/imgproc.hpp>
+	#include <opencv2/highgui/highgui.hpp>
+	#include <opencv2/features2d/features2d.hpp>
+	using namespace cv;
+#endif
+
+//Structure used for saving various components for each pixel
+struct gaussian
+{								
+	double mean[3], covariance;
+	double weight;	// Represents the measure to which a particular component defines the pixel value
+	gaussian* Next;
+	gaussian* Previous;
+
+	gaussian()
+	{
+		mean[0] = 0.0;mean[1] = 0.0;mean[2] = 0.0;covariance = 0.0;weight = 0.0;
+	};
+};
+
+struct MyNode
+{
+	gaussian* pixel_s;
+	gaussian* pixel_r;
+	int no_of_components;
+	MyNode* Next;
+};
+
+class myGMM 
+{
+public:
+	gaussian  *ptr, *start, *rear, *g_temp, *save, *next, *previous, *nptr, *temp_ptr;
+	MyNode  *N_ptr,*N_rear, *N_start;//, 
+
+	//Some constants for the algorithm
+	double pi;
+	double cthr;
+	double alpha;
+	double cT;
+	double covariance0;
+	double cf;
+	double cfbar;
+	double temp_thr;
+	double prune;
+	double alpha_bar;
+	//Temperory variable
+	int overall;
+	
+	double del[3], mal_dist;
+	double sum;
+	double sum1;
+	int count;
+	bool close;
+	int background;
+	double mult;
+	double duration,duration1,duration2,duration3;
+	double temp_cov;
+	double weight;
+	double var;
+	double muR,muG,muB,dR,dG,dB,rVal,gVal,bVal;
+
+	unsigned char * r_ptr;
+	unsigned char * b_ptr;
+	
+	//Some function associated with the structure management
+	MyNode* Create_MyNode(double info1, double info2, double info3);
+	void Insert_End_MyNode(MyNode* np);
+	gaussian* Create_gaussian(double info1, double info2, double info3);
+	void Insert_End_gaussian(gaussian* nptr);
+	gaussian* Delete_gaussian(gaussian* nptr);
+
+	//main founction of GMM
+	myGMM(double LearningRate);
+	~myGMM();
+	void process(myImage * inputRGB, myImage * outputBIN);
+	void initial(myImage * inputRGB);
+	void ChangeLearningRate( float new_learn_rate);
+
+#ifdef USE_OPENCV
+	void process(Mat orig_img, Mat bin_img);//this function is for opencv user
+	void initial(Mat orig_img);//this function is for opencv user
+#endif
+
+private:
+};
+
+#endif

myUtility.h

+#ifndef _MY_UTILITY_H
+#define _MY_UTILITY_H
+
+#include <vector>
+#include <utility>
+#include <opencv2/opencv.hpp>
+#include <opencv2/core/types.hpp>
+
+#define LF_ALPHA 0.3    //learning rate of leader follower
+#define  PI 3.1415926
+
+
+struct kmeansData
+{
+	double **means;
+	int *index;
+};
+int LeaderFollower1D(std::vector<float> &cluster, float &samples, float &thresh);
+
+int otsu(int *hist, int grid);
+
+kmeansData kmeans(double** data, int row, int col, int k);
+
+int nearestNeighbor(double* sample, double **data, int row, int col);
+
+float point_dist(float x1, float y1, float x2, float y2);
+
+float path_Length(std::vector<cv::Point> path);
+
+std::pair<int, int> path_Tag(std::vector<cv::Point> path, float reserveRatio);
+
+std::vector<int> path_resample_id(std::vector<cv::Point> path, float reserveRatio, int reLength);
+
+cv::Scalar Get2D(cv::Mat img, int i, int j);
+
+void Set2D(cv::Mat img ,int i, int j, cv::Scalar pt);
+
+void Set2D_ADD(cv::Mat img ,int i, int j, cv::Scalar pt);
+
+int update_min_index(double*, int length);
+
+int myRound(double value);
+
+cv::Point cv_center(cv::Rect rect);
+
+float findTheta(cv::Point, cv::Point);
+
+int findMaxIdx(double *, int dim);
+
+void new_findMaxIdx(double *data, int dim, int * output);
+
+void myMemset(double *arr, double value, int dim);
+
+
+#endif

parameter.h

+extern int OWNER_SEARCH_ROI;			// 以遺留物為中心,建立一個搜尋owner的圓形ROI, 此參數為圓形ROI的半徑長度(pixel單位)
+extern int GMM_LEARN_FRAME;				// 系統初始化,用多少張frame學習背景
+extern int MAX_SFG;						// 遺留物最大閥值(pixel單位)
+extern int MIN_SFG;						// 遺留物最小閥值(pixel單位)
+extern int MAX_FG;						// 前景最大閥值(pixel單位)
+extern int MIN_FG;						// 前景最小閥值(pixel單位)
+extern int BUFFER_LENGTH;				// 暫存前景資訊的buffer
+extern double GMM_LONG_LEARN_RATE;		// Long term model學習速度
+extern double GMM_SHORT_LEARN_RATE;		// Short term model學習速度
+extern double INPUT_RESIZE;				// 將輸入影像縮小的倍數, 可以加速演算法處裡
\ No newline at end of file

parameter.txt

+50
+500
+120
+10
+1000
+20
+900
+0.0001
+0.002
+1.0
+
+1. OWNER_SEARCH_ROI: �H��d��������,�إߤ@�ӷj�Mowner�����ROI, ���ѼƬ����ROI���b�|����(pixel���)
+2. GMM_LEARN_FRAME: �t�Ϊ�l��,�Φh�ֱiframe�Dz߭I��
+3. MAX_SFG: ��d���̤j�֭�(pixel���)
+4. MIN_SFG: ��d���̤p�֭�(pixel���)
+5. MAX_FG: �e���̤j�֭�(pixel���)
+6. MIN_FG: �e���̤p�֭�(pixel���)
+7. BUFFER_LENGTH: �Ȧs�e����T��buffer
+8. GMM_LONG_LEARN_RATE:  Long term model�Dz߳t��
+9. GMM_SHORT_LEARN_RATE:  Short term model�Dz߳t��
+10. INPUT_RESIZE: �N��J�v���Y�p������, �i�H�[�t�t��k�B��
+
+50
+500
+120
+15
+1000
+20
+900
+0.0001
+0.002
+0.5
+
+
+

process.py

+import os
+import glob
+import shutil
+
+
+with open('train_test_split/vehicle_info.txt') as f:
+    lines = f.readlines()
+    fline = lines[0]
+    img = fline.strip().split(';')[0]
+    classname = fline.strip().split(';')[-2]
+    print('path = {}, class = {}',format(img, classname))
+    
\ No newline at end of file

yolov3/LICENSE

+MIT License
+
+Copyright (c) 2020 enazoe
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.