Skip to content

O
opencv_contrib
Commit b831fc3b authored by Wangyida's avatar Wangyida
Browse files
Options

remove leveldb dependency, using Input/OutputArray for feature extraction, add… 

remove leveldb dependency, using Input/OutputArray for feature extraction, add the newest model, format fix for OpenCV
parent 4fe5498a
Expand all Show whitespace changes
Inline Side-by-side
Showing with 58 additions and 711 deletions
modules/cnn_3dobj/CMakeLists.txt
View file @ b831fc3b
set(the_description "CNN for 3D object recognition and pose estimation including a completed Sphere View on 3D objects")
ocv_define_module(cnn_3dobj opencv_core opencv_imgproc opencv_viz opencv_highgui caffe protobuf leveldb glog OPTIONAL WRAP python)
target_link_libraries(opencv_cnn_3dobj caffe protobuf leveldb glog)
ocv_define_module(cnn_3dobj opencv_core opencv_imgproc opencv_viz opencv_highgui caffe protobuf glog OPTIONAL WRAP python)
target_link_libraries(opencv_cnn_3dobj caffe protobuf glog)
modules/cnn_3dobj/README.md
View file @ b831fc3b
......@@ -63,27 +63,11 @@ $ ./examples/triplet/train_3d_triplet.sh
###After doing this, you will get .caffemodel files as the trained net work. I have already provide the net definition .prototxt files and the trained .caffemodel in <opencv_contrib>/modules/cnn_3dobj/samples/build folder, you could just use them without training in caffe. If you are not interested on feature analysis with the help of binary files provided in Demo2, just skip to Demo3 for feature extraction or Demo4 for classifier.
==============
#Demo4:
```
$ cd
$ cd <opencv_contrib>/modules/cnn_3dobj/samples/build
```
#Demo2:
###Convert data into leveldb format from folder ../data/images_all for feature extraction afterwards. The leveldb files including all data will be stored in ../data/dbfile. If you will use the OpenCV defined feature extraction process, you could also skip Demo2 for data converting, just run Demo3 after Demo1 for feature extraction because Demo3 also includes the db file converting process before feature extraction, but if you want to use demo4 for classification, this demo will be used in advance to generate a file name list for the prediction list.
```
$ ./datatrans_test
```
==============
#Demo3:
###feature extraction, this demo will convert a set of images in a particular path into leveldb database for feature extraction using Caffe and outputting a binary file including all extracted feature.
```
$ ./feature_extract_test
```
###This will extract feature from a set of images in a folder as vector<cv::Mat> for further classification and a binary file with containing all feature vectors of each sample.
###After running this, you will get a binary file storing features in ../data/feature folder, I can provide a Matlab script reading this file if someone need it. If you don't need the binary file, the feature could also be stored in vector<cv::Mat>.
==============
#Demo4:
###Classifier, this will extracting the feature of a single image and compare it with features of gallery samples for prediction. Demo2 should be used in advance to generate a file name list for the prediction list. This demo uses a set of images for feature extraction in a given path, these features will be a reference for prediction on target image. Just run:
```
$ ./classify_test
......
modules/cnn_3dobj/include/opencv2/cnn_3dobj.hpp
View file @ b831fc3b
......@@ -57,16 +57,13 @@ the use of this software, even if advised of the possibility of such damage.
#include <stdlib.h>
#include <tr1/memory>
#include <dirent.h>
#include <glog/logging.h>
#include <google/protobuf/text_format.h>
#include <leveldb/db.h>
#define CPU_ONLY
#include <caffe/blob.hpp>
#include <caffe/common.hpp>
#include <caffe/net.hpp>
#include <caffe/proto/caffe.pb.h>
#include <caffe/util/io.hpp>
#include <caffe/vision_layers.hpp>
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/net.hpp"
#include "caffe/proto/caffe.pb.h"
#include "caffe/util/io.hpp"
#include "caffe/vision_layers.hpp"
#include "opencv2/viz/vizcore.hpp"
#include "opencv2/highgui.hpp"
#include "opencv2/highgui/highgui_c.h"
......@@ -135,33 +132,6 @@ class CV_EXPORTS_W IcoSphere
};
class CV_EXPORTS_W DataTrans
{
private:
std::set<string> all_class_name;
std::map<string,int> class2id;
public:
DataTrans();
CV_WRAP void list_dir(const char *path,std::vector<string>& files,bool r);
/** @brief Use directory of the file including images starting with an int label as the name of each image.
*/
CV_WRAP string get_classname(string path);
/** @brief
*/
CV_WRAP int get_labelid(string fileName);
/** @brief Get the label of each image.
*/
CV_WRAP void loadimg(string path,char* buffer,bool is_color);
/** @brief Load images.
*/
CV_WRAP void convert(string imgdir,string outputdb,string attachdir,int channel,int width,int height);
/** @brief Convert a set of images as a leveldb database for CNN training.
*/
CV_WRAP std::vector<cv::Mat> feature_extraction_pipeline(std::string pretrained_binary_proto, std::string feature_extraction_proto, std::string save_feature_dataset_names, std::string extract_feature_blob_names, int num_mini_batches, std::string device, int dev_id);
/** @brief Extract feature into a binary file and vector<cv::Mat> for classification, the model proto and network proto are needed, All images in the file root will be used for feature extraction.
*/
};
class CV_EXPORTS_W Classification
{
private:
......@@ -180,13 +150,20 @@ class CV_EXPORTS_W Classification
/** @brief Convert the input image to the input image format of the network.
*/
public:
Classification(const string& model_file, const string& trained_file, const string& mean_file, const string& label_file);
Classification();
void list_dir(const char *path,std::vector<string>& files,bool r);
/** @brief Get the file name from a root dictionary.
*/
void NetSetter(const string& model_file, const string& trained_file, const string& mean_file, const string& cpu_only, int device_id);
/** @brief Initiate a classification structure.
*/
std::vector<std::pair<string, float> > Classify(const std::vector<cv::Mat>& reference, const cv::Mat& img, int N = 4, bool mean_substract = false);
void GetLabellist(const std::vector<string>& name_gallery);
/** @brief Get the label of the gallery images for result displaying in prediction.
*/
std::vector<std::pair<string, float> > Classify(const cv::Mat& reference, const cv::Mat& img, int N, bool mean_substract = false);
/** @brief Make a classification.
*/
cv::Mat feature_extract(const cv::Mat& img, bool mean_subtract);
void FeatureExtract(InputArray inputimg, OutputArray feature, bool mean_subtract);
/** @brief Extract a single featrue of one image.
*/
std::vector<int> Argmax(const std::vector<float>& v, int N);
......
modules/cnn_3dobj/samples/CMakeLists.txt
View file @ b831fc3b
......@@ -3,19 +3,11 @@ SET(CMAKE_CXX_FLAGS_DEBUG "$ENV{CXXFLAGS} -O0 -Wall -g -ggdb ")
SET(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O3 -Wall")
project(sphereview_test)
find_package(OpenCV REQUIRED)
set(SOURCES_1 sphereview_3dobj_demo.cpp)
set(SOURCES_generator sphereview_3dobj_demo.cpp)
include_directories(${OpenCV_INCLUDE_DIRS})
add_executable(sphereview_test ${SOURCES_1})
add_executable(sphereview_test ${SOURCES_generator})
target_link_libraries(sphereview_test ${OpenCV_LIBS})
set(SOURCES_2 datatrans_demo.cpp)
add_executable(datatrans_test ${SOURCES_2})
target_link_libraries(datatrans_test ${OpenCV_LIBS})
set(SOURCES_3 feature_extract_demo.cpp)
add_executable(feature_extract_test ${SOURCES_3})
target_link_libraries(feature_extract_test ${OpenCV_LIBS})
set(SOURCES_4 classifyIMG_demo.cpp)
add_executable(classify_test ${SOURCES_4})
set(SOURCES_classifier classifyIMG_demo.cpp)
add_executable(classify_test ${SOURCES_classifier})
target_link_libraries(classify_test ${OpenCV_LIBS})
modules/cnn_3dobj/samples/classifyDB_demo.cpp deleted 100644 → 0
View file @ 4fe5498a
/*
* Software License Agreement (BSD License)
*
* Copyright (c) 2009, Willow Garage, Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
* * Neither the name of Willow Garage, Inc. nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
*/
#include <opencv2/cnn_3dobj.hpp>
#include <iomanip>
using namespace cv;
using namespace std;
using namespace cv::cnn_3dobj;
int main(int argc, char** argv)
{
const String keys = "{help | | this demo will convert a set of images in a particular path into leveldb database for feature extraction using Caffe.}"
"{src_dir | ../data/images_all/ | Source direction of the images ready for being converted to leveldb dataset.}"
"{src_dst | ../data/dbfile | Aim direction of the converted to leveldb dataset. }"
"{attach_dir | ../data/dbfile | Path for saving additional files which describe the transmission results. }"
"{channel | 1 | Channel of the images. }"
"{width | 64 | Width of images}"
"{height | 64 | Height of images}"
"{caffemodel | ../data/3d_triplet_iter_10000.caffemodel | caffe model for feature exrtaction.}"
"{network_forDB | ../data/3d_triplet_galleryIMG.prototxt | Network definition file used for extracting feature from levelDB data, causion: the path of levelDB training samples must be wrotten in in .prototxt files in Phase TEST}"
"{save_feature_dataset_names | ../data/feature/feature_iter_10000.bin | Output of the extracted feature in form of binary files together with the vector<cv::Mat> features as the feature.}"
"{extract_feature_blob_names | feat | Layer used for feature extraction in CNN.}"
"{num_mini_batches | 4 | Batches suit for the batches defined in the .proto for the aim of extracting feature from all images.}"
"{device | CPU | Device: CPU or GPU.}"
"{dev_id | 0 | ID of GPU.}"
"{network_forIMG | ../data/3d_triplet_testIMG.prototxt | Network definition file used for extracting feature from a single image and making a classification}"
"{mean_file | ../data/images_mean/triplet_mean.binaryproto | The mean file generated by Caffe from all gallery images, this could be used for mean value substraction from all images.}"
"{label_file | ../data/dbfileimage_filename | A namelist including all gallery images.}"
"{target_img | ../data/images_all/2_13.png | Path of image waiting to be classified.}"
"{num_candidate | 6 | Number of candidates in gallery as the prediction result.}";
cv::CommandLineParser parser(argc, argv, keys);
parser.about("Demo for Sphere View data generation");
if (parser.has("help"))
{
parser.printMessage();
return 0;
}
string src_dir = parser.get<string>("src_dir");
string src_dst = parser.get<string>("src_dst");
string attach_dir = parser.get<string>("attach_dir");
int channel = parser.get<int>("channel");
int width = parser.get<int>("width");
int height = parser.get<int>("height");
string caffemodel = parser.get<string>("caffemodel");
string network_forDB = parser.get<string>("network_forDB");
string save_feature_dataset_names = parser.get<string>("save_feature_dataset_names");
string extract_feature_blob_names = parser.get<string>("extract_feature_blob_names");
int num_mini_batches = parser.get<int>("num_mini_batches");
string device = parser.get<string>("device");
int dev_id = parser.get<int>("dev_id");
string network_forIMG = parser.get<string>("network_forIMG");
string mean_file = parser.get<string>("mean_file");
string label_file = parser.get<string>("label_file");
string target_img = parser.get<string>("target_img");
int num_candidate = parser.get<int>("num_candidate");
cv::cnn_3dobj::DataTrans transTemp;
transTemp.convert(src_dir,src_dst,attach_dir,channel,width,height);
std::vector<cv::Mat> feature_reference = transTemp.feature_extraction_pipeline(caffemodel, network_forDB, save_feature_dataset_names, extract_feature_blob_names, num_mini_batches, device, dev_id);
////start another demo
cv::cnn_3dobj::Classification classifier(network_forIMG, caffemodel, mean_file, label_file);
std::cout << std::endl << "---------- Prediction for "
<< target_img << " ----------" << std::endl;
cv::Mat img = cv::imread(target_img, -1);
// CHECK(!img.empty()) << "Unable to decode image " << target_img;
std::cout << std::endl << "---------- Featrue of gallery images ----------" << std::endl;
std::vector<std::pair<string, float> > prediction;
for (unsigned int i = 0; i < feature_reference.size(); i++)
std::cout << feature_reference[i] << endl;
cv::Mat feature_test = classifier.feature_extract(img, false);
std::cout << std::endl << "---------- Featrue of target image: " << target_img << "----------" << endl << feature_test.t() << std::endl;
prediction = classifier.Classify(feature_reference, img, num_candidate, false);
// Print the top N prediction.
std::cout << std::endl << "---------- Prediction result(distance - file name in gallery) ----------" << std::endl;
for (size_t i = 0; i < prediction.size(); ++i) {
std::pair<string, float> p = prediction[i];
std::cout << std::fixed << std::setprecision(2) << p.second << " - \""
<< p.first << "\"" << std::endl;
}
return 0;
}
modules/cnn_3dobj/samples/classifyIMG_demo.cpp
View file @ b831fc3b
......@@ -44,9 +44,10 @@ int main(int argc, char** argv)
"{caffemodel | ../data/3d_triplet_iter_10000.caffemodel | caffe model for feature exrtaction.}"
"{network_forIMG | ../data/3d_triplet_testIMG.prototxt | Network definition file used for extracting feature from a single image and making a classification}"
"{mean_file | ../data/images_mean/triplet_mean.binaryproto | The mean file generated by Caffe from all gallery images, this could be used for mean value substraction from all images.}"
"{label_file | ../data/label_all.txt | A namelist including all gallery images.}"
"{target_img | ../data/images_all/2_13.png | Path of image waiting to be classified.}"
"{num_candidate | 6 | Number of candidates in gallery as the prediction result.}";
"{target_img | ../data/images_all/3_13.png | Path of image waiting to be classified.}"
"{num_candidate | 6 | Number of candidates in gallery as the prediction result.}"
"{device | CPU | device}"
"{dev_id | 0 | dev_id}";
cv::CommandLineParser parser(argc, argv, keys);
parser.about("Demo for Sphere View data generation");
if (parser.has("help"))
......@@ -58,22 +59,25 @@ int main(int argc, char** argv)
string caffemodel = parser.get<string>("caffemodel");
string network_forIMG = parser.get<string>("network_forIMG");
string mean_file = parser.get<string>("mean_file");
string label_file = parser.get<string>("label_file");
string target_img = parser.get<string>("target_img");
int num_candidate = parser.get<int>("num_candidate");
cv::cnn_3dobj::DataTrans transTemp;
string device = parser.get<string>("device");
int dev_id = parser.get<int>("dev_id");
cv::cnn_3dobj::Classification classifier;
classifier.NetSetter(network_forIMG, caffemodel, mean_file, device, dev_id);
std::vector<string> name_gallery;
transTemp.list_dir(src_dir.c_str(), name_gallery, false);
classifier.list_dir(src_dir.c_str(), name_gallery, false);
classifier.GetLabellist(name_gallery);
for (unsigned int i = 0; i < name_gallery.size(); i++) {
name_gallery[i] = src_dir + name_gallery[i];
}
////start another demo
cv::cnn_3dobj::Classification classifier(network_forIMG, caffemodel, mean_file, label_file);
std::vector<cv::Mat> feature_reference;
std::vector<cv::Mat> img_gallery;
cv::Mat feature_reference;
for (unsigned int i = 0; i < name_gallery.size(); i++) {
cv::Mat img_gallery = cv::imread(name_gallery[i], -1);
feature_reference.push_back(classifier.feature_extract(img_gallery, false));
img_gallery.push_back(cv::imread(name_gallery[i], -1));
}
classifier.FeatureExtract(img_gallery, feature_reference, false);
std::cout << std::endl << "---------- Prediction for "
<< target_img << " ----------" << std::endl;
......@@ -82,13 +86,14 @@ int main(int argc, char** argv)
// CHECK(!img.empty()) << "Unable to decode image " << target_img;
std::cout << std::endl << "---------- Featrue of gallery images ----------" << std::endl;
std::vector<std::pair<string, float> > prediction;
for (unsigned int i = 0; i < feature_reference.size(); i++)
std::cout << feature_reference[i].t() << endl;
cv::Mat feature_test = classifier.feature_extract(img, false);
std::cout << std::endl << "---------- Featrue of target image: " << target_img << "----------" << endl << feature_test.t() << std::endl;
for (unsigned int i = 0; i < feature_reference.rows; i++)
std::cout << feature_reference.row(i) << endl;
cv::Mat feature_test;
classifier.FeatureExtract(img, feature_test, false);
std::cout << std::endl << "---------- Featrue of target image: " << target_img << "----------" << endl << feature_test << std::endl;
prediction = classifier.Classify(feature_reference, img, num_candidate, false);
// Print the top N prediction.
std::cout << std::endl << "---------- Prediction result(distance - file name in gallery) ----------" << std::endl;
std::cout << std::endl << "---------- Prediction result(Distance - File Name in Gallery) ----------" << std::endl;
for (size_t i = 0; i < prediction.size(); ++i) {
std::pair<string, float> p = prediction[i];
std::cout << std::fixed << std::setprecision(2) << p.second << " - \""
......
modules/cnn_3dobj/samples/data/3d_triplet_galleryIMG.prototxt deleted 100644 → 0
View file @ 4fe5498a
name: "3d_triplet"
layer {
name: "data"
type: "Data"
top: "data"
top: "label"
include {
phase: TEST
}
data_param {
source: "/home/wangyida/Desktop/opencv_contrib/modules/nouse_test/samples/data/dbfile"
batch_size: 69
}
}
layer {
name: "conv1"
type: "Convolution"
bottom: "data"
top: "conv1"
convolution_param {
num_output: 16
kernel_size: 8
stride: 1
}
}
layer {
name: "pool1"
type: "Pooling"
bottom: "conv1"
top: "pool1"
pooling_param {
pool: MAX
kernel_size: 2
stride: 2
}
}
layer {
name: "relu1"
type: "ReLU"
bottom: "pool1"
top: "pool1"
}
layer {
name: "conv2"
type: "Convolution"
bottom: "pool1"
top: "conv2"
convolution_param {
num_output: 7
kernel_size: 5
stride: 1
}
}
layer {
name: "pool2"
type: "Pooling"
bottom: "conv2"
top: "pool2"
pooling_param {
pool: MAX
kernel_size: 2
stride: 2
}
}
layer {
name: "relu2"
type: "ReLU"
bottom: "pool2"
top: "pool2"
}
layer {
name: "ip1"
type: "InnerProduct"
bottom: "pool2"
top: "ip1"
inner_product_param {
num_output: 256
}
}
layer {
name: "relu3"
type: "ReLU"
bottom: "ip1"
top: "ip1"
}
layer {
name: "feat"
type: "InnerProduct"
bottom: "ip1"
top: "feat"
inner_product_param {
num_output: 4
}
}
modules/cnn_3dobj/samples/datatrans_demo.cpp deleted 100644 → 0
View file @ 4fe5498a
/*
* Software License Agreement (BSD License)
*
* Copyright (c) 2009, Willow Garage, Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
* * Neither the name of Willow Garage, Inc. nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
*/
#include <opencv2/cnn_3dobj.hpp>
using namespace cv;
using namespace std;
using namespace cv::cnn_3dobj;
int main(int argc, char* argv[])
{
const String keys = "{help | | this demo will convert a set of images in a particular path into leveldb database for feature extraction using Caffe.}"
"{src_dir | ../data/images_all | Source direction of the images ready for being converted to leveldb dataset.}"
"{src_dst | ../data/dbfile | Aim direction of the converted to leveldb dataset. }"
"{attach_dir | ../data/dbfile | Path for saving additional files which describe the transmission results. }"
"{channel | 1 | Channel of the images. }"
"{width | 64 | Width of images}"
"{height | 64 | Height of images}";
cv::CommandLineParser parser(argc, argv, keys);
parser.about("Demo for Sphere View data generation");
if (parser.has("help"))
{
parser.printMessage();
return 0;
}
string src_dir = parser.get<string>("src_dir");
string src_dst = parser.get<string>("src_dst");
string attach_dir = parser.get<string>("attach_dir");
int channel = parser.get<int>("channel");
int width = parser.get<int>("width");
int height = parser.get<int>("height");
cv::cnn_3dobj::DataTrans Trans;
Trans.convert(src_dir,src_dst,attach_dir,channel,width,height);
std::cout << std::endl << "All featrues of images in: " << std::endl << src_dir << std::endl << "have been converted to levelDB data in: " << std::endl << src_dst << std::endl << "for extracting feature of gallery images in classification efficiently, this convertion is not needed in feature extraction of test image" << std::endl;
}
modules/cnn_3dobj/samples/feature_extract_demo.cpp deleted 100644 → 0
View file @ 4fe5498a
/*
* Software License Agreement (BSD License)
*
* Copyright (c) 2009, Willow Garage, Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
* * Neither the name of Willow Garage, Inc. nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
*/
#include <opencv2/cnn_3dobj.hpp>
#include <stdio.h> // for snprintf
#include <tr1/memory>
#include <string>
#include <vector>
#include "google/protobuf/text_format.h"
#include <opencv2/opencv.hpp>
#include <opencv2/core/core.hpp>
#define CPU_ONLY
#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/net.hpp"
#include "caffe/proto/caffe.pb.h"
#include "caffe/util/io.hpp"
#include "caffe/vision_layers.hpp"
using caffe::Blob;
using caffe::Caffe;
using caffe::Datum;
using caffe::Net;
//using boost::shared_ptr;
using std::string;
//namespace db = caffe::db;
using namespace cv;
using namespace std;
using namespace cv::cnn_3dobj;
int main(int argc, char* argv[])
{
const String keys = "{help | | this demo will convert a set of images in a particular path into leveldb database for feature extraction using Caffe.}"
"{src_dir | ../data/images_all/ | Source direction of the images ready for being converted to leveldb dataset.}"
"{src_dst | ../data/dbfile | Aim direction of the converted to leveldb dataset. }"
"{attach_dir | ../data/dbfile | Path for saving additional files which describe the transmission results. }"
"{channel | 1 | Channel of the images. }"
"{width | 64 | Width of images}"
"{height | 64 | Height of images}"
"{pretrained_binary_proto | ../data/3d_triplet_iter_10000.caffemodel | caffe model for feature exrtaction.}"
"{feature_extraction_proto | ../data/3d_triplet_train_test.prototxt | network definition in .prototxt the path of the training samples must be wrotten in in .prototxt files in Phase TEST}"
"{save_feature_dataset_names | ../data/feature/feature_iter_10000.bin | the output of the extracted feature in form of binary files together with the vector<cv::Mat> features as the feature.}"
"{extract_feature_blob_names | feat | the layer used for feature extraction in CNN.}"
"{num_mini_batches | 6 | batches suit for the batches defined in the .proto for the aim of extracting feature from all images.}"
"{device | CPU | device}"
"{dev_id | 0 | dev_id}";
cv::CommandLineParser parser(argc, argv, keys);
parser.about("Demo for Sphere View data generation");
if (parser.has("help"))
{
parser.printMessage();
return 0;
}
string src_dir = parser.get<string>("src_dir");
string src_dst = parser.get<string>("src_dst");
string attach_dir = parser.get<string>("attach_dir");
int channel = parser.get<int>("channel");
int width = parser.get<int>("width");
int height = parser.get<int>("height");
string pretrained_binary_proto = parser.get<string>("pretrained_binary_proto");
string feature_extraction_proto = parser.get<string>("feature_extraction_proto");
string save_feature_dataset_names = parser.get<string>("save_feature_dataset_names");
string extract_feature_blob_names = parser.get<string>("extract_feature_blob_names");
int num_mini_batches = parser.get<int>("num_mini_batches");
string device = parser.get<string>("device");
int dev_id = parser.get<int>("dev_id");
cv::cnn_3dobj::DataTrans transTemp;
transTemp.convert(src_dir,src_dst,attach_dir,channel,width,height);
std::vector<cv::Mat> extractedFeature = transTemp.feature_extraction_pipeline(pretrained_binary_proto, feature_extraction_proto, save_feature_dataset_names, extract_feature_blob_names, num_mini_batches, device, dev_id);
}
modules/cnn_3dobj/src/cnn_datatrans.cpp deleted 100644 → 0
View file @ 4fe5498a
#include "precomp.hpp"
using std::string;
using namespace std;
namespace cv
{
namespace cnn_3dobj
{
DataTrans::DataTrans()
{
};
void DataTrans::list_dir(const char *path,vector<string>& files,bool r)
{
DIR *pDir;
struct dirent *ent;
char childpath[512];
pDir = opendir(path);
memset(childpath, 0, sizeof(childpath));
while ((ent = readdir(pDir)) != NULL)
{
if (ent->d_type & DT_DIR)
{
if (strcmp(ent->d_name, ".") == 0 || strcmp(ent->d_name, "..") == 0)
{
continue;
}
if(r)
{
sprintf(childpath, "%s/%s", path, ent->d_name);
DataTrans::list_dir(childpath,files,false);
}
}
else
{
files.push_back(ent->d_name);
}
}
sort(files.begin(),files.end());
};
string DataTrans::get_classname(string path)
{
int index = path.find_last_of('_');
return path.substr(0, index);
}
int DataTrans::get_labelid(string fileName)
{
string class_name_tmp = get_classname(fileName);
all_class_name.insert(class_name_tmp);
map<string,int>::iterator name_iter_tmp = class2id.find(class_name_tmp);
if (name_iter_tmp == class2id.end())
{
int id = class2id.size();
class2id.insert(name_iter_tmp, std::make_pair(class_name_tmp, id));
return id;
}
else
{
return name_iter_tmp->second;
}
}
void DataTrans::loadimg(string path,char* buffer,const bool is_color)
{
cv::Mat img = cv::imread(path, is_color);
string val;
int rows = img.rows;
int cols = img.cols;
int pos=0;
int channel;
if (is_color == 0)
{
channel = 1;
}else{
channel = 3;
}
for (int c = 0; c < channel; c++)
{
for (int row = 0; row < rows; row++)
{
for (int col = 0; col < cols; col++)
{
buffer[pos++]=img.at<cv::Vec3b>(row,col)[c];
}
}
}
};
void DataTrans::convert(string imgdir,string outputdb,string attachdir,int channel,int width,int height)
{
leveldb::DB* db;
leveldb::Options options;
options.create_if_missing = true;
// options.error_if_exists = true;
caffe::Datum datum;
datum.set_channels(channel);
datum.set_height(height);
datum.set_width(width);
int image_size = channel*width*height;
char buffer[image_size];
string value;
CHECK(leveldb::DB::Open(options, outputdb, &db).ok());
vector<string> filenames;
list_dir(imgdir.c_str(),filenames, false);
string img_log = attachdir+"image_filename";
ofstream writefile(img_log.c_str());
for(int i=0;i<(int)filenames.size();i++)
{
string path= imgdir;
path.append(filenames[i]);
loadimg(path,buffer,false);
int labelid = get_labelid(filenames[i]);
datum.set_label(labelid);
datum.set_data(buffer,image_size);
datum.SerializeToString(&value);
snprintf(buffer, image_size, "%05d", i);
printf("\nclassid:%d classname:%s abspath:%s",labelid,get_classname(filenames[i]).c_str(),path.c_str());
db->Put(leveldb::WriteOptions(),string(buffer),value);
//printf("%d %s\n",i,fileNames[i].c_str());
assert(writefile.is_open());
writefile<<i<<" "<<filenames[i]<<"\n";
}
delete db;
writefile.close();
img_log = attachdir+"image_classname";
writefile.open(img_log.c_str());
set<string>::iterator iter = all_class_name.begin();
while(iter != all_class_name.end())
{
assert(writefile.is_open());
writefile<<(*iter)<<"\n";
//printf("%s\n",(*iter).c_str());
iter++;
}
writefile.close();
};
std::vector<cv::Mat> DataTrans::feature_extraction_pipeline(std::string pretrained_binary_proto, std::string feature_extraction_proto, std::string save_feature_dataset_names, std::string extract_feature_blob_names, int num_mini_batches, std::string device, int dev_id) {
if (strcmp(device.c_str(), "GPU") == 0) {
LOG(ERROR)<< "Using GPU";
int device_id = 0;
if (strcmp(device.c_str(), "GPU") == 0) {
device_id = dev_id;
CHECK_GE(device_id, 0);
}
LOG(ERROR) << "Using Device_id=" << device_id;
Caffe::SetDevice(device_id);
Caffe::set_mode(Caffe::GPU);
} else {
LOG(ERROR) << "Using CPU";
Caffe::set_mode(Caffe::CPU);
}
boost::shared_ptr<Net<float> > feature_extraction_net(
new Net<float>(feature_extraction_proto, caffe::TEST));
feature_extraction_net->CopyTrainedLayersFrom(pretrained_binary_proto);
std::vector<std::string> blob_names;
blob_names.push_back(extract_feature_blob_names);
std::vector<std::string> dataset_names;
dataset_names.push_back(save_feature_dataset_names);
CHECK_EQ(blob_names.size(), dataset_names.size()) <<
" the number of blob names and dataset names must be equal";
size_t num_features = blob_names.size();
for (size_t i = 0; i < num_features; i++) {
CHECK(feature_extraction_net->has_blob(blob_names[i]))
<< "Unknown feature blob name " << blob_names[i]
<< " in the network " << feature_extraction_proto;
}
std::vector<FILE*> files;
for (size_t i = 0; i < num_features; ++i)
{
LOG(INFO) << "Opening file " << dataset_names[i];
FILE * temp = fopen(dataset_names[i].c_str(), "wb");
files.push_back(temp);
}
LOG(ERROR)<< "Extacting Features";
Datum datum;
std::vector<cv::Mat> featureVec;
std::vector<Blob<float>*> input_vec;
std::vector<int> image_indices(num_features, 0);
for (int batch_index = 0; batch_index < num_mini_batches; ++batch_index) {
feature_extraction_net->Forward(input_vec);
for (size_t i = 0; i < num_features; ++i) {
const boost::shared_ptr<Blob<float> > feature_blob = feature_extraction_net
->blob_by_name(blob_names[i]);
int batch_size = feature_blob->num();
int dim_features = feature_blob->count() / batch_size;
if (batch_index == 0)
{
int fea_num = batch_size*num_mini_batches;
fwrite(&dim_features, sizeof(int), 1, files[i]);
fwrite(&fea_num, sizeof(int), 1, files[i]);
}
const float* feature_blob_data;
for (int n = 0; n < batch_size; ++n) {
feature_blob_data = feature_blob->cpu_data() +
feature_blob->offset(n);
fwrite(feature_blob_data, sizeof(float), dim_features, files[i]);
cv::Mat tempfeat = cv::Mat(1, dim_features, CV_32FC1);
for (int dim = 0; dim < dim_features; dim++) {
tempfeat.at<float>(0,dim) = *(feature_blob_data++);
}
featureVec.push_back(tempfeat);
++image_indices[i];
if (image_indices[i] % 1000 == 0) {
LOG(ERROR)<< "Extracted features of " << image_indices[i] <<
" query images for feature blob " << blob_names[i];
}
} // for (int n = 0; n < batch_size; ++n)
} // for (int i = 0; i < num_features; ++i)
} // for (int batch_index = 0; batch_index < num_mini_batches; ++batch_index)
// write the last batch
for (size_t i = 0; i < num_features; ++i) {
fclose(files[i]);
}
LOG(ERROR)<< "Successfully extracted the features!";
return featureVec;
};
}}
modules/cnn_3dobj/src/cnn_sphereview.cpp
View file @ b831fc3b
......@@ -18,8 +18,6 @@ namespace cnn_3dobj
{ -X, 0.0f, -Z }, { X, 0.0f, -Z }, { 0.0f, Z, X }, { 0.0f, Z, -X },
{ 0.0f, -Z, X }, { 0.0f, -Z, -X }, { Z, X, 0.0f }, { -Z, X, 0.0f },
{ Z, -X, 0.0f }, { -Z, -X, 0.0f } };
int tindices[20][3] = { { 0, 4, 1 }, { 0, 9, 4 }, { 9, 5, 4 },
{ 4, 5, 8 }, { 4, 8, 1 }, { 8, 10, 1 }, { 8, 3, 10 }, { 5, 3, 8 },
{ 5, 2, 3 }, { 2, 7, 3 }, { 7, 10, 3 }, { 7, 6, 10 }, { 7, 11, 6 },
......@@ -27,8 +25,8 @@ namespace cnn_3dobj
{ 9, 11, 2 }, { 9, 2, 5 }, { 7, 2, 11 } };
// Iterate over points
for (int i = 0; i < 20; ++i) {
for (int i = 0; i < 20; ++i)
{
subdivide(vdata[tindices[i][0]], vdata[tindices[i][1]],
vdata[tindices[i][2]], depth_in);
}
......@@ -46,22 +44,21 @@ namespace cnn_3dobj
CameraPos = CameraPos_temp;
cout << "View points in total: " << CameraPos.size() << endl;
cout << "The coordinate of view point: " << endl;
for(int i=0; i < (int)CameraPos.size(); i++) {
for(int i=0; i < (int)CameraPos.size(); i++)
{
cout << CameraPos.at(i).x <<' '<< CameraPos.at(i).y << ' ' << CameraPos.at(i).z << endl;
}
};
void IcoSphere::norm(float v[])
{
float len = 0;
for (int i = 0; i < 3; ++i) {
for (int i = 0; i < 3; ++i)
{
len += v[i] * v[i];
}
len = sqrt(len);
for (int i = 0; i < 3; ++i) {
for (int i = 0; i < 3; ++i)
{
v[i] /= ((float)len);
}
};
......@@ -70,7 +67,8 @@ namespace cnn_3dobj
{
Point3f temp_Campos;
std::vector<float>* temp = new std::vector<float>;
for (int k = 0; k < 3; ++k) {
for (int k = 0; k < 3; ++k)
{
vertexList.push_back(v[k]);
vertexNormalsList.push_back(v[k]);
temp->push_back(v[k]);
......@@ -81,31 +79,28 @@ namespace cnn_3dobj
void IcoSphere::subdivide(float v1[], float v2[], float v3[], int depth)
{
norm(v1);
norm(v2);
norm(v3);
if (depth == 0) {
if (depth == 0)
{
add(v1);
add(v2);
add(v3);
return;
}
float* v12 = new float[3];
float* v23 = new float[3];
float* v31 = new float[3];
for (int i = 0; i < 3; ++i) {
for (int i = 0; i < 3; ++i)
{
v12[i] = (v1[i] + v2[i]) / 2;
v23[i] = (v2[i] + v3[i]) / 2;
v31[i] = (v3[i] + v1[i]) / 2;
}
norm(v12);
norm(v23);
norm(v31);
subdivide(v1, v12, v31, depth - 1);
subdivide(v2, v23, v12, depth - 1);
subdivide(v3, v31, v23, depth - 1);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment