diff --git a/src/KnnAnnotator.cpp b/src/KnnAnnotator.cpp index c323adc..ce504be 100644 --- a/src/KnnAnnotator.cpp +++ b/src/KnnAnnotator.cpp @@ -26,100 +26,102 @@ class KnnAnnotator : public DrawingAnnotator { private: - cv::Mat color; + cv::Mat color; - //set_mode should be GT(groundTruth) or CL (classify).... - std::string mode; + //set_mode should be GT(groundTruth) or CL (classify).... + std::string mode; - //the value of k-neighbors in the knn-classifier - int default_k; + //the value of k-neighbors in the knn-classifier + int default_k; - //feature_use should be VFH, CVFH, CNN, VGG16 ..... - std::string feature_use; + //feature_use should be VFH, CVFH, CNN, VGG16 ..... + std::string feature_use; - //the name of train matrix and its labels in path rs_resources/objects_dataset/extractedFeat/ - std::string trainKNN_matrix; - std::string trainKNNLabel_matrix; + //the name of train matrix and its labels in path rs_resources/objects_dataset/extractedFeat/ + std::string trainKNN_matrix; + std::string trainKNNLabel_matrix; - //vector to hold split trained_model_name - std::vector split_model; + //vector to hold split trained_model_name + std::vector split_model; - //the name of actual_class_label map file in path rs_resources/objects_dataset/extractedFeat/ - std::string classNrMapping; + //the name of actual_class_label map file in path rs_resources/objects_dataset/extractedFeat/ + std::string classNrMapping; - //vector to hold classes name - std::vector model_labels; + //vector to hold classes name + std::vector model_labels; - RSKNN *knnObject; + RSKNN *knnObject; public: - KnnAnnotator(): DrawingAnnotator(__func__) - {} + KnnAnnotator(): DrawingAnnotator(__func__) + {} - TyErrorId initialize(AnnotatorContext &ctx) - { - ctx.extractValue("set_mode", mode); - outInfo("set_mode:" << mode << std::endl); + TyErrorId initialize(AnnotatorContext &ctx) + { + ctx.extractValue("set_mode", mode); + outInfo("set_mode:" << mode << std::endl); - ctx.extractValue("default_k", default_k); - outInfo("Value of k-neighbors: " << default_k); + ctx.extractValue("default_k", default_k); + outInfo("Value of k-neighbors: " << default_k); - ctx.extractValue("training_data", trainKNN_matrix); - outInfo("training_data:" << trainKNN_matrix); + ctx.extractValue("training_data", trainKNN_matrix); + outInfo("training_data:" << trainKNN_matrix); - ctx.extractValue("class_label_mapping", classNrMapping); - outInfo("class_label_mapping:" << classNrMapping ); + ctx.extractValue("class_label_mapping", classNrMapping); + outInfo("class_label_mapping:" << classNrMapping ); - ctx.extractValue("feature_descriptor_type", feature_use); - outInfo("feature descriptor set: "<loadModelFile(trainKNN_matrix); - knnObject->setLabels(classNrMapping, model_labels); + knnObject = new RSKNN(default_k); + knnObject->loadModelFile(trainKNN_matrix); + knnObject->setLabels(classNrMapping, model_labels); - return UIMA_ERR_NONE; - } - - TyErrorId destroy() - { - outInfo("destroy"); - return UIMA_ERR_NONE; - } - - TyErrorId processWithLock(CAS &tcas, ResultSpecification const &res_spec) - { - outInfo("RSKNNAnnotator is running:"); - rs::SceneCas cas(tcas); - rs::Scene scene = cas.getScene(); - cas.get(VIEW_COLOR_IMAGE_HD, color); - std::vector clusters; - scene.identifiables.filter(clusters); - outInfo("Feature to use: "<processPCLFeatureKNN(mode, feature_use, clusters, color, model_labels, tcas); - } - else if(feature_use == "BVLC_REF" || feature_use == "VGG16") { - outInfo("Calculation starts with : " << mode << "::" << feature_use); - knnObject->processCaffeFeatureKNN(mode, feature_use, clusters, color, model_labels, tcas); + return UIMA_ERR_NONE; } - else { - outError("Please sellect the correct value of parameter(feature_use): VFH, CVFH, BVLC_REF, VGG16"); + + TyErrorId destroy() + { + outInfo("destroy"); + return UIMA_ERR_NONE; } - outInfo("calculation is done with RSKNN" << std::endl); - return UIMA_ERR_NONE; - } + TyErrorId processWithLock(CAS &tcas, ResultSpecification const &res_spec) + { + rs::StopWatch clock; + outInfo("RSKNNAnnotator is running:"); + rs::SceneCas cas(tcas); + rs::Scene scene = cas.getScene(); + cas.get(VIEW_COLOR_IMAGE_HD, color); + std::vector clusters; + scene.identifiables.filter(clusters); + outInfo("Feature to use: "<processPCLFeatureKNN(mode, feature_use, clusters, color, model_labels, tcas); + } + else if(feature_use == "BVLC_REF" || feature_use == "VGG16") { + outInfo("Calculation starts with : " << mode << "::" << feature_use); + knnObject->processCaffeFeatureKNN(mode, feature_use, clusters, color, model_labels, tcas); + } + else { + outError("Please sellect the correct value of parameter(feature_use): VFH, CVFH, BVLC_REF, VGG16"); + } + + outInfo("calculation is done with RSKNN" << std::endl); + outInfo("took: " << clock.getTime() << " ms. with the Cluster size of : " << clusters.size()); + return UIMA_ERR_NONE; + } - void drawImageWithLock(cv::Mat &disp) - { - disp = color.clone(); - } + void drawImageWithLock(cv::Mat &disp) + { + disp = color.clone(); + } }; diff --git a/src/RfAnnotator.cpp b/src/RfAnnotator.cpp index f7f3541..0c7fcdf 100644 --- a/src/RfAnnotator.cpp +++ b/src/RfAnnotator.cpp @@ -25,102 +25,103 @@ class RfAnnotator : public DrawingAnnotator { private: - cv::Mat color; + cv::Mat color; - //set_mode should be GT(groundTruth) or CF (classify) ........... - std::string set_mode; + //set_mode should be GT(groundTruth) or CF (classify) ........... + std::string set_mode; - //dataset_use should be IAI (kitchen data from IAI) or WU (data from Washington University) or both... - std::string dataset_use; + //dataset_use should be IAI (kitchen data from IAI) or WU (data from Washington University) or both... + std::string dataset_use; - //feature_use should be VFH, CVFH, CNN, VGG16 ..... - std::string feature_use; + //feature_use should be VFH, CVFH, CNN, VGG16 ..... + std::string feature_use; - //the name of trained model ifle in folder rs_addons/trainedData/ - std::string trained_model_name; + //the name of trained model ifle in folder rs_addons/trainedData/ + std::string trained_model_name; - //vector to hold split trained_model_name - std::vector split_model; + //vector to hold split trained_model_name + std::vector split_model; - //the name of actual_class_label map file in path rs_resources/objects_dataset/extractedFeat/ - std::string actual_class_label; + //the name of actual_class_label map file in path rs_resources/objects_dataset/extractedFeat/ + std::string actual_class_label; - //vector to hold classes name - std::vector model_labels; + //vector to hold classes name + std::vector model_labels; public: - RfAnnotator(): DrawingAnnotator(__func__) - {} + RfAnnotator(): DrawingAnnotator(__func__) + {} - RSClassifier* rfObject= new RSRF; + RSClassifier* rfObject= new RSRF; - TyErrorId initialize(AnnotatorContext &ctx) - { - outInfo("initialize"); - ctx.extractValue("set_mode", set_mode); - ctx.extractValue("trained_model_name", trained_model_name); - ctx.extractValue("actual_class_label", actual_class_label); - - outInfo("Name of the loaded files for RF are:"<setLabels(actual_class_label, model_labels); + TyErrorId initialize(AnnotatorContext &ctx) + { + outInfo("initialize"); + ctx.extractValue("set_mode", set_mode); + ctx.extractValue("trained_model_name", trained_model_name); + ctx.extractValue("actual_class_label", actual_class_label); - boost::split(split_model, trained_model_name, boost::is_any_of("_")); + outInfo("Name of the loaded files for RF are:"<setLabels(actual_class_label, model_labels); - return UIMA_ERR_NONE; - } + boost::split(split_model, trained_model_name, boost::is_any_of("_")); - TyErrorId destroy() - { - outInfo("destroy"); - return UIMA_ERR_NONE; - } + dataset_use= split_model[0]; + outInfo("dataset_use:"< clusters; - scene.identifiables.filter(clusters); + feature_use= split_model[1]; + outInfo("feature_use:"<processPCLFeature(trained_model_name,set_mode,feature_use,clusters, rfObject, color,model_labels, tcas); + outInfo("destroy"); + return UIMA_ERR_NONE; } - else if(feature_use == "CNN" || feature_use == "VGG16") + + TyErrorId processWithLock(CAS &tcas, ResultSpecification const &res_spec) { - outInfo("Calculation starts with : " << set_mode << "::" << dataset_use << "::" << feature_use); - rfObject->processCaffeFeature(trained_model_name,set_mode,feature_use,clusters, rfObject, color, model_labels, tcas); + rs::StopWatch clock; + outInfo("RSRFAnnotator is running:"); + rs::SceneCas cas(tcas); + rs::Scene scene = cas.getScene(); + cas.get(VIEW_COLOR_IMAGE_HD, color); + std::vector clusters; + scene.identifiables.filter(clusters); + + if(feature_use == "VFH" || feature_use == "CVFH") + { + outInfo("Calculation starts with : " << set_mode << "::" << dataset_use << "::" << feature_use); + rfObject->processPCLFeature(trained_model_name,set_mode,feature_use,clusters, rfObject, color,model_labels, tcas); + } + else if(feature_use == "CNN" || feature_use == "VGG16") + { + outInfo("Calculation starts with : " << set_mode << "::" << dataset_use << "::" << feature_use); + rfObject->processCaffeFeature(trained_model_name,set_mode,feature_use,clusters, rfObject, color, model_labels, tcas); + } + else + { + outError("Please sellect the correct value of parameter(feature_use): VFH, CVFH, CNN, VGG16"); + } + + outInfo("calculation is done with RSRF"< #include #include +#include + #include #include @@ -32,209 +34,224 @@ RSClassifier::RSClassifier() void RSClassifier::setLabels(std::string file_name, std::vector &my_annotation) { - std::string packagePath = ros::package::getPath("rs_resources") + "/"; - std::string pathToFile; - - if(boost::filesystem::exists(file_name)) { - pathToFile = file_name; - } - else { - pathToFile = packagePath + file_name; - } - //To check the resource path................................................ - - if(!boost::filesystem::exists(pathToFile)) { - outError(file_name << " file does not exist in path " << packagePath << " to read the object's class label." << std::endl); - } - else { - std::ifstream file((pathToFile).c_str()); + std::string packagePath = ros::package::getPath("rs_resources") + "/"; + std::string pathToFile; - std::string str; - std::vector split_str; + if(boost::filesystem::exists(file_name)) { + pathToFile = file_name; + } + else { + pathToFile = packagePath + file_name; + } + //To check the resource path................................................ - while(std::getline(file, str)) { - boost::split(split_str, str, boost::is_any_of(":")); - my_annotation.push_back(split_str[0]); + if(!boost::filesystem::exists(pathToFile)) { + outError(file_name << " file does not exist in path " << packagePath << " to read the object's class label." << std::endl); + } + else { + std::ifstream file((pathToFile).c_str()); + + std::string str; + std::vector split_str; + + while(std::getline(file, str)) { + boost::split(split_str, str, boost::is_any_of(":")); + my_annotation.push_back(split_str[0]); + } } - } } void RSClassifier::getLabels(const std::string path, std::map &input_file) { - double class_label = 1; - std::ifstream file(path.c_str()); - std::string str; - - while(std::getline(file, str)) { - input_file[str] = class_label; - class_label = class_label + 1; - } + double class_label = 1; + std::ifstream file(path.c_str()); + std::string str; + + while(std::getline(file, str)) { + input_file[str] = class_label; + class_label = class_label + 1; + } } // To read the descriptors matrix and it's label from /rs_resources/objects_dataset/extractedFeat folder........... void RSClassifier::readFeaturesFromFile(std::string data_file_path, std::string label_name, cv::Mat &des_matrix, cv::Mat &des_label) { - cv::FileStorage fs; - std::string pathToDataFile; - std::string packagePath = ros::package::getPath("rs_resources") + '/'; - if(boost::filesystem::exists(data_file_path)) { - pathToDataFile = data_file_path; - } - else { - pathToDataFile = packagePath + data_file_path; - } - - if(!boost::filesystem::exists(pathToDataFile)) { - outError(data_file_path << " does not exist; Needs to be a full path or relateive to rs_resources package; please check" << std::endl); - } - else { - fs.open(pathToDataFile, cv::FileStorage::READ); - fs["descriptors"] >> des_matrix; - fs["label"] >> des_label; - } + cv::FileStorage fs; + std::string pathToDataFile; + std::string packagePath = ros::package::getPath("rs_resources") + '/'; + if(boost::filesystem::exists(data_file_path)) { + pathToDataFile = data_file_path; + } + else { + pathToDataFile = packagePath + data_file_path; + } + + if(!boost::filesystem::exists(pathToDataFile)) { + outError(data_file_path << " does not exist; Needs to be a full path or relateive to rs_resources package; please check" << std::endl); + } + else { + fs.open(pathToDataFile, cv::FileStorage::READ); + fs["descriptors"] >> des_matrix; + fs["label"] >> des_label; + } } // To show the confusion matrix and accuracy result........................... void RSClassifier::evaluation(std::vector test_label, std::vector predicted_label, std::string obj_classInDouble) { - std::map < std::string, double > object_label; - std::string resourcePath; - resourcePath = ros::package::getPath("rs_resources") + '/'; - std::string label_path = "objects_dataset/extractedFeat/" + obj_classInDouble + ".txt"; - - if(!boost::filesystem::exists(resourcePath + label_path)) { - outError(obj_classInDouble << " file does not exist in path " << resourcePath + label_path << std::endl); - }; - - //To read the object class names from rs_resources/object_dataset/objects.txt....... - getLabels(resourcePath + label_path, object_label); - - //Declare the confusion matrix which takes test data label (test_label) and predicted_label or class as inputs. - //It's size is defined by the number of classes. - std::vector >confusion_matrix(object_label.size(), std::vector(object_label.size(), 0)); - - for(int i = 0; i < test_label.size(); i++) { - confusion_matrix[test_label[i] - 1][predicted_label[i] - 1] = confusion_matrix[test_label[i] - 1][predicted_label[i] - 1] + 1; - } - - //To show the results of confusion matrix ................................. - std::cout << "confusion_matrix:" << std::endl; - for(int i = 0; i < object_label.size(); i++) { - for(int j = 0; j < object_label.size(); j++) { - std::cout << confusion_matrix[i][j] << " "; + std::map < std::string, double > object_label; + std::string resourcePath; + resourcePath = ros::package::getPath("rs_resources") + '/'; + std::string label_path = "objects_dataset/extractedFeat/" + obj_classInDouble + ".txt"; + + if(!boost::filesystem::exists(resourcePath + label_path)) { + outError(obj_classInDouble << " file does not exist in path " << resourcePath + label_path << std::endl); + }; + + //To read the object class names from rs_resources/object_dataset/objects.txt....... + getLabels(resourcePath + label_path, object_label); + + //Declare the confusion matrix which takes test data label (test_label) and predicted_label or class as inputs. + //It's size is defined by the number of classes. + std::vector >confusion_matrix(object_label.size(), std::vector(object_label.size(), 0)); + + for(int i = 0; i < test_label.size(); i++) { + confusion_matrix[test_label[i] - 1][predicted_label[i] - 1] = confusion_matrix[test_label[i] - 1][predicted_label[i] - 1] + 1; } - std::cout << std::endl; - } - //calculation of classifier accuracy........................................ - double c = 0; - for(int i = 0; i < object_label.size(); i++) { - c = c + confusion_matrix[i][i]; - } - double Accuracy = (c / test_label.size()) * 100; - std::cout << "classifier Accuray:" << Accuracy << std::endl; + + //To show the results of confusion matrix ................................. + std::cout << "confusion_matrix:" << std::endl; + for(int i = 0; i < object_label.size(); i++) { + for(int j = 0; j < object_label.size(); j++) { + std::cout << confusion_matrix[i][j] << " "; + } + std::cout << std::endl; + } + //calculation of classifier accuracy........................................ + double c = 0; + for(int i = 0; i < object_label.size(); i++) { + c = c + confusion_matrix[i][i]; + } + double Accuracy = (c / test_label.size()) * 100; + std::cout << "classifier Accuray:" << Accuracy << std::endl; } //To save trained model frmom path in rs_addons/trainedData.... std::string RSClassifier::saveTrained(std::string trained_file_name) { - std::string packagePath; - std::string save_train = "trainedData/"; - std::string a; - packagePath = ros::package::getPath("rs_addons") + '/'; - - if(!boost::filesystem::exists(packagePath + save_train)) { - boost::filesystem::create_directory(boost::filesystem::path(packagePath + save_train)); - // outError("Folder called (trainedData) is not found to save or load the generated trained model. " - // " Please create the folder in rs_addons/ and name it as trainedData, then run the annotator again "< clusters, RSClassifier *obj_VFH, cv::Mat &color, std::vector models_label, uima::CAS &tcas) { - outInfo("Number of cluster:" << clusters.size() << std::endl); - - for(size_t i = 0; i < clusters.size(); ++i) { - rs::ObjectHypothesis &cluster = clusters[i]; - std::vector features; - cluster.annotations.filter(features); - - for(size_t j = 0; j < features.size(); ++j) { - rs::PclFeature &feats = features[j]; - outInfo("type of feature:" << feats.feat_type() << std::endl); - std::vector featDescriptor = feats.feature(); - outInfo("Size after conversion:" << featDescriptor.size()); - cv::Mat test_mat(1, featDescriptor.size(), CV_32F); - for(size_t k = 0; k < featDescriptor.size(); ++k) { - test_mat.at(0, k) = featDescriptor[k]; - } - outInfo("number of elements in :" << i << std::endl); - double classLabel; - double confi; - obj_VFH->classifyOnLiveData(memory_name, test_mat, classLabel, confi); - int classLabelInInt = classLabel; - std::string classLabelInString = models_label[classLabelInInt - 1]; - - //To annotate the clusters.................. - annotate_hypotheses(tcas, classLabelInString, feature_use, cluster, set_mode, confi); - - //set roi on image - rs::ImageROI image_roi = cluster.rois.get(); - cv::Rect rect; - rs::conversion::from(image_roi.roi_hires.get(), rect); - - //Draw result on image........... - obj_VFH->drawCluster(color, rect, classLabelInString); - - outInfo("calculation is done" << std::endl); + outInfo("Number of cluster:" << clusters.size() << std::endl); + + for(size_t i = 0; i < clusters.size(); ++i) { + rs::ObjectHypothesis &cluster = clusters[i]; + std::vector features; + cluster.annotations.filter(features); + + for(size_t j = 0; j < features.size(); ++j) { + rs::PclFeature &feats = features[j]; + outInfo("type of feature:" << feats.feat_type() << std::endl); + std::vector featDescriptor = feats.feature(); + outInfo("Size after conversion:" << featDescriptor.size()); + cv::Mat test_mat(1, featDescriptor.size(), CV_32F); + for(size_t k = 0; k < featDescriptor.size(); ++k) { + test_mat.at(0, k) = featDescriptor[k]; + } + outInfo("number of elements in :" << i << std::endl); + double classLabel; + double confi; + obj_VFH->classifyOnLiveData(memory_name, test_mat, classLabel, confi); + int classLabelInInt = classLabel; + std::string classLabelInString = models_label[classLabelInInt - 1]; + + //To annotate the clusters.................. + annotate_hypotheses(tcas, classLabelInString, feature_use, cluster, set_mode, confi); + + //set roi on image + rs::ImageROI image_roi = cluster.rois.get(); + cv::Rect rect; + rs::conversion::from(image_roi.roi_hires.get(), rect); + + //Draw result on image........... + obj_VFH->drawCluster(color, rect, classLabelInString); + + outInfo("calculation is done" << std::endl); + } } - } } //the function process and classify RGB images, which run from a .bag file. void RSClassifier::processCaffeFeature(std::string memory_name, std::string set_mode, std::string feature_use, std::vector clusters, - RSClassifier *obj_caffe, cv::Mat &color, std::vector models_label, uima::CAS &tcas) -{ - //clusters comming from RS pipeline............................ - outInfo("Number of cluster:" << clusters.size() << std::endl); + RSClassifier *obj_caffe, cv::Mat &color, std::vector models_label, uima::CAS &tcas) { + //clusters comming from RS pipeline............................ + outInfo("Number of cluster:" << clusters.size() << std::endl); + + std::vector threads; + + //multithreading the iterations for the clusters + for (int i = 0; i < clusters.size(); ++i) { + threads.push_back(std::thread(&RSClassifier::calculateCaffeFeature, this, memory_name, set_mode, feature_use, clusters, std::ref(obj_caffe), + std::ref(color), models_label, std::ref(tcas), i)); + } + for (auto &th: threads) { + th.join(); + } +} + +void RSClassifier::calculateCaffeFeature(std::string memory_name, std::string set_mode, std::string feature_use, + std::vector clusters, + RSClassifier *obj_caffe, cv::Mat &color, std::vector models_label, uima::CAS &tcas, int i) { + - for(size_t i = 0; i < clusters.size(); ++i) { rs::ObjectHypothesis &cluster = clusters[i]; std::vector features; cluster.annotations.filter(features); @@ -242,41 +259,40 @@ void RSClassifier::processCaffeFeature(std::string memory_name, std::string set outInfo("feature size:" << features.size()); for(size_t j = 0; j < features.size(); ++j) { - rs::Features &feats = features[j]; - outInfo("type of feature:" << feats.descriptorType() << std::endl); - outInfo("size of feature:" << feats.descriptors << std::endl); - outInfo("size of source:" << feats.source() << std::endl); - - //variable to store caffe feature.......... - cv::Mat featDescriptor; - double classLabel; - double confi; - - if(feats.source() == "Caffe") { - rs::conversion::from(feats.descriptors(), featDescriptor); - outInfo("Size after conversion:" << featDescriptor.size()); - - //The function generate the prediction result................ - obj_caffe->classifyOnLiveData(memory_name, featDescriptor, classLabel, confi); - - //class label in integer, which is used as index of vector model_label. - int classLabelInInt = classLabel; - std::string classLabelInString = models_label[classLabelInInt - 1]; - - //To annotate the clusters.................. - annotate_hypotheses(tcas, classLabelInString, feature_use, cluster, set_mode, confi); - - //set roi on image - rs::ImageROI image_roi = cluster.rois.get(); - cv::Rect rect; - rs::conversion::from(image_roi.roi_hires.get(), rect); - - //Draw result on image........................... - obj_caffe->drawCluster(color, rect, classLabelInString); - } - outInfo("calculation is done" << std::endl); + rs::Features &feats = features[j]; + outInfo("type of feature:" << feats.descriptorType() << std::endl); + outInfo("size of feature:" << feats.descriptors << std::endl); + outInfo("size of source:" << feats.source() << std::endl); + + //variable to store caffe feature.......... + cv::Mat featDescriptor; + double classLabel; + double confi; + + if(feats.source() == "Caffe") { + rs::conversion::from(feats.descriptors(), featDescriptor); + outInfo("Size after conversion: " << featDescriptor.size() << i); + + //The function generate the prediction result................ + obj_caffe->classifyOnLiveData(memory_name, featDescriptor, classLabel, confi); + + //class label in integer, which is used as index of vector model_label. + int classLabelInInt = classLabel; + std::string classLabelInString = models_label[classLabelInInt - 1]; + + //To annotate the clusters.................. + annotate_hypotheses(tcas, classLabelInString, feature_use, cluster, set_mode, confi); + + //set roi on image + rs::ImageROI image_roi = cluster.rois.get(); + cv::Rect rect; + rs::conversion::from(image_roi.roi_hires.get(), rect); + + //Draw result on image........................... + obj_caffe->drawCluster(color, rect, classLabelInString); + } + outInfo("calculation is done for thread: " << i << std::endl); } - } } RSClassifier::~ RSClassifier() diff --git a/src/classifiers/RSKNN.cpp b/src/classifiers/RSKNN.cpp index e39f190..80a6726 100644 --- a/src/classifiers/RSKNN.cpp +++ b/src/classifiers/RSKNN.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -29,212 +30,220 @@ using namespace cv; //..............................k-Nearest Neighbor Classifier......................................... -RSKNN::RSKNN(int K) -{ - knncalld = cv::ml::KNearest::create(); - knncalld->setDefaultK(K); +RSKNN::RSKNN(int K) { + knncalld = cv::ml::KNearest::create(); + knncalld->setDefaultK(K); } -void RSKNN:: trainModel(std::string train_matrix_name, std::string train_label_name, std::string train_label_n) -{ +void RSKNN::trainModel(std::string train_matrix_name, std::string train_label_name, std::string train_label_n) { } -void RSKNN:: classify(std::string trained_file_name_saved, std::string test_matrix_name, std::string test_label_name, std::string obj_classInDouble) -{ +void RSKNN::classify(std::string trained_file_name_saved, std::string test_matrix_name, std::string test_label_name, + std::string obj_classInDouble) { } -void RSKNN::classifyOnLiveData(std::string trained_file_name_saved, cv::Mat test_mat, double &det, double &confi) -{ +void RSKNN::classifyOnLiveData(std::string trained_file_name_saved, cv::Mat test_mat, double &det, double &confi) { } -void RSKNN::loadModelFile(std::string pathToModelFile) -{ - readFeaturesFromFile(pathToModelFile, "", trainingData_, dataLabels_); - knncalld->train(trainingData_, cv::ml::ROW_SAMPLE, dataLabels_); +void RSKNN::loadModelFile(std::string pathToModelFile) { + readFeaturesFromFile(pathToModelFile, "", trainingData_, dataLabels_); + knncalld->train(trainingData_, cv::ml::ROW_SAMPLE, dataLabels_); } -void RSKNN:: classifyKNN(std::string train_matrix_name, std::string train_label_name, - std::string test_matrix_name, std::string test_label_name, std::string obj_classInDouble, int default_k) -{ - //To load the train data................ - cv::Mat train_matrix; - cv::Mat train_label; - readFeaturesFromFile(train_matrix_name, train_label_name, train_matrix, train_label); - std::cout << "size of train matrix:" << train_matrix.size() << std::endl; - std::cout << "size of train label:" << train_label.size() << std::endl; +void RSKNN::classifyKNN(std::string train_matrix_name, std::string train_label_name, + std::string test_matrix_name, std::string test_label_name, std::string obj_classInDouble, + int default_k) { + //To load the train data................ + cv::Mat train_matrix; + cv::Mat train_label; + readFeaturesFromFile(train_matrix_name, train_label_name, train_matrix, train_label); + std::cout << "size of train matrix:" << train_matrix.size() << std::endl; + std::cout << "size of train label:" << train_label.size() << std::endl; - //To load the test data............................. - cv::Mat test_matrix; - cv::Mat test_label; - readFeaturesFromFile(test_matrix_name, test_label_name, test_matrix, test_label); - std::cout << "size of test matrix :" << test_matrix.size() << std::endl; - std::cout << "size of test label" << test_label.size() << std::endl; + //To load the test data............................. + cv::Mat test_matrix; + cv::Mat test_label; + readFeaturesFromFile(test_matrix_name, test_label_name, test_matrix, test_label); + std::cout << "size of test matrix :" << test_matrix.size() << std::endl; + std::cout << "size of test label" << test_label.size() << std::endl; - int k_max = knncalld->getDefaultK(); + int k_max = knncalld->getDefaultK(); - //convert test label matrix into a vector....................... - std::vector con_test_label; - test_label.col(0).copyTo(con_test_label); + //convert test label matrix into a vector....................... + std::vector con_test_label; + test_label.col(0).copyTo(con_test_label); - //Container to hold the integer value of labels............................ - std::vector actual_label; - std::vector predicted_label; + //Container to hold the integer value of labels............................ + std::vector actual_label; + std::vector predicted_label; - for(int i = 0; i < test_label.rows; i++) { + for (int i = 0; i < test_label.rows; i++) { - double res = knncalld->findNearest(test_matrix.row(i), k_max, cv::noArray()); + double res = knncalld->findNearest(test_matrix.row(i), k_max, cv::noArray()); - int prediction = res; - predicted_label.push_back(prediction); - double lab = con_test_label[i]; - int actual_convert = lab; - actual_label.push_back(actual_convert); - } - std::cout << "K-Nearest Neighbor Result :" << std::endl; - evaluation(actual_label, predicted_label, obj_classInDouble); + int prediction = res; + predicted_label.push_back(prediction); + double lab = con_test_label[i]; + int actual_convert = lab; + actual_label.push_back(actual_convert); + } + std::cout << "K-Nearest Neighbor Result :" << std::endl; + evaluation(actual_label, predicted_label, obj_classInDouble); } -std::pair RSKNN::classifyOnLiveDataKNN(cv::Mat test_mat) -{ - int k_max = knncalld->getDefaultK(); +std::pair RSKNN::classifyOnLiveDataKNN(cv::Mat test_mat) { + int k_max = knncalld->getDefaultK(); - cv::normalize(test_mat, test_mat, 1, 0, cv::NORM_L2); + cv::normalize(test_mat, test_mat, 1, 0, cv::NORM_L2); - cv::Mat results, neighborResponses, dists; - double res = knncalld->findNearest(test_mat, k_max, results, neighborResponses, dists); - outInfo(dists); - double confidence = (2 - dists.at(0))/2; + cv::Mat results, neighborResponses, dists; + double res = knncalld->findNearest(test_mat, k_max, results, neighborResponses, dists); + outInfo(dists); + double confidence = (2 - dists.at(0)) / 2; - return std::pair(res,confidence); + return std::pair(res, confidence); } -void RSKNN::processPCLFeatureKNN(std::string set_mode, std::string feature_use, - std::vector clusters, cv::Mat &color, std::vector models_label, uima::CAS &tcas) -{ - outInfo("Number of cluster:" << clusters.size()); - - for(size_t i = 0; i < clusters.size(); ++i) { - rs::ObjectHypothesis &cluster = clusters[i]; - std::vector features; - cluster.annotations.filter(features); - - for(size_t j = 0; j < features.size(); ++j) { - rs::PclFeature &feats = features[j]; - outInfo("type of feature:" << feats.feat_type() << std::endl); - std::vector featDescriptor = feats.feature(); - outInfo("Size after conversion:" << featDescriptor.size()); - cv::Mat test_mat(1, featDescriptor.size(), CV_32F); - - for(size_t k = 0; k < featDescriptor.size(); ++k) { - test_mat.at(0, k) = featDescriptor[k]; - } - outInfo("number of elements in :" << i); - std::pair result = classifyOnLiveDataKNN(test_mat); - int classLabelInInt = result.first; - double confidence = result.second; - - std::string classLabelInString = models_label[classLabelInInt - 1]; - outInfo("prediction class is :" << classLabelInInt <<"class label being: " < clusters, cv::Mat &color, + std::vector models_label, uima::CAS &tcas) { + outInfo("Number of cluster:" << clusters.size()); + + for (size_t i = 0; i < clusters.size(); ++i) { + rs::ObjectHypothesis &cluster = clusters[i]; + std::vector features; + cluster.annotations.filter(features); + + for (size_t j = 0; j < features.size(); ++j) { + rs::PclFeature &feats = features[j]; + outInfo("type of feature:" << feats.feat_type() << std::endl); + std::vector featDescriptor = feats.feature(); + outInfo("Size after conversion:" << featDescriptor.size()); + cv::Mat test_mat(1, featDescriptor.size(), CV_32F); + + for (size_t k = 0; k < featDescriptor.size(); ++k) { + test_mat.at(0, k) = featDescriptor[k]; + } + outInfo("number of elements in :" << i); + std::pair result = classifyOnLiveDataKNN(test_mat); + int classLabelInInt = result.first; + double confidence = result.second; + + std::string classLabelInString = models_label[classLabelInInt - 1]; + outInfo("prediction class is :" << classLabelInInt << "class label being: " << classLabelInString + << " with confidence: " << confidence); + + //To annotate the clusters.................. + annotate_hypotheses(tcas, classLabelInString, feature_use, cluster, set_mode, confidence); + + //set roi on image + rs::ImageROI image_roi = cluster.rois.get(); + cv::Rect rect; + rs::conversion::from(image_roi.roi_hires.get(), rect); + + //Draw result on image........... + drawCluster(color, rect, classLabelInString); + + outInfo("calculation is done" << std::endl); + } + } +} - //set roi on image - rs::ImageROI image_roi = cluster.rois.get(); - cv::Rect rect; - rs::conversion::from(image_roi.roi_hires.get(), rect); +void RSKNN::processCaffeFeatureKNN(std::string set_mode, std::string feature_use, + std::vector clusters, + cv::Mat &color, std::vector models_label, uima::CAS &tcas) { + //clusters comming from RS pipeline............................ + outInfo("Number of cluster:" << clusters.size() << std::endl); - //Draw result on image........... - drawCluster(color, rect, classLabelInString); + std::vector threads; - outInfo("calculation is done" << std::endl); + //multithreading the iterations for the clusters + for (int i = 0; i < clusters.size(); ++i) { + threads.push_back(std::thread(&RSKNN::calculateCaffeFeatureKNN, this, set_mode, feature_use, clusters, + std::ref(color), models_label, std::ref(tcas), i)); + } + for (auto &th : threads) { + th.join(); } - } } -void RSKNN::processCaffeFeatureKNN(std::string set_mode, std::string feature_use, std::vector clusters, - cv::Mat &color, std::vector models_label, uima::CAS &tcas) -{ - //clusters comming from RS pipeline............................ - outInfo("Number of cluster:" << clusters.size() << std::endl); - - for(size_t i = 0; i < clusters.size(); ++i) { +void RSKNN::calculateCaffeFeatureKNN(std::string set_mode, std::string feature_use, + std::vector clusters, + cv::Mat &color, std::vector models_label, uima::CAS &tcas, int i) { rs::ObjectHypothesis &cluster = clusters[i]; std::vector features; cluster.annotations.filter(features); - for(size_t j = 0; j < features.size(); ++j) { - rs::Features &feats = features[j]; - //variable to store caffe feature.......... - cv::Mat featDescriptor; - - if(feats.source() == "Caffe") { - rs::conversion::from(feats.descriptors(), featDescriptor); - outInfo("Size after conversion:" << featDescriptor.size()); - - //The function generate the prediction result................ - - std::pair result = classifyOnLiveDataKNN(featDescriptor); - int classLabelInInt = result.first; - double confidence = result.second; - - //class label in integer, which is used as index of vector model_label. - std::string classLabelInString = models_label[classLabelInInt - 1]; - outInfo("prediction class is :" << classLabelInInt <<" class label being: " < result = classifyOnLiveDataKNN(featDescriptor); + int classLabelInInt = result.first; + double confidence = result.second; + + //class label in integer, which is used as index of vector model_label. + std::string classLabelInString = models_label[classLabelInInt - 1]; + outInfo("prediction class is :" << classLabelInInt << " class label being: " << classLabelInString + << " with confidence: " << confidence); + //To annotate the clusters.................. + annotate_hypotheses(tcas, classLabelInString, feature_use, cluster, set_mode, confidence); + + //set roi on image + rs::ImageROI image_roi = cluster.rois.get(); + cv::Rect rect; + rs::conversion::from(image_roi.roi_hires.get(), rect); + + //Draw result on image........................... + drawCluster(color, rect, classLabelInString, confidence); + } + outInfo("calculation is done: thread " << i); } - } } -void RSKNN::annotate_hypotheses(uima::CAS &tcas, std::string class_name, std::string feature_name, rs::ObjectHypothesis &cluster, std::string set_mode, double &confi) -{ if(confi> 0.6) - {rs::Classification classResult = rs::create(tcas); - classResult.classname.set(class_name); - classResult.classifier("k-Nearest Neighbor"); - classResult.featurename(feature_name); - classResult.source.set("Knn"); - rs::ClassConfidence confidence = rs::create(tcas); - confidence.score.set(confi); - confidence.name.set(class_name); - classResult.confidences.set({confidence}); - - if(feature_name == "BVLC_REF") { - classResult.classification_type("INSTANCE"); - } - else if(feature_name == "VFH") { - classResult.classification_type("INSTANCE"); - } - - if(set_mode == "CL") { - cluster.annotations.append(classResult); - } - else if(set_mode == "GT") { - rs::GroundTruth setGT = rs::create(tcas); - setGT.classificationGT.set(classResult); - cluster.annotations.append(setGT); - } - else { - outError("You should set the parameter (set_mode) as CL or GT"); - }} +void RSKNN::annotate_hypotheses(uima::CAS &tcas, std::string class_name, std::string feature_name, rs::ObjectHypothesis &cluster, std::string set_mode, double &confi) { + if (confi > 0.6) { + rs::Classification classResult = rs::create(tcas); + classResult.classname.set(class_name); + classResult.classifier("k-Nearest Neighbor"); + classResult.featurename(feature_name); + classResult.source.set("Knn"); + rs::ClassConfidence confidence = rs::create(tcas); + confidence.score.set(confi); + confidence.name.set(class_name); + classResult.confidences.set({confidence}); + + if (feature_name == "BVLC_REF") { + classResult.classification_type("INSTANCE"); + } else if (feature_name == "VFH") { + classResult.classification_type("INSTANCE"); + } + + if (set_mode == "CL") { + cluster.annotations.append(classResult); + cluster.annotations.append(confidence); + } else if (set_mode == "GT") { + rs::GroundTruth setGT = rs::create(tcas); + setGT.classificationGT.set(classResult); + cluster.annotations.append(setGT); + } else { + outError("You should set the parameter (set_mode) as CL or GT"); + } + } } -RSKNN::~RSKNN() -{ -} + RSKNN::~RSKNN() { + }