From 0f1328b2a14e84397c5781c06bf66fdbfa8574c4 Mon Sep 17 00:00:00 2001 From: Marko Bjelonic Date: Thu, 8 Apr 2021 10:25:11 +0200 Subject: [PATCH 1/8] First changes in CMakeLists. --- darknet_ros/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/darknet_ros/CMakeLists.txt b/darknet_ros/CMakeLists.txt index 56fcfba3d..17e388b45 100644 --- a/darknet_ros/CMakeLists.txt +++ b/darknet_ros/CMakeLists.txt @@ -1,8 +1,8 @@ -cmake_minimum_required(VERSION 2.8.12) +cmake_minimum_required(VERSION 3.5.1) project(darknet_ros) # Set c++11 cmake flags -set(CMAKE_CXX_FLAGS "-std=c++11 ${CMAKE_CXX_FLAGS}") +set(CMAKE_CXX_STANDARD 11) set(CMAKE_C_FLAGS "-Wall -Wno-unused-result -Wno-unknown-pragmas -Wno-unused-variable -Wfatal-errors -fPIC ${CMAKE_C_FLAGS}") set(CMAKE_EXPORT_COMPILE_COMMANDS ON) From 078cdfd6bd019be8e53514c24abb7c70d1b4a77e Mon Sep 17 00:00:00 2001 From: Marko Bjelonic Date: Thu, 8 Apr 2021 12:54:30 +0200 Subject: [PATCH 2/8] Updated to newest darknet and fixed opencv on Ubuntu 20.04. --- darknet | 2 +- darknet_ros/CMakeLists.txt | 12 +++---- .../darknet_ros/YoloObjectDetector.hpp | 19 +++++----- ...{image_interface.h => image_interface.hpp} | 3 +- darknet_ros/src/YoloObjectDetector.cpp | 35 ++++++++----------- ...{image_interface.c => image_interface.cpp} | 12 +++---- darknet_ros/test/ObjectDetection.cpp | 2 +- 7 files changed, 39 insertions(+), 46 deletions(-) rename darknet_ros/include/darknet_ros/{image_interface.h => image_interface.hpp} (81%) rename darknet_ros/src/{image_interface.c => image_interface.cpp} (74%) diff --git a/darknet b/darknet index 508381b37..17d47744a 160000 --- a/darknet +++ b/darknet @@ -1 +1 @@ -Subproject commit 508381b37fe75e0e1a01bcb2941cb0b31eb0e4c9 +Subproject commit 17d47744a290a6d95f9f4d592e8955b5baa8615f diff --git a/darknet_ros/CMakeLists.txt b/darknet_ros/CMakeLists.txt index 17e388b45..807b0e571 100644 --- a/darknet_ros/CMakeLists.txt +++ b/darknet_ros/CMakeLists.txt @@ -23,12 +23,7 @@ if (CUDA_FOUND) CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS}; -O3 - -gencode arch=compute_30,code=sm_30 - -gencode arch=compute_35,code=sm_35 - -gencode arch=compute_50,code=[sm_50,compute_50] -gencode arch=compute_52,code=[sm_52,compute_52] - -gencode arch=compute_61,code=sm_61 - -gencode arch=compute_62,code=sm_62 ) add_definitions(-DGPU) else() @@ -92,7 +87,7 @@ include_directories( ) set(PROJECT_LIB_FILES - src/YoloObjectDetector.cpp src/image_interface.c + src/YoloObjectDetector.cpp src/image_interface.cpp ) set(DARKNET_CORE_FILES @@ -118,16 +113,17 @@ set(DARKNET_CORE_FILES ${DARKNET_PATH}/src/gru_layer.c ${DARKNET_PATH}/src/utils.c ${DARKNET_PATH}/src/upsample_layer.c ${DARKNET_PATH}/src/logistic_layer.c ${DARKNET_PATH}/src/l2norm_layer.c ${DARKNET_PATH}/src/yolo_layer.c + ${DARKNET_PATH}/src/iseg_layer.c ${DARKNET_PATH}/src/image_opencv.cpp ${DARKNET_PATH}/examples/art.c ${DARKNET_PATH}/examples/lsd.c - ${DARKNET_PATH}/examples/attention.c ${DARKNET_PATH}/examples/nightmare.c + ${DARKNET_PATH}/examples/nightmare.c ${DARKNET_PATH}/examples/instance-segmenter.c ${DARKNET_PATH}/examples/captcha.c ${DARKNET_PATH}/examples/regressor.c ${DARKNET_PATH}/examples/cifar.c ${DARKNET_PATH}/examples/rnn.c ${DARKNET_PATH}/examples/classifier.c ${DARKNET_PATH}/examples/segmenter.c ${DARKNET_PATH}/examples/coco.c ${DARKNET_PATH}/examples/super.c ${DARKNET_PATH}/examples/darknet.c ${DARKNET_PATH}/examples/tag.c ${DARKNET_PATH}/examples/detector.c ${DARKNET_PATH}/examples/yolo.c - ${DARKNET_PATH}/examples/go.c + ${DARKNET_PATH}/examples/go.c ${DARKNET_PATH}/examples/go.c ) set(DARKNET_CUDA_FILES diff --git a/darknet_ros/include/darknet_ros/YoloObjectDetector.hpp b/darknet_ros/include/darknet_ros/YoloObjectDetector.hpp index daef6ef67..304b09395 100644 --- a/darknet_ros/include/darknet_ros/YoloObjectDetector.hpp +++ b/darknet_ros/include/darknet_ros/YoloObjectDetector.hpp @@ -49,7 +49,6 @@ extern "C" { #include #include "box.h" #include "cost_layer.h" -#include "darknet_ros/image_interface.h" #include "detection_layer.h" #include "network.h" #include "parser.h" @@ -57,9 +56,12 @@ extern "C" { #include "utils.h" } -extern "C" void ipl_into_image(IplImage* src, image im); -extern "C" image ipl_to_image(IplImage* src); -extern "C" void show_image_cv(image p, const char* name, IplImage* disp); +// Image interface. +#include "darknet_ros/image_interface.hpp" + +extern "C" cv::Mat image_to_mat(image im); +extern "C" image mat_to_image(cv::Mat m); +extern "C" int show_image(image p, const char* name, int ms); namespace darknet_ros { @@ -70,9 +72,9 @@ typedef struct { } RosBox_; typedef struct { - IplImage* image; + cv::Mat image; std_msgs::Header header; -} IplImageWithHeader_; +} CvMatWithHeader_; class YoloObjectDetector { public: @@ -174,12 +176,11 @@ class YoloObjectDetector { image buffLetter_[3]; int buffId_[3]; int buffIndex_ = 0; - IplImage* ipl_; float fps_ = 0; float demoThresh_ = 0; float demoHier_ = .5; int running_ = 0; - + cv::Mat disp_; int demoDelay_ = 0; int demoFrame_ = 3; float** predictions_; @@ -234,7 +235,7 @@ class YoloObjectDetector { void yolo(); - IplImageWithHeader_ getIplImageWithHeader(); + CvMatWithHeader_ getCvMatWithHeader(); bool getImageStatus(void); diff --git a/darknet_ros/include/darknet_ros/image_interface.h b/darknet_ros/include/darknet_ros/image_interface.hpp similarity index 81% rename from darknet_ros/include/darknet_ros/image_interface.h rename to darknet_ros/include/darknet_ros/image_interface.hpp index 7fb3d49b1..705bff602 100644 --- a/darknet_ros/include/darknet_ros/image_interface.h +++ b/darknet_ros/include/darknet_ros/image_interface.hpp @@ -10,9 +10,10 @@ #define IMAGE_INTERFACE_H #include "image.h" +#include "opencv2/opencv.hpp" static float get_pixel(image m, int x, int y, int c); image** load_alphabet_with_file(char* datafile); -void generate_image(image p, IplImage* disp); +void generate_image(image p, cv::Mat& disp); #endif diff --git a/darknet_ros/src/YoloObjectDetector.cpp b/darknet_ros/src/YoloObjectDetector.cpp index 17cb41e7a..966972ace 100644 --- a/darknet_ros/src/YoloObjectDetector.cpp +++ b/darknet_ros/src/YoloObjectDetector.cpp @@ -368,9 +368,8 @@ void* YoloObjectDetector::detectInThread() { void* YoloObjectDetector::fetchInThread() { { boost::shared_lock lock(mutexImageCallback_); - IplImageWithHeader_ imageAndHeader = getIplImageWithHeader(); - IplImage* ROS_img = imageAndHeader.image; - ipl_into_image(ROS_img, buff_[buffIndex_]); + CvMatWithHeader_ imageAndHeader = getCvMatWithHeader(); + buff_[buffIndex_] = mat_to_image(imageAndHeader.image); headerBuff_[buffIndex_] = imageAndHeader.header; buffId_[buffIndex_] = actionId_; } @@ -380,8 +379,7 @@ void* YoloObjectDetector::fetchInThread() { } void* YoloObjectDetector::displayInThread(void* ptr) { - show_image_cv(buff_[(buffIndex_ + 1) % 3], "YOLO V3", ipl_); - int c = cv::waitKey(waitKeyDelay_); + int c = show_image(buff_[(buffIndex_ + 1) % 3], "YOLO", 1); if (c != -1) c = c % 256; if (c == 27) { demoDone_ = 1; @@ -424,7 +422,7 @@ void YoloObjectDetector::setupNetwork(char* cfgfile, char* weightfile, char* dat demoThresh_ = thresh; demoHier_ = hier; fullScreen_ = fullscreen; - printf("YOLO V3\n"); + printf("YOLO\n"); net_ = load_network(cfgfile, weightfile, 0); set_batch_network(net_, 1); } @@ -457,9 +455,8 @@ void YoloObjectDetector::yolo() { { boost::shared_lock lock(mutexImageCallback_); - IplImageWithHeader_ imageAndHeader = getIplImageWithHeader(); - IplImage* ROS_img = imageAndHeader.image; - buff_[0] = ipl_to_image(ROS_img); + CvMatWithHeader_ imageAndHeader = getCvMatWithHeader(); + buff_[0] = mat_to_image(imageAndHeader.image); headerBuff_[0] = imageAndHeader.header; } buff_[1] = copy_image(buff_[0]); @@ -469,17 +466,16 @@ void YoloObjectDetector::yolo() { buffLetter_[0] = letterbox_image(buff_[0], net_->w, net_->h); buffLetter_[1] = letterbox_image(buff_[0], net_->w, net_->h); buffLetter_[2] = letterbox_image(buff_[0], net_->w, net_->h); - ipl_ = cvCreateImage(cvSize(buff_[0].w, buff_[0].h), IPL_DEPTH_8U, buff_[0].c); + disp_ = image_to_mat(buff_[0]); int count = 0; - if (!demoPrefix_ && viewImage_) { - cv::namedWindow("YOLO V3", cv::WINDOW_NORMAL); + cv::namedWindow("YOLO", cv::WINDOW_NORMAL); if (fullScreen_) { - cv::setWindowProperty("YOLO V3", cv::WND_PROP_FULLSCREEN, cv::WINDOW_FULLSCREEN); + cv::setWindowProperty("YOLO", cv::WND_PROP_FULLSCREEN, cv::WINDOW_FULLSCREEN); } else { - cv::moveWindow("YOLO V3", 0, 0); - cv::resizeWindow("YOLO V3", 640, 480); + cv::moveWindow("YOLO", 0, 0); + cv::resizeWindow("YOLO", 640, 480); } } @@ -495,7 +491,7 @@ void YoloObjectDetector::yolo() { if (viewImage_) { displayInThread(0); } else { - generate_image(buff_[(buffIndex_ + 1) % 3], ipl_); + generate_image(buff_[(buffIndex_ + 1) % 3], disp_); } publishInThread(); } else { @@ -512,9 +508,8 @@ void YoloObjectDetector::yolo() { } } -IplImageWithHeader_ YoloObjectDetector::getIplImageWithHeader() { - IplImage* ROS_img = new IplImage(camImageCopy_); - IplImageWithHeader_ header = {.image = ROS_img, .header = imageHeader_}; +CvMatWithHeader_ YoloObjectDetector::getCvMatWithHeader() { + CvMatWithHeader_ header = {.image = camImageCopy_, .header = imageHeader_}; return header; } @@ -530,7 +525,7 @@ bool YoloObjectDetector::isNodeRunning(void) { void* YoloObjectDetector::publishInThread() { // Publish image. - cv::Mat cvImage = cv::cvarrToMat(ipl_); + cv::Mat cvImage = disp_; if (!publishDetectionImage(cv::Mat(cvImage))) { ROS_DEBUG("Detection image has not been broadcasted."); } diff --git a/darknet_ros/src/image_interface.c b/darknet_ros/src/image_interface.cpp similarity index 74% rename from darknet_ros/src/image_interface.c rename to darknet_ros/src/image_interface.cpp index e1ab068cb..d5dde86ad 100644 --- a/darknet_ros/src/image_interface.c +++ b/darknet_ros/src/image_interface.cpp @@ -6,7 +6,7 @@ * Institute: ETH Zurich, Robotic Systems Lab */ -#include "darknet_ros/image_interface.h" +#include "darknet_ros/image_interface.hpp" static float get_pixel(image m, int x, int y, int c) { assert(x < m.w && y < m.h && c < m.c); @@ -16,13 +16,13 @@ static float get_pixel(image m, int x, int y, int c) { image** load_alphabet_with_file(char* datafile) { int i, j; const int nsize = 8; - image** alphabets = calloc(nsize, sizeof(image)); + image** alphabets = (image**)calloc(nsize, sizeof(image)); char* labels = "/labels/%d_%d.png"; char* files = (char*)malloc(1 + strlen(datafile) + strlen(labels)); strcpy(files, datafile); strcat(files, labels); for (j = 0; j < nsize; ++j) { - alphabets[j] = calloc(128, sizeof(image)); + alphabets[j] = (image*)calloc(128, sizeof(image)); for (i = 32; i < 127; ++i) { char buff[256]; sprintf(buff, files, i, j); @@ -33,16 +33,16 @@ image** load_alphabet_with_file(char* datafile) { } #ifdef OPENCV -void generate_image(image p, IplImage* disp) { +void generate_image(image p, cv::Mat& disp) { int x, y, k; if (p.c == 3) rgbgr_image(p); // normalize_image(copy); - int step = disp->widthStep; + int step = disp.step; for (y = 0; y < p.h; ++y) { for (x = 0; x < p.w; ++x) { for (k = 0; k < p.c; ++k) { - disp->imageData[y * step + x * p.c + k] = (unsigned char)(get_pixel(p, x, y, k) * 255); + disp.data[y * step + x * p.c + k] = (unsigned char)(get_pixel(p, x, y, k) * 255); } } } diff --git a/darknet_ros/test/ObjectDetection.cpp b/darknet_ros/test/ObjectDetection.cpp index 4478924fd..f662a7bf1 100644 --- a/darknet_ros/test/ObjectDetection.cpp +++ b/darknet_ros/test/ObjectDetection.cpp @@ -70,7 +70,7 @@ bool sendImageToYolo(ros::NodeHandle nh, const std::string& pathToTestImage) { // Get test image cv_bridge::CvImagePtr cv_ptr(new cv_bridge::CvImage); - cv_ptr->image = cv::imread(pathToTestImage, CV_LOAD_IMAGE_COLOR); + cv_ptr->image = cv::imread(pathToTestImage, cv::IMREAD_COLOR); cv_ptr->encoding = sensor_msgs::image_encodings::RGB8; sensor_msgs::ImagePtr image = cv_ptr->toImageMsg(); From 2049ed612c52fba44ec45004cc1a7160abb78151 Mon Sep 17 00:00:00 2001 From: Marko Bjelonic Date: Thu, 8 Apr 2021 14:05:15 +0200 Subject: [PATCH 3/8] Changed submodule url. --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 000b92b76..7e7bece16 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "darknet"] path = darknet - url = https://github.com/pjreddie/darknet + url = https://github.com/leggedrobotics/darknet From 535ab157051eff100637bab40a333bf8daa5643c Mon Sep 17 00:00:00 2001 From: Marko Bjelonic Date: Thu, 8 Apr 2021 14:09:56 +0200 Subject: [PATCH 4/8] Updated commit destination. --- darknet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/darknet b/darknet index 17d47744a..d22bbf38b 160000 --- a/darknet +++ b/darknet @@ -1 +1 @@ -Subproject commit 17d47744a290a6d95f9f4d592e8955b5baa8615f +Subproject commit d22bbf38bd012f13d2b50c8d98149cd4a9889b7a From e4693eb5fa765bc2e5c0d8d56577b1cdda695c43 Mon Sep 17 00:00:00 2001 From: Marko Bjelonic Date: Thu, 8 Apr 2021 14:15:16 +0200 Subject: [PATCH 5/8] Updated readme. --- README.md | 2 +- darknet_ros/CMakeLists.txt | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 68baa0cc1..b687b5170 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This is a ROS package developed for object detection in camera images. You only look once (YOLO) is a state-of-the-art, real-time object detection system. In the following ROS package you are able to use YOLO (V3) on GPU and CPU. The pre-trained model of the convolutional neural network is able to detect pre-trained classes including the data set from VOC and COCO, or you can also create a network with your own detection objects. For more information about YOLO, Darknet, available training data and training YOLO see the following link: [YOLO: Real-Time Object Detection](http://pjreddie.com/darknet/yolo/). -The YOLO packages have been tested under ROS Melodic and Ubuntu 18.04. This is research code, expect that it changes often and any fitness for a particular purpose is disclaimed. +The YOLO packages have been tested under ROS Noetic and Ubuntu 20.04 (Please check the branches for melodic, foxy, and ROS2 versions). This is research code, expect that it changes often and any fitness for a particular purpose is disclaimed. **Author: [Marko Bjelonic](https://www.markobjelonic.com), marko.bjelonic@mavt.ethz.ch** diff --git a/darknet_ros/CMakeLists.txt b/darknet_ros/CMakeLists.txt index 807b0e571..919eb3b87 100644 --- a/darknet_ros/CMakeLists.txt +++ b/darknet_ros/CMakeLists.txt @@ -23,7 +23,12 @@ if (CUDA_FOUND) CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS}; -O3 + -gencode arch=compute_30,code=sm_30 + -gencode arch=compute_35,code=sm_35 + -gencode arch=compute_50,code=[sm_50,compute_50] -gencode arch=compute_52,code=[sm_52,compute_52] + -gencode arch=compute_61,code=sm_61 + -gencode arch=compute_62,code=sm_62 ) add_definitions(-DGPU) else() From 973f6683911e6081fdd714f019bd2be70e1d91f1 Mon Sep 17 00:00:00 2001 From: Marko Bjelonic Date: Thu, 8 Apr 2021 14:37:59 +0200 Subject: [PATCH 6/8] Minor fix in test. --- darknet_ros/test/ObjectDetection.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/darknet_ros/test/ObjectDetection.cpp b/darknet_ros/test/ObjectDetection.cpp index f662a7bf1..e1601300f 100644 --- a/darknet_ros/test/ObjectDetection.cpp +++ b/darknet_ros/test/ObjectDetection.cpp @@ -154,7 +154,8 @@ TEST(ObjectDetection, DetectANYmal) { pathToTestImage += "quadruped_anymal_and_person"; pathToTestImage += ".JPG"; - // Send dog image to yolo. + // Send ANYmal and person image to yolo. + ASSERT_TRUE(sendImageToYolo(nodeHandle, pathToTestImage)); ASSERT_TRUE(sendImageToYolo(nodeHandle, pathToTestImage)); ASSERT_TRUE(sendImageToYolo(nodeHandle, pathToTestImage)); From 6f53ddc33ee224f8b2e9824b0bc04350f4896e87 Mon Sep 17 00:00:00 2001 From: Marko Bjelonic Date: Thu, 8 Apr 2021 14:46:27 +0200 Subject: [PATCH 7/8] Minor change in Readme. --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b687b5170..4432a5f9b 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,11 @@ ## Overview -This is a ROS package developed for object detection in camera images. You only look once (YOLO) is a state-of-the-art, real-time object detection system. In the following ROS package you are able to use YOLO (V3) on GPU and CPU. The pre-trained model of the convolutional neural network is able to detect pre-trained classes including the data set from VOC and COCO, or you can also create a network with your own detection objects. For more information about YOLO, Darknet, available training data and training YOLO see the following link: [YOLO: Real-Time Object Detection](http://pjreddie.com/darknet/yolo/). +This is a ROS package developed for **object detection in camera images**. You only look once (YOLO) is a state-of-the-art, real-time object detection system. In the following ROS package you are able to use **YOLO (V3) on GPU and CPU**. The pre-trained model of the convolutional neural network is able to detect pre-trained classes including the data set from VOC and COCO, or you can also create a network with your own detection objects. For more information about YOLO, Darknet, available training data and training YOLO see the following link: [YOLO: Real-Time Object Detection](http://pjreddie.com/darknet/yolo/). -The YOLO packages have been tested under ROS Noetic and Ubuntu 20.04 (Please check the branches for melodic, foxy, and ROS2 versions). This is research code, expect that it changes often and any fitness for a particular purpose is disclaimed. +The YOLO packages have been tested under **ROS Noetic** and **Ubuntu 20.04**. Note: We also provide branches that work under **ROS Melodic**, **ROS Foxy** and **ROS2**. + +This is research code, expect that it changes often and any fitness for a particular purpose is disclaimed. **Author: [Marko Bjelonic](https://www.markobjelonic.com), marko.bjelonic@mavt.ethz.ch** From d8514fb565753a4322649d7b735f2fac56187373 Mon Sep 17 00:00:00 2001 From: Marko Bjelonic Date: Thu, 8 Apr 2021 14:54:03 +0200 Subject: [PATCH 8/8] Minor fix in CMake. --- darknet_ros/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/darknet_ros/CMakeLists.txt b/darknet_ros/CMakeLists.txt index 919eb3b87..9a7be7da5 100644 --- a/darknet_ros/CMakeLists.txt +++ b/darknet_ros/CMakeLists.txt @@ -128,7 +128,7 @@ set(DARKNET_CORE_FILES ${DARKNET_PATH}/examples/coco.c ${DARKNET_PATH}/examples/super.c ${DARKNET_PATH}/examples/darknet.c ${DARKNET_PATH}/examples/tag.c ${DARKNET_PATH}/examples/detector.c ${DARKNET_PATH}/examples/yolo.c - ${DARKNET_PATH}/examples/go.c ${DARKNET_PATH}/examples/go.c + ${DARKNET_PATH}/examples/go.c ) set(DARKNET_CUDA_FILES