From f748b3ded8a3904c8dba2216e9b2bb536474fef5 Mon Sep 17 00:00:00 2001 From: zcy <290198252@qq.com> Date: Thu, 19 May 2022 18:01:35 +0800 Subject: [PATCH] no message --- client/webrtc_capture/src/cv_ssd.cpp | 64 +++++++++ client/webrtc_capture/src/cv_ssd.h | 17 +++ client/webrtc_capture/src/cv_yolo.cpp | 168 +++++++++++++++++++++++ client/webrtc_capture/src/cv_yolo.h | 19 +++ client/webrtc_capture/src/main.cpp | 28 +++- client/webrtc_capture/src/mainwindow.cpp | 34 +++-- client/webrtc_capture/src/mainwindow.h | 2 + client/webrtc_capture/src/mainwindow.ui | 9 +- client/webrtc_capture/webrtc_capture.pro | 10 +- 9 files changed, 332 insertions(+), 19 deletions(-) create mode 100644 client/webrtc_capture/src/cv_ssd.cpp create mode 100644 client/webrtc_capture/src/cv_ssd.h create mode 100644 client/webrtc_capture/src/cv_yolo.cpp create mode 100644 client/webrtc_capture/src/cv_yolo.h diff --git a/client/webrtc_capture/src/cv_ssd.cpp b/client/webrtc_capture/src/cv_ssd.cpp new file mode 100644 index 0000000..2f04590 --- /dev/null +++ b/client/webrtc_capture/src/cv_ssd.cpp @@ -0,0 +1,64 @@ +#include "cv_ssd.h" +#include +const size_t width = 1280; +const size_t height = 720; +String labelFile = "ssd/labelmap_det.txt"; +String modelFile = "D:/project/multimedia/client/build-webrtc_capture-Desktop_Qt_5_15_2_MSVC2019_64bit-Debug/debug/ssd/MobileNetSSD_deploy.caffemodel"; +String model_text_file = "D:/project/multimedia/client/build-webrtc_capture-Desktop_Qt_5_15_2_MSVC2019_64bit-Debug/debug/ssd/MobileNetSSD_deploy.prototxt"; + +String objNames[] = { "background", +"aeroplane", "bicycle", "bird", "boat", +"bottle", "bus", "car", "cat", "chair", +"cow", "diningtable", "dog", "horse", +"motorbike", "person", "pottedplant", +"sheep", "sofa", "train", "tvmonitor" }; + + +cv::Mat * ssd_detect(cv::Mat *inframe) { + if (inframe->empty()) { + printf("could not load image...\n"); + return inframe; + } + namedWindow("input image", WINDOW_AUTOSIZE); + imshow("input image", *inframe); + + Net net = readNetFromCaffe(model_text_file, modelFile); + +// image：这个就是我们将要输入神经网络进行处理或者分类的图片。 +// mean：需要将图片整体减去的平均值，如果我们需要对RGB图片的三个通道分别减去不同的值，那么可以使用3组平均值，如果只使用一组，那么就默认对三个通道减去一样的值。减去平均值（mean）：为了消除同一场景下不同光照的图片，对我们最终的分类或者神经网络的影响，我们常常对图片的R、G、B通道的像素求一个平均值，然后将每个像素值减去我们的平均值，这样就可以得到像素之间的相对值，就可以排除光照的影响。 +// scalefactor：当我们将图片减去平均值之后，还可以对剩下的像素值进行一定的尺度缩放，它的默认值是1，如果希望减去平均像素之后的值，全部缩小一半，那么可以将scalefactor设为1/2。 +// size：这个参数是我们神经网络在训练的时候要求输入的图片尺寸。 +// swapRB：OpenCV中认为我们的图片通道顺序是BGR，但是我平均值假设的顺序是RGB，所以如果需要交换R和G，那么就要使swapRB=true + Mat blobImage = blobFromImage(*inframe, 0.007843, + Size(300, 300), + Scalar(127.5, 127.5, 127.5), true, false); + qDebug()<<"blobImage width : " << blobImage.cols + <<"blobImage.cols: " << blobImage.rows; + + imshow("ssd-demo", blobImage); + +// net.setInput(blobImage, "data"); +// Mat detection = net.forward("detection_out"); +// Mat detectionMat(detection.size[2], detection.size[3], CV_32F, detection.ptr()); +// float confidence_threshold = 0.2; +// for (int i = 0; i < detectionMat.rows; i++) { +// float confidence = detectionMat.at(i, 2); +// if (confidence > confidence_threshold) { +// size_t objIndex = (size_t)(detectionMat.at(i, 1)); +// float tl_x = detectionMat.at(i, 3) * frame->cols; +// float tl_y = detectionMat.at(i, 4) * frame->rows; +// float br_x = detectionMat.at(i, 5) * frame->cols; +// float br_y = detectionMat.at(i, 6) * frame->rows; + +// Rect object_box((int)tl_x, (int)tl_y, (int)(br_x - tl_x), (int)(br_y - tl_y)); +// rectangle(*frame, object_box, Scalar(0, 0, 255), 2, 8, 0); +// putText(*frame, format("%s", objNames[objIndex].c_str()), Point(tl_x, tl_y), +// FONT_HERSHEY_SIMPLEX, 1.0, Scalar(255, 0, 0), 2); +// } +// } + +// waitKey(0); + return inframe; +} + + diff --git a/client/webrtc_capture/src/cv_ssd.h b/client/webrtc_capture/src/cv_ssd.h new file mode 100644 index 0000000..66f20db --- /dev/null +++ b/client/webrtc_capture/src/cv_ssd.h @@ -0,0 +1,17 @@ +#ifndef CV_SSD_H +#define CV_SSD_H + +#include +#include +#include + +using namespace cv; +using namespace cv::dnn; +using namespace std; + +cv::Mat * ssd_detect(cv::Mat*); + + + + +#endif // CV_SSD_H diff --git a/client/webrtc_capture/src/cv_yolo.cpp b/client/webrtc_capture/src/cv_yolo.cpp new file mode 100644 index 0000000..c20e7ca --- /dev/null +++ b/client/webrtc_capture/src/cv_yolo.cpp @@ -0,0 +1,168 @@ +#include "cv_yolo.h" + +float confidenceThreshold = 0.25; + +void test_yolo() +{ + image_detection(); +} + +void video_detection() { + String modelConfiguration = "D:/vcprojects/images/dnn/yolov2-tiny-voc/yolov2-tiny-voc.cfg"; + String modelBinary = "D:/vcprojects/images/dnn/yolov2-tiny-voc/yolov2-tiny-voc.weights"; + dnn::Net net = readNetFromDarknet(modelConfiguration, modelBinary); + if (net.empty()) + { + printf("Could not load net...\n"); + return; + } + + vector classNamesVec; + ifstream classNamesFile("D:/vcprojects/images/dnn/yolov2-tiny-voc/voc.names"); + if (classNamesFile.is_open()) + { + string className = ""; + while (std::getline(classNamesFile, className)) + classNamesVec.push_back(className); + } + + // VideoCapture capture(0); + VideoCapture capture; + capture.open("D:/vcprojects/images/fbb.avi"); + if (!capture.isOpened()) { + printf("could not open the camera...\n"); + return; + } + + Mat frame; + while (capture.read(frame)) + { + if (frame.empty()) + if (frame.channels() == 4) + cvtColor(frame, frame, COLOR_BGRA2BGR); + Mat inputBlob = blobFromImage(frame, 1 / 255.F, Size(416, 416), Scalar(), true, false); + net.setInput(inputBlob, "data"); + Mat detectionMat = net.forward("detection_out"); + vector layersTimings; + double freq = getTickFrequency() / 1000; + double time = net.getPerfProfile(layersTimings) / freq; + ostringstream ss; + ss << "FPS: " << 1000 / time << " ; time: " << time << " ms"; + putText(frame, ss.str(), Point(20, 20), 0, 0.5, Scalar(0, 0, 255)); + + for (int i = 0; i < detectionMat.rows; i++) + { + const int probability_index = 5; + const int probability_size = detectionMat.cols - probability_index; + float *prob_array_ptr = &detectionMat.at(i, probability_index); + size_t objectClass = max_element(prob_array_ptr, prob_array_ptr + probability_size) - prob_array_ptr; + float confidence = detectionMat.at(i, (int)objectClass + probability_index); + if (confidence > confidenceThreshold) + { + float x = detectionMat.at(i, 0); + float y = detectionMat.at(i, 1); + float width = detectionMat.at(i, 2); + float height = detectionMat.at(i, 3); + int xLeftBottom = static_cast((x - width / 2) * frame.cols); + int yLeftBottom = static_cast((y - height / 2) * frame.rows); + int xRightTop = static_cast((x + width / 2) * frame.cols); + int yRightTop = static_cast((y + height / 2) * frame.rows); + Rect object(xLeftBottom, yLeftBottom, + xRightTop - xLeftBottom, + yRightTop - yLeftBottom); + rectangle(frame, object, Scalar(0, 255, 0)); + if (objectClass < classNamesVec.size()) + { + ss.str(""); + ss << confidence; + String conf(ss.str()); + String label = String(classNamesVec[objectClass]) + ": " + conf; + int baseLine = 0; + Size labelSize = getTextSize(label, FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine); + rectangle(frame, Rect(Point(xLeftBottom, yLeftBottom), + Size(labelSize.width, labelSize.height + baseLine)), + Scalar(255, 255, 255), FILLED); + putText(frame, label, Point(xLeftBottom, yLeftBottom + labelSize.height), + FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 0, 0)); + } + } + } + imshow("YOLOv3: Detections", frame); + if (waitKey(1) >= 0) break; + } +} + +void image_detection() { + String modelConfiguration = "D:/vcprojects/images/dnn/yolov2-tiny-voc/yolov2-tiny-voc.cfg"; + String modelBinary = "D:/vcprojects/images/dnn/yolov2-tiny-voc/yolov2-tiny-voc.weights"; + dnn::Net net = readNetFromDarknet(modelConfiguration, modelBinary); + if (net.empty()) + { + printf("Could not load net...\n"); + return; + } + vector classNamesVec; + ifstream classNamesFile("D:/vcprojects/images/dnn/yolov2-tiny-voc/voc.names"); + if (classNamesFile.is_open()) + { + string className = ""; + while (std::getline(classNamesFile, className)) + classNamesVec.push_back(className); + } + + // ͼ + Mat frame = imread("D:/vcprojects/images/fastrcnn.jpg"); + Mat inputBlob = blobFromImage(frame, 1 / 255.F, Size(416, 416), Scalar(), true, false); + net.setInput(inputBlob, "data"); + + // + Mat detectionMat = net.forward("detection_out"); + vector layersTimings; + double freq = getTickFrequency() / 1000; + double time = net.getPerfProfile(layersTimings) / freq; + ostringstream ss; + ss << "detection time: " << time << " ms"; + putText(frame, ss.str(), Point(20, 20), 0, 0.5, Scalar(0, 0, 255)); + + // + for (int i = 0; i < detectionMat.rows; i++) + { + const int probability_index = 5; + const int probability_size = detectionMat.cols - probability_index; + float *prob_array_ptr = &detectionMat.at(i, probability_index); + size_t objectClass = max_element(prob_array_ptr, prob_array_ptr + probability_size) - prob_array_ptr; + float confidence = detectionMat.at(i, (int)objectClass + probability_index); + if (confidence > confidenceThreshold) + { + float x = detectionMat.at(i, 0); + float y = detectionMat.at(i, 1); + float width = detectionMat.at(i, 2); + float height = detectionMat.at(i, 3); + int xLeftBottom = static_cast((x - width / 2) * frame.cols); + int yLeftBottom = static_cast((y - height / 2) * frame.rows); + int xRightTop = static_cast((x + width / 2) * frame.cols); + int yRightTop = static_cast((y + height / 2) * frame.rows); + Rect object(xLeftBottom, yLeftBottom, + xRightTop - xLeftBottom, + yRightTop - yLeftBottom); + rectangle(frame, object, Scalar(0, 0, 255), 2, 8); + if (objectClass < classNamesVec.size()) + { + ss.str(""); + ss << confidence; + String conf(ss.str()); + String label = String(classNamesVec[objectClass]) + ": " + conf; + int baseLine = 0; + Size labelSize = getTextSize(label, FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine); + rectangle(frame, Rect(Point(xLeftBottom, yLeftBottom), + Size(labelSize.width, labelSize.height + baseLine)), + Scalar(255, 255, 255), FILLED); + putText(frame, label, Point(xLeftBottom, yLeftBottom + labelSize.height), + FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 0, 0)); + } + } + } + imshow("YOLO-Detections", frame); + waitKey(0); + return; +} diff --git a/client/webrtc_capture/src/cv_yolo.h b/client/webrtc_capture/src/cv_yolo.h new file mode 100644 index 0000000..71fca6b --- /dev/null +++ b/client/webrtc_capture/src/cv_yolo.h @@ -0,0 +1,19 @@ +#ifndef CV_YOLO_H +#define CV_YOLO_H + +#include +#include + +#include +#include +#include +#include +using namespace std; +using namespace cv; +using namespace cv::dnn; + + +void video_detection(); +void image_detection(); +void test_yolo(); +#endif // CV_YOLO_H diff --git a/client/webrtc_capture/src/main.cpp b/client/webrtc_capture/src/main.cpp index ebbfe32..2bb2190 100644 --- a/client/webrtc_capture/src/main.cpp +++ b/client/webrtc_capture/src/main.cpp @@ -1,7 +1,7 @@ #include "mainwindow.h" #include "rtc.h" #include - +#include "cv_yolo.h" #include "modules/video_capture/video_capture.h" #include "video_source_impl.h" #include @@ -11,6 +11,7 @@ #include "camera_video_sink.h" #include "video_source_impl.h" #include +#include "cv_ssd.h" # pragma comment(lib, "secur32.lib") @@ -21,6 +22,21 @@ # pragma comment(lib, "Strmiids.lib") # pragma comment(lib, "User32.lib") + +int BubbleSort(int *p,int len){ + if(nullptr == p) + return -1; + for(int i = 0;i < len - 1;i++){ + for(int j = 0;j < len - 1;j++){ + if(p[j] < p[j + 1]){ + int tmp = p[j]; + p[j] = p[j+1]; + p[j+1] = tmp; + } + } + } +} + void EnumCapture() { rtc::WinsockInitializer winsock_init; @@ -29,7 +45,6 @@ void EnumCapture() rtc::ThreadManager::Instance()->SetCurrentThread(&w32_thread); rtc::InitializeSSL(); - std::unique_ptr info( webrtc::VideoCaptureFactory::CreateDeviceInfo()); @@ -56,6 +71,11 @@ void EnumCapture() int main(int argc, char *argv[]) { + int p[5] = {3,1,49,23,23}; + BubbleSort(p,5); + for(int i = 0;i< 5;i++){ + qDebug()<>("rtc::scoped_refptr"); qRegisterMetaType>("rtc::scoped_refptr&"); @@ -73,3 +93,7 @@ int main(int argc, char *argv[]) w.show(); return a.exec(); } + + + + diff --git a/client/webrtc_capture/src/mainwindow.cpp b/client/webrtc_capture/src/mainwindow.cpp index b65773c..7bab47c 100644 --- a/client/webrtc_capture/src/mainwindow.cpp +++ b/client/webrtc_capture/src/mainwindow.cpp @@ -8,6 +8,7 @@ #include #include #include "cvhelper.h" +#include "cv_ssd.h" class AsyncRennder :public QSSASyncProcess{ public: @@ -168,20 +169,25 @@ cv::Mat *QImage2cvMat(QImage image) void MainWindow::on_pushButton_2_clicked() { QPixmap pix = ui->openGLWidget->grab(); - QImage image = pix.toImage(); - image = image.scaled(1280,720); origin_picture->setPixmap(pix); - qDebug()<<"format is "<processed_picture->setPixmap(QPixmap::fromImage(image)); +} + +// 识别 +void MainWindow::on_pushButton_3_clicked() +{ + if(nullptr != origin_picture->pixmap()){ + QImage image = origin_picture->pixmap()->toImage(); + image = image.scaled(1280,720); + cv::Mat pic = *QImage2cvMat(image); + + cv::Mat *result = ssd_detect(&pic); +// try{ +//// cv::imshow("img", *result); +// }catch (std::exception &e){ +// qDebug()<processed_picture->setPixmap(QPixmap::fromImage(image)); + } } diff --git a/client/webrtc_capture/src/mainwindow.h b/client/webrtc_capture/src/mainwindow.h index e9aec31..bacffd4 100644 --- a/client/webrtc_capture/src/mainwindow.h +++ b/client/webrtc_capture/src/mainwindow.h @@ -23,6 +23,8 @@ private slots: void RenderDone(); void on_pushButton_2_clicked(); + void on_pushButton_3_clicked(); + private: Ui::MainWindow *ui; std::unique_ptr m_capturer; diff --git a/client/webrtc_capture/src/mainwindow.ui b/client/webrtc_capture/src/mainwindow.ui index 713bd32..47a59a7 100644 --- a/client/webrtc_capture/src/mainwindow.ui +++ b/client/webrtc_capture/src/mainwindow.ui @@ -22,7 +22,7 @@ - + @@ -63,6 +63,13 @@ + + + + 检测 + + + diff --git a/client/webrtc_capture/webrtc_capture.pro b/client/webrtc_capture/webrtc_capture.pro index 00cd386..0c0427a 100644 --- a/client/webrtc_capture/webrtc_capture.pro +++ b/client/webrtc_capture/webrtc_capture.pro @@ -23,6 +23,8 @@ INCLUDEPATH += third/include/ SOURCES += \ src/camera_video_sink.cpp \ src/cplaywidget.cpp \ + src/cv_ssd.cpp \ + src/cv_yolo.cpp \ src/cvhelper.cpp \ src/main.cpp \ src/mainwindow.cpp \ @@ -31,6 +33,8 @@ SOURCES += \ HEADERS += \ src/camera_video_sink.h \ src/cplaywidget.h \ + src/cv_ssd.h \ + src/cv_yolo.h \ src/cvhelper.h \ src/mainwindow.h \ src/rtc.h \ @@ -46,8 +50,10 @@ else: unix:!android: target.path = /opt/$${TARGET}/bin CONFIG(debug, debug|release){ message("debug mode") - LIBS += -L$$PWD/third/lib libwebrtc.lib ole32.lib oleaut32.lib strmiids.lib advapi32.lib opencv_core455d.lib - LIBS +=opencv_stitching455d.lib opencv_objdetect455d.lib opencv_ml455d.lib opencv_imgcodecs455d.lib opencv_imgproc455d.lib + LIBS += -L$$PWD/third/lib libwebrtc.lib ole32.lib oleaut32.lib strmiids.lib + LIBS += advapi32.lib opencv_core455d.lib + LIBS += opencv_stitching455d.lib opencv_objdetect455d.lib opencv_videoio455d.lib + LIBS += opencv_ml455d.lib opencv_imgcodecs455d.lib opencv_imgproc455d.lib opencv_dnn455d.lib LIBS+= opencv_highgui455d.lib DEFINES += DEBUG_FLAG Qt += debug