no message

client/webrtc_capture/src/cv_ssd.cpp

@@ -0,0 +1,64 @@
+#include "cv_ssd.h"
+#include <QDebug>
+const size_t width = 1280;
+const size_t height = 720;
+String labelFile = "ssd/labelmap_det.txt";
+String modelFile = "D:/project/multimedia/client/build-webrtc_capture-Desktop_Qt_5_15_2_MSVC2019_64bit-Debug/debug/ssd/MobileNetSSD_deploy.caffemodel";
+String model_text_file = "D:/project/multimedia/client/build-webrtc_capture-Desktop_Qt_5_15_2_MSVC2019_64bit-Debug/debug/ssd/MobileNetSSD_deploy.prototxt";
+
+String objNames[] = { "background",
+"aeroplane", "bicycle", "bird", "boat",
+"bottle", "bus", "car", "cat", "chair",
+"cow", "diningtable", "dog", "horse",
+"motorbike", "person", "pottedplant",
+"sheep", "sofa", "train", "tvmonitor" };
+
+
+cv::Mat * ssd_detect(cv::Mat *inframe) {
+    if (inframe->empty()) {
+        printf("could not load image...\n");
+        return inframe;
+    }
+    namedWindow("input image", WINDOW_AUTOSIZE);
+    imshow("input image", *inframe);
+
+    Net net = readNetFromCaffe(model_text_file, modelFile);
+
+//    image：这个就是我们将要输入神经网络进行处理或者分类的图片。
+//    mean：需要将图片整体减去的平均值，如果我们需要对RGB图片的三个通道分别减去不同的值，那么可以使用3组平均值，如果只使用一组，那么就默认对三个通道减去一样的值。减去平均值（mean）：为了消除同一场景下不同光照的图片，对我们最终的分类或者神经网络的影响，我们常常对图片的R、G、B通道的像素求一个平均值，然后将每个像素值减去我们的平均值，这样就可以得到像素之间的相对值，就可以排除光照的影响。
+//    scalefactor：当我们将图片减去平均值之后，还可以对剩下的像素值进行一定的尺度缩放，它的默认值是1，如果希望减去平均像素之后的值，全部缩小一半，那么可以将scalefactor设为1/2。
+//    size：这个参数是我们神经网络在训练的时候要求输入的图片尺寸。
+//    swapRB：OpenCV中认为我们的图片通道顺序是BGR，但是我平均值假设的顺序是RGB，所以如果需要交换R和G，那么就要使swapRB=true
+    Mat blobImage = blobFromImage(*inframe, 0.007843,
+        Size(300, 300),
+        Scalar(127.5, 127.5, 127.5), true, false);
+    qDebug()<<"blobImage width : " << blobImage.cols
+           <<"blobImage.cols: " << blobImage.rows;
+
+    imshow("ssd-demo", blobImage);
+
+//    net.setInput(blobImage, "data");
+//    Mat detection = net.forward("detection_out");
+//    Mat detectionMat(detection.size[2], detection.size[3], CV_32F, detection.ptr<float>());
+//    float confidence_threshold = 0.2;
+//    for (int i = 0; i < detectionMat.rows; i++) {
+//        float confidence = detectionMat.at<float>(i, 2);
+//        if (confidence > confidence_threshold) {
+//            size_t objIndex = (size_t)(detectionMat.at<float>(i, 1));
+//            float tl_x = detectionMat.at<float>(i, 3) * frame->cols;
+//            float tl_y = detectionMat.at<float>(i, 4) * frame->rows;
+//            float br_x = detectionMat.at<float>(i, 5) * frame->cols;
+//            float br_y = detectionMat.at<float>(i, 6) * frame->rows;
+
+//            Rect object_box((int)tl_x, (int)tl_y, (int)(br_x - tl_x), (int)(br_y - tl_y));
+//            rectangle(*frame, object_box, Scalar(0, 0, 255), 2, 8, 0);
+//            putText(*frame, format("%s", objNames[objIndex].c_str()), Point(tl_x, tl_y),
+//                    FONT_HERSHEY_SIMPLEX, 1.0, Scalar(255, 0, 0), 2);
+//        }
+//    }
+
+//    waitKey(0);
+    return inframe;
+}
+
+

client/webrtc_capture/src/cv_ssd.h

@@ -0,0 +1,17 @@
+#ifndef CV_SSD_H
+#define CV_SSD_H
+
+#include <opencv2/opencv.hpp>
+#include <opencv2/dnn.hpp>
+#include <iostream>
+
+using namespace cv;
+using namespace cv::dnn;
+using namespace std;
+
+cv::Mat * ssd_detect(cv::Mat*);
+
+
+
+
+#endif // CV_SSD_H

client/webrtc_capture/src/cv_yolo.cpp

@@ -0,0 +1,168 @@
+#include "cv_yolo.h"
+
+float confidenceThreshold = 0.25;
+
+void test_yolo()
+{
+    image_detection();
+}
+
+void video_detection() {
+    String modelConfiguration = "D:/vcprojects/images/dnn/yolov2-tiny-voc/yolov2-tiny-voc.cfg";
+    String modelBinary = "D:/vcprojects/images/dnn/yolov2-tiny-voc/yolov2-tiny-voc.weights";
+    dnn::Net net = readNetFromDarknet(modelConfiguration, modelBinary);
+    if (net.empty())
+    {
+        printf("Could not load net...\n");
+        return;
+    }
+
+    vector<string> classNamesVec;
+    ifstream classNamesFile("D:/vcprojects/images/dnn/yolov2-tiny-voc/voc.names");
+    if (classNamesFile.is_open())
+    {
+        string className = "";
+        while (std::getline(classNamesFile, className))
+            classNamesVec.push_back(className);
+    }
+
+    // VideoCapture capture(0);
+    VideoCapture capture;
+    capture.open("D:/vcprojects/images/fbb.avi");
+    if (!capture.isOpened()) {
+        printf("could not open the camera...\n");
+        return;
+    }
+
+    Mat frame;
+    while (capture.read(frame))
+    {
+        if (frame.empty())
+            if (frame.channels() == 4)
+                cvtColor(frame, frame, COLOR_BGRA2BGR);
+        Mat inputBlob = blobFromImage(frame, 1 / 255.F, Size(416, 416), Scalar(), true, false);
+        net.setInput(inputBlob, "data");
+        Mat detectionMat = net.forward("detection_out");
+        vector<double> layersTimings;
+        double freq = getTickFrequency() / 1000;
+        double time = net.getPerfProfile(layersTimings) / freq;
+        ostringstream ss;
+        ss << "FPS: " << 1000 / time << " ; time: " << time << " ms";
+        putText(frame, ss.str(), Point(20, 20), 0, 0.5, Scalar(0, 0, 255));
+
+        for (int i = 0; i < detectionMat.rows; i++)
+        {
+            const int probability_index = 5;
+            const int probability_size = detectionMat.cols - probability_index;
+            float *prob_array_ptr = &detectionMat.at<float>(i, probability_index);
+            size_t objectClass = max_element(prob_array_ptr, prob_array_ptr + probability_size) - prob_array_ptr;
+            float confidence = detectionMat.at<float>(i, (int)objectClass + probability_index);
+            if (confidence > confidenceThreshold)
+            {
+                float x = detectionMat.at<float>(i, 0);
+                float y = detectionMat.at<float>(i, 1);
+                float width = detectionMat.at<float>(i, 2);
+                float height = detectionMat.at<float>(i, 3);
+                int xLeftBottom = static_cast<int>((x - width / 2) * frame.cols);
+                int yLeftBottom = static_cast<int>((y - height / 2) * frame.rows);
+                int xRightTop = static_cast<int>((x + width / 2) * frame.cols);
+                int yRightTop = static_cast<int>((y + height / 2) * frame.rows);
+                Rect object(xLeftBottom, yLeftBottom,
+                    xRightTop - xLeftBottom,
+                    yRightTop - yLeftBottom);
+                rectangle(frame, object, Scalar(0, 255, 0));
+                if (objectClass < classNamesVec.size())
+                {
+                    ss.str("");
+                    ss << confidence;
+                    String conf(ss.str());
+                    String label = String(classNamesVec[objectClass]) + ": " + conf;
+                    int baseLine = 0;
+                    Size labelSize = getTextSize(label, FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine);
+                    rectangle(frame, Rect(Point(xLeftBottom, yLeftBottom),
+                        Size(labelSize.width, labelSize.height + baseLine)),
+                        Scalar(255, 255, 255), FILLED);
+                    putText(frame, label, Point(xLeftBottom, yLeftBottom + labelSize.height),
+                        FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 0, 0));
+                }
+            }
+        }
+        imshow("YOLOv3: Detections", frame);
+        if (waitKey(1) >= 0) break;
+    }
+}
+
+void image_detection() {
+    String modelConfiguration = "D:/vcprojects/images/dnn/yolov2-tiny-voc/yolov2-tiny-voc.cfg";
+    String modelBinary = "D:/vcprojects/images/dnn/yolov2-tiny-voc/yolov2-tiny-voc.weights";
+    dnn::Net net = readNetFromDarknet(modelConfiguration, modelBinary);
+    if (net.empty())
+    {
+        printf("Could not load net...\n");
+        return;
+    }
+    vector<string> classNamesVec;
+    ifstream classNamesFile("D:/vcprojects/images/dnn/yolov2-tiny-voc/voc.names");
+    if (classNamesFile.is_open())
+    {
+        string className = "";
+        while (std::getline(classNamesFile, className))
+            classNamesVec.push_back(className);
+    }
+
+    // ͼ
+    Mat frame = imread("D:/vcprojects/images/fastrcnn.jpg");
+    Mat inputBlob = blobFromImage(frame, 1 / 255.F, Size(416, 416), Scalar(), true, false);
+    net.setInput(inputBlob, "data");
+
+    //
+    Mat detectionMat = net.forward("detection_out");
+    vector<double> layersTimings;
+    double freq = getTickFrequency() / 1000;
+    double time = net.getPerfProfile(layersTimings) / freq;
+    ostringstream ss;
+    ss << "detection time: " << time << " ms";
+    putText(frame, ss.str(), Point(20, 20), 0, 0.5, Scalar(0, 0, 255));
+
+    //
+    for (int i = 0; i < detectionMat.rows; i++)
+    {
+        const int probability_index = 5;
+        const int probability_size = detectionMat.cols - probability_index;
+        float *prob_array_ptr = &detectionMat.at<float>(i, probability_index);
+        size_t objectClass = max_element(prob_array_ptr, prob_array_ptr + probability_size) - prob_array_ptr;
+        float confidence = detectionMat.at<float>(i, (int)objectClass + probability_index);
+        if (confidence > confidenceThreshold)
+        {
+            float x = detectionMat.at<float>(i, 0);
+            float y = detectionMat.at<float>(i, 1);
+            float width = detectionMat.at<float>(i, 2);
+            float height = detectionMat.at<float>(i, 3);
+            int xLeftBottom = static_cast<int>((x - width / 2) * frame.cols);
+            int yLeftBottom = static_cast<int>((y - height / 2) * frame.rows);
+            int xRightTop = static_cast<int>((x + width / 2) * frame.cols);
+            int yRightTop = static_cast<int>((y + height / 2) * frame.rows);
+            Rect object(xLeftBottom, yLeftBottom,
+                xRightTop - xLeftBottom,
+                yRightTop - yLeftBottom);
+            rectangle(frame, object, Scalar(0, 0, 255), 2, 8);
+            if (objectClass < classNamesVec.size())
+            {
+                ss.str("");
+                ss << confidence;
+                String conf(ss.str());
+                String label = String(classNamesVec[objectClass]) + ": " + conf;
+                int baseLine = 0;
+                Size labelSize = getTextSize(label, FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine);
+                rectangle(frame, Rect(Point(xLeftBottom, yLeftBottom),
+                    Size(labelSize.width, labelSize.height + baseLine)),
+                    Scalar(255, 255, 255), FILLED);
+                putText(frame, label, Point(xLeftBottom, yLeftBottom + labelSize.height),
+                    FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 0, 0));
+            }
+        }
+    }
+    imshow("YOLO-Detections", frame);
+    waitKey(0);
+    return;
+}

client/webrtc_capture/src/cv_yolo.h

@@ -0,0 +1,19 @@
+#ifndef CV_YOLO_H
+#define CV_YOLO_H
+
+#include <opencv2/opencv.hpp>
+#include <opencv2/dnn.hpp>
+
+#include <fstream>
+#include <iostream>
+#include <algorithm>
+#include <cstdlib>
+using namespace std;
+using namespace cv;
+using namespace cv::dnn;
+
+
+void video_detection();
+void image_detection();
+void test_yolo();
+#endif // CV_YOLO_H

client/webrtc_capture/src/main.cpp

@@ -1,7 +1,7 @@
 #include "mainwindow.h"
 #include "rtc.h"
 #include <QApplication>
-
+#include "cv_yolo.h"
 #include "modules/video_capture/video_capture.h"
 #include "video_source_impl.h"
 #include <QString>
@@ -11,6 +11,7 @@
 #include "camera_video_sink.h"
 #include "video_source_impl.h"
 #include <QMetaType>
+#include "cv_ssd.h"
 
 
 # pragma comment(lib, "secur32.lib")
@@ -21,6 +22,21 @@
 # pragma comment(lib, "Strmiids.lib")
 # pragma comment(lib, "User32.lib")
 
+
+int BubbleSort(int *p,int len){
+    if(nullptr ==  p)
+        return -1;
+    for(int i = 0;i < len - 1;i++){
+        for(int j = 0;j < len - 1;j++){
+            if(p[j] < p[j + 1]){
+                int tmp = p[j];
+                p[j] = p[j+1];
+                p[j+1] = tmp;
+            }
+        }
+    }
+}
+
 void EnumCapture()
 {
     rtc::WinsockInitializer winsock_init;
@@ -29,7 +45,6 @@ void EnumCapture()
     rtc::ThreadManager::Instance()->SetCurrentThread(&w32_thread);
     rtc::InitializeSSL();
 
-
     std::unique_ptr<webrtc::VideoCaptureModule::DeviceInfo> info(
       webrtc::VideoCaptureFactory::CreateDeviceInfo());
 
@@ -56,6 +71,11 @@ void EnumCapture()
 
 int main(int argc, char *argv[])
 {
+    int p[5] = {3,1,49,23,23};
+    BubbleSort(p,5);
+    for(int i = 0;i< 5;i++){
+        qDebug()<<p[i]<<"  ";
+    }
     qRegisterMetaType<rtc::scoped_refptr<webrtc::I420BufferInterface>>("rtc::scoped_refptr<webrtc::I420BufferInterface>");
     qRegisterMetaType<rtc::scoped_refptr<webrtc::I420BufferInterface>>("rtc::scoped_refptr<webrtc::I420BufferInterface>&");
 
@@ -73,3 +93,7 @@ int main(int argc, char *argv[])
     w.show();
     return a.exec();
 }
+
+
+
+

client/webrtc_capture/src/mainwindow.cpp

@@ -8,6 +8,7 @@
 #include <opencv2/highgui.hpp>
 #include <opencv2/imgproc.hpp>
 #include "cvhelper.h"
+#include "cv_ssd.h"
 
 class AsyncRennder :public QSSASyncProcess{
 public:
@@ -168,20 +169,25 @@ cv::Mat *QImage2cvMat(QImage image)
 void MainWindow::on_pushButton_2_clicked()
 {
     QPixmap pix = ui->openGLWidget->grab();
-    QImage image = pix.toImage();
-    image = image.scaled(1280,720);
     origin_picture->setPixmap(pix);
-    qDebug()<<"format is "<<image.format();
+}
-//    cv::Mat Img;
+
-
+// 识别
-    cv::Mat pic =  *QImage2cvMat(image);
+void MainWindow::on_pushButton_3_clicked()
-    try{
+{
-        cv::imshow("img", pic);
+    if(nullptr != origin_picture->pixmap()){
-    }catch (std::exception &e){
+        QImage image = origin_picture->pixmap()->toImage();
-        qDebug()<<e.what();
+        image = image.scaled(1280,720);
-    }
+        cv::Mat pic =  *QImage2cvMat(image);
-
+
-    cv::waitKey(1);
+        cv::Mat *result =  ssd_detect(&pic);
-    this->processed_picture->setPixmap(QPixmap::fromImage(image));
+//        try{
+////            cv::imshow("img", *result);
+//        }catch (std::exception &e){
+//            qDebug()<<e.what();
+//        }
+//        cv::waitKey(1);
+        this->processed_picture->setPixmap(QPixmap::fromImage(image));
+    }
 }
 

client/webrtc_capture/src/mainwindow.h

@@ -23,6 +23,8 @@ private slots:
     void RenderDone();
     void on_pushButton_2_clicked();
 
+    void on_pushButton_3_clicked();
+
 private:
     Ui::MainWindow *ui;
     std::unique_ptr<CameraVideoSink> m_capturer;

client/webrtc_capture/src/mainwindow.ui

@@ -22,7 +22,7 @@
   <widget class="QWidget" name="centralwidget">
    <layout class="QVBoxLayout" name="verticalLayout_2" stretch="1,9">
     <item>
-     <layout class="QHBoxLayout" name="horizontalLayout" stretch="1,1,1,1,1,0,3">
+     <layout class="QHBoxLayout" name="horizontalLayout" stretch="1,1,1,1,1,0,0,3">
       <item>
        <widget class="QLabel" name="label">
         <property name="text">
@@ -63,6 +63,13 @@
         </property>
        </widget>
       </item>
+      <item>
+       <widget class="QPushButton" name="pushButton_3">
+        <property name="text">
+         <string>检测</string>
+        </property>
+       </widget>
+      </item>
       <item>
        <spacer name="horizontalSpacer">
         <property name="orientation">

client/webrtc_capture/webrtc_capture.pro

@@ -23,6 +23,8 @@ INCLUDEPATH += third/include/
 SOURCES += \
     src/camera_video_sink.cpp \
     src/cplaywidget.cpp \
+    src/cv_ssd.cpp \
+    src/cv_yolo.cpp \
     src/cvhelper.cpp \
     src/main.cpp \
     src/mainwindow.cpp \
@@ -31,6 +33,8 @@ SOURCES += \
 HEADERS += \
     src/camera_video_sink.h \
     src/cplaywidget.h \
+    src/cv_ssd.h \
+    src/cv_yolo.h \
     src/cvhelper.h \
     src/mainwindow.h \
     src/rtc.h \
@@ -46,8 +50,10 @@ else: unix:!android: target.path = /opt/$${TARGET}/bin
 
 CONFIG(debug, debug|release){
     message("debug mode")
-    LIBS += -L$$PWD/third/lib libwebrtc.lib  ole32.lib oleaut32.lib strmiids.lib advapi32.lib opencv_core455d.lib
+    LIBS += -L$$PWD/third/lib libwebrtc.lib  ole32.lib oleaut32.lib strmiids.lib
-    LIBS +=opencv_stitching455d.lib opencv_objdetect455d.lib opencv_ml455d.lib opencv_imgcodecs455d.lib opencv_imgproc455d.lib
+    LIBS += advapi32.lib opencv_core455d.lib
+    LIBS += opencv_stitching455d.lib opencv_objdetect455d.lib opencv_videoio455d.lib
+    LIBS += opencv_ml455d.lib opencv_imgcodecs455d.lib opencv_imgproc455d.lib opencv_dnn455d.lib
     LIBS+= opencv_highgui455d.lib
     DEFINES += DEBUG_FLAG
     Qt += debug