From ed2d2e9e09a8e559ed09643f06faa0b0ddc64b4b Mon Sep 17 00:00:00 2001
From: zcy <290198252@qq.com>
Date: Sat, 21 May 2022 20:44:46 +0800
Subject: [PATCH] no message

---
 .../webrtc_capture/src/camera_video_sink.cpp  |  7 +-
 client/webrtc_capture/src/cv_ssd.cpp          | 97 ++++++++++++++-----
 client/webrtc_capture/src/mainwindow.ui       |  9 +-
 3 files changed, 87 insertions(+), 26 deletions(-)

diff --git a/client/webrtc_capture/src/camera_video_sink.cpp b/client/webrtc_capture/src/camera_video_sink.cpp
index cd55558..ae7afac 100644
--- a/client/webrtc_capture/src/camera_video_sink.cpp
+++ b/client/webrtc_capture/src/camera_video_sink.cpp
@@ -76,20 +76,21 @@ void CameraVideoSink::OnFrame(const webrtc::VideoFrame& frame) {
             std::chrono::system_clock::now().time_since_epoch()).count();
     static size_t cnt = 0;
 
-    qDebug()<<int(frame.video_frame_buffer()->type());
+//    qDebug()<<int(frame.video_frame_buffer()->type());
     rtc::scoped_refptr<webrtc::I420BufferInterface> frameBuffer =
             frame.video_frame_buffer()->ToI420();
     cnt++;
     int width = this->m_capability.width;
     int height = this->m_capability.height;
+    /*
     qDebug()<<this->Capability().height<<this->Capability().width
            <<int(this->Capability().videoType)
           <<int(webrtc::VideoType::kI420)
-         <<this->Capability().interlaced<<width * height*4;
+         <<this->Capability().interlaced<<width * height*4;*/
 
     uint8_t *data =
             new uint8_t[width * height*4];
-    qDebug()<<width*height<<int(frameBuffer->GetI420()->type());
+//    qDebug()<<width*height<<int(frameBuffer->GetI420()->type());
     memcpy(data,frameBuffer->GetI420()->DataY(),width*height);
     memcpy(data + width*height ,frameBuffer->GetI420()->DataU(),
            width*height/4);
diff --git a/client/webrtc_capture/src/cv_ssd.cpp b/client/webrtc_capture/src/cv_ssd.cpp
index 2f04590..cbed0bf 100644
--- a/client/webrtc_capture/src/cv_ssd.cpp
+++ b/client/webrtc_capture/src/cv_ssd.cpp
@@ -13,29 +13,44 @@ String objNames[] = { "background",
 "motorbike", "person", "pottedplant",
 "sheep", "sofa", "train", "tvmonitor" };
 
-
 cv::Mat * ssd_detect(cv::Mat *inframe) {
     if (inframe->empty()) {
         printf("could not load image...\n");
         return inframe;
     }
     namedWindow("input image", WINDOW_AUTOSIZE);
-    imshow("input image", *inframe);
-
-    Net net = readNetFromCaffe(model_text_file, modelFile);
-
-//    image：这个就是我们将要输入神经网络进行处理或者分类的图片。
-//    mean：需要将图片整体减去的平均值，如果我们需要对RGB图片的三个通道分别减去不同的值，那么可以使用3组平均值，如果只使用一组，那么就默认对三个通道减去一样的值。减去平均值（mean）：为了消除同一场景下不同光照的图片，对我们最终的分类或者神经网络的影响，我们常常对图片的R、G、B通道的像素求一个平均值，然后将每个像素值减去我们的平均值，这样就可以得到像素之间的相对值，就可以排除光照的影响。
-//    scalefactor：当我们将图片减去平均值之后，还可以对剩下的像素值进行一定的尺度缩放，它的默认值是1，如果希望减去平均像素之后的值，全部缩小一半，那么可以将scalefactor设为1/2。
-//    size：这个参数是我们神经网络在训练的时候要求输入的图片尺寸。
-//    swapRB：OpenCV中认为我们的图片通道顺序是BGR，但是我平均值假设的顺序是RGB，所以如果需要交换R和G，那么就要使swapRB=true
-    Mat blobImage = blobFromImage(*inframe, 0.007843,
+    cv::Mat  resize;
+    cv::resize(*inframe, resize,
+        Size(480,560),
+        0, 0, INTER_LINEAR);// X Y各缩小一半
+    qDebug()<<"frame.type()"<<resize.type();
+    cv::Mat image2(480,640,CV_8UC3);
+    qDebug()<<"image2 ()"<<image2.type()<<CV_8UC3;
+    cv::cvtColor(resize,image2,COLOR_RGBA2RGB);
+    imshow("input image", image2);
+    qDebug()<<"image2 frame.type()"<<image2.type()<<CV_8UC3;
+    Mat blobImage1 = blobFromImage(image2, 0.007843,
         Size(300, 300),
         Scalar(127.5, 127.5, 127.5), true, false);
-    qDebug()<<"blobImage width : " << blobImage.cols
-           <<"blobImage.cols: " << blobImage.rows;
 
-    imshow("ssd-demo", blobImage);
+//    Net net = readNetFromCaffe(model_text_file, modelFile);
+
+////    image：这个就是我们将要输入神经网络进行处理或者分类的图片。
+////    mean：需要将图片整体减去的平均值，如果我们需要对RGB图片的三个通道分别减去不同的值，那么可以使用3组平均值，
+////    如果只使用一组，那么就默认对三个通道减去一样的值。减去平均值（mean）：为了消除同一场景下不同光照的图片，
+////     对我们最终的分类或者神经网络的影响，我们常常对图片的R、G、B通道的像素求一个平均值，
+////   然后将每个像素值减去我们的平均值，这样就可以得到像素之间的相对值，就可以排除光照的影响。
+////    scalefactor：当我们将图片减去平均值之后，还可以对剩下的像素值进行一定的尺度缩放，
+////    它的默认值是1，如果希望减去平均像素之后的值，全部缩小一半，那么可以将scalefactor设为1/2。
+////    size：这个参数是我们神经网络在训练的时候要求输入的图片尺寸。
+////    swapRB：OpenCV中认为我们的图片通道顺序是BGR，但是我平均值假设的顺序是RGB，
+/// 所以如果需要交换R和G，那么就要使swapRB=true
+//    cv::Mat blobImage = blobFromImage(resize,
+//        0.007843,
+//        Size(300, 300),
+//        Scalar(127.5, 127.5, 127.5), true, false);
+//    qDebug()<<"blobImage width : " << blobImage.cols
+//           <<"blobImage.cols: " << blobImage.rows;
 
 //    net.setInput(blobImage, "data");
 //    Mat detection = net.forward("detection_out");
@@ -45,20 +60,58 @@ cv::Mat * ssd_detect(cv::Mat *inframe) {
 //        float confidence = detectionMat.at<float>(i, 2);
 //        if (confidence > confidence_threshold) {
 //            size_t objIndex = (size_t)(detectionMat.at<float>(i, 1));
-//            float tl_x = detectionMat.at<float>(i, 3) * frame->cols;
-//            float tl_y = detectionMat.at<float>(i, 4) * frame->rows;
-//            float br_x = detectionMat.at<float>(i, 5) * frame->cols;
-//            float br_y = detectionMat.at<float>(i, 6) * frame->rows;
+//            float tl_x = detectionMat.at<float>(i, 3) * resize.cols;
+//            float tl_y = detectionMat.at<float>(i, 4) * resize.rows;
+//            float br_x = detectionMat.at<float>(i, 5) * resize.cols;
+//            float br_y = detectionMat.at<float>(i, 6) * resize.rows;
 
 //            Rect object_box((int)tl_x, (int)tl_y, (int)(br_x - tl_x), (int)(br_y - tl_y));
-//            rectangle(*frame, object_box, Scalar(0, 0, 255), 2, 8, 0);
-//            putText(*frame, format("%s", objNames[objIndex].c_str()), Point(tl_x, tl_y),
+//            rectangle(resize, object_box, Scalar(0, 0, 255), 2, 8, 0);
+//            putText(resize, format("%s", objNames[objIndex].c_str()), Point(tl_x, tl_y),
 //                    FONT_HERSHEY_SIMPLEX, 1.0, Scalar(255, 0, 0), 2);
 //        }
 //    }
+//    imshow("ssd-demo", resize);
 
-//    waitKey(0);
-    return inframe;
+//////    waitKey(0);
+//    return inframe;
+    Mat frame = imread("D:/project/opencv_tutorial_data/images/gaoyy.png");
+
+    if (frame.empty()) {
+        printf("could not load image...\n");
+        return nullptr;
+    }
+    qDebug()<<"frame.type()"<<frame.type()<<frame.cols<<frame.rows<<frame.depth()
+           <<frame.elemSize();
+    namedWindow("input image", WINDOW_AUTOSIZE);
+
+    Net net = readNetFromCaffe(model_text_file, modelFile);
+    Mat blobImage = blobFromImage(frame, 0.007843,
+        Size(300, 300),
+        Scalar(127.5, 127.5, 127.5), true, false);
+    qDebug()<<"blobImage width "<< blobImage.cols<<blobImage.rows;
+
+    net.setInput(blobImage1, "data");
+    Mat detection = net.forward("detection_out");
+    Mat detectionMat(detection.size[2], detection.size[3], CV_32F, detection.ptr<float>());
+    float confidence_threshold = 0.2;
+    for (int i = 0; i < detectionMat.rows; i++) {
+        float confidence = detectionMat.at<float>(i, 2);
+        if (confidence > confidence_threshold) {
+            size_t objIndex = (size_t)(detectionMat.at<float>(i, 1));
+            float tl_x = detectionMat.at<float>(i, 3) * image2.cols;
+            float tl_y = detectionMat.at<float>(i, 4) * image2.rows;
+            float br_x = detectionMat.at<float>(i, 5) * image2.cols;
+            float br_y = detectionMat.at<float>(i, 6) * image2.rows;
+
+            Rect object_box((int)tl_x, (int)tl_y, (int)(br_x - tl_x), (int)(br_y - tl_y));
+            rectangle(image2, object_box, Scalar(0, 0, 255), 2, 8, 0);
+            putText(image2, format("%s", objNames[objIndex].c_str()), Point(tl_x, tl_y), FONT_HERSHEY_SIMPLEX, 1.0, Scalar(255, 0, 0), 2);
+        }
+    }
+    imshow("ssd-demo", image2);
+    waitKey(0);
+    return nullptr;
 }
 
 
diff --git a/client/webrtc_capture/src/mainwindow.ui b/client/webrtc_capture/src/mainwindow.ui
index 47a59a7..5c231b8 100644
--- a/client/webrtc_capture/src/mainwindow.ui
+++ b/client/webrtc_capture/src/mainwindow.ui
@@ -22,7 +22,7 @@
   <widget class="QWidget" name="centralwidget">
    <layout class="QVBoxLayout" name="verticalLayout_2" stretch="1,9">
     <item>
-     <layout class="QHBoxLayout" name="horizontalLayout" stretch="1,1,1,1,1,0,0,3">
+     <layout class="QHBoxLayout" name="horizontalLayout" stretch="1,1,1,1,1,0,0,0,3">
       <item>
        <widget class="QLabel" name="label">
         <property name="text">
@@ -70,6 +70,13 @@
         </property>
        </widget>
       </item>
+      <item>
+       <widget class="QPushButton" name="pushButton_4">
+        <property name="text">
+         <string>动态检测</string>
+        </property>
+       </widget>
+      </item>
       <item>
        <spacer name="horizontalSpacer">
         <property name="orientation">