From 1ba89778f5de940fe876a2dbd5af5be8408dd779 Mon Sep 17 00:00:00 2001
From: zcy <290198252@qq.com>
Date: Mon, 23 May 2022 01:20:11 +0800
Subject: [PATCH] add dynamic detect

---
 .../webrtc_capture/src/camera_video_sink.cpp  |   8 +-
 client/webrtc_capture/src/camera_video_sink.h |   2 -
 client/webrtc_capture/src/cplaywidget.cpp     |  26 +++-
 client/webrtc_capture/src/cplaywidget.h       |   2 +
 client/webrtc_capture/src/cv_ssd.cpp          |  72 ++--------
 client/webrtc_capture/src/mainwindow.cpp      | 134 ++++++++++++++----
 client/webrtc_capture/src/mainwindow.h        |   4 +
 7 files changed, 146 insertions(+), 102 deletions(-)

diff --git a/client/webrtc_capture/src/camera_video_sink.cpp b/client/webrtc_capture/src/camera_video_sink.cpp
index ae7afac..179f666 100644
--- a/client/webrtc_capture/src/camera_video_sink.cpp
+++ b/client/webrtc_capture/src/camera_video_sink.cpp
@@ -82,14 +82,8 @@ void CameraVideoSink::OnFrame(const webrtc::VideoFrame& frame) {
     cnt++;
     int width = this->m_capability.width;
     int height = this->m_capability.height;
-    /*
-    qDebug()<<this->Capability().height<<this->Capability().width
-           <<int(this->Capability().videoType)
-          <<int(webrtc::VideoType::kI420)
-         <<this->Capability().interlaced<<width * height*4;*/
 
-    uint8_t *data =
-            new uint8_t[width * height*4];
+    uint8_t *data = new uint8_t[width * height*4];
 //    qDebug()<<width*height<<int(frameBuffer->GetI420()->type());
     memcpy(data,frameBuffer->GetI420()->DataY(),width*height);
     memcpy(data + width*height ,frameBuffer->GetI420()->DataU(),
diff --git a/client/webrtc_capture/src/camera_video_sink.h b/client/webrtc_capture/src/camera_video_sink.h
index 4eb512f..9043a87 100644
--- a/client/webrtc_capture/src/camera_video_sink.h
+++ b/client/webrtc_capture/src/camera_video_sink.h
@@ -2,8 +2,6 @@
 #define CAMERA_VIDEO_SINK_H
 
 // vcm_capturer_test.h
-
-
 #include <memory>
 #include <QObject>
 #include "modules/video_capture/video_capture.h"
diff --git a/client/webrtc_capture/src/cplaywidget.cpp b/client/webrtc_capture/src/cplaywidget.cpp
index c2aca64..00c47d0 100644
--- a/client/webrtc_capture/src/cplaywidget.cpp
+++ b/client/webrtc_capture/src/cplaywidget.cpp
@@ -83,6 +83,28 @@ void CPlayWidget::OnPaintData(const rtc::scoped_refptr<webrtc::I420BufferInterfa
     update();
 }
 
+CPlayWidget::CPlayWidget(QWidget *parent, IMG_TYPE type)
+{
+    textureUniformY = 0;
+    textureUniformU = 0;
+    textureUniformV = 0;
+    id_y = 0;
+    id_u = 0;
+    id_v = 0;
+    m_pTextureRGB = nullptr;
+    m_pBufYuv420p = nullptr;
+    m_pVSHader = NULL;
+    m_pFSHader = NULL;
+    m_pShaderProgram = NULL;
+    m_pTextureY = NULL;
+    m_pTextureU = NULL;
+    m_pTextureV = NULL;
+    m_nVideoH = 0;
+    m_nVideoW = 0;
+    mType = type;
+    m_start_render = false;
+}
+
 CPlayWidget::CPlayWidget(QWidget *parent):QOpenGLWidget(parent) {
     textureUniformY = 0;
     textureUniformU = 0;
@@ -100,7 +122,7 @@ CPlayWidget::CPlayWidget(QWidget *parent):QOpenGLWidget(parent) {
     m_pTextureV = NULL;
     m_nVideoH = 0;
     m_nVideoW = 0;
-    mType = TYPE_I420;
+    mType = TYPE_YUV420P;
     m_start_render = false;
 }
 
@@ -138,6 +160,8 @@ void CPlayWidget::OnCameraData( rtc::scoped_refptr<webrtc::I420BufferInterface>
     update();
 }
 
+
+
 int CPlayWidget::OnCameraData(uint8_t *p)
 {
     memcpy(m_pBufYuv420p,p,m_nVideoH*m_nVideoW/2*3);
diff --git a/client/webrtc_capture/src/cplaywidget.h b/client/webrtc_capture/src/cplaywidget.h
index a257af5..8bd5a8a 100644
--- a/client/webrtc_capture/src/cplaywidget.h
+++ b/client/webrtc_capture/src/cplaywidget.h
@@ -36,6 +36,8 @@ public:
         TYPE_I420,
         TYPE_UNSET,
     }IMG_TYPE;
+    CPlayWidget(QWidget* parent,IMG_TYPE type);
+
     CPlayWidget(QWidget* parent);
     ~CPlayWidget();
     int SetDataType(IMG_TYPE);
diff --git a/client/webrtc_capture/src/cv_ssd.cpp b/client/webrtc_capture/src/cv_ssd.cpp
index cbed0bf..1a7e8c1 100644
--- a/client/webrtc_capture/src/cv_ssd.cpp
+++ b/client/webrtc_capture/src/cv_ssd.cpp
@@ -18,18 +18,13 @@ cv::Mat * ssd_detect(cv::Mat *inframe) {
         printf("could not load image...\n");
         return inframe;
     }
-    namedWindow("input image", WINDOW_AUTOSIZE);
     cv::Mat  resize;
     cv::resize(*inframe, resize,
         Size(480,560),
         0, 0, INTER_LINEAR);// X Y各缩小一半
-    qDebug()<<"frame.type()"<<resize.type();
-    cv::Mat image2(480,640,CV_8UC3);
-    qDebug()<<"image2 ()"<<image2.type()<<CV_8UC3;
-    cv::cvtColor(resize,image2,COLOR_RGBA2RGB);
-    imshow("input image", image2);
-    qDebug()<<"image2 frame.type()"<<image2.type()<<CV_8UC3;
-    Mat blobImage1 = blobFromImage(image2, 0.007843,
+    cv::Mat *image2 = new cv::Mat(480,640,CV_8UC3);
+    cv::cvtColor(resize,*image2,COLOR_RGBA2RGB);
+    Mat blobImage1 = blobFromImage(*image2, 0.007843,
         Size(300, 300),
         Scalar(127.5, 127.5, 127.5), true, false);
 
@@ -45,52 +40,7 @@ cv::Mat * ssd_detect(cv::Mat *inframe) {
 ////    size：这个参数是我们神经网络在训练的时候要求输入的图片尺寸。
 ////    swapRB：OpenCV中认为我们的图片通道顺序是BGR，但是我平均值假设的顺序是RGB，
 /// 所以如果需要交换R和G，那么就要使swapRB=true
-//    cv::Mat blobImage = blobFromImage(resize,
-//        0.007843,
-//        Size(300, 300),
-//        Scalar(127.5, 127.5, 127.5), true, false);
-//    qDebug()<<"blobImage width : " << blobImage.cols
-//           <<"blobImage.cols: " << blobImage.rows;
-
-//    net.setInput(blobImage, "data");
-//    Mat detection = net.forward("detection_out");
-//    Mat detectionMat(detection.size[2], detection.size[3], CV_32F, detection.ptr<float>());
-//    float confidence_threshold = 0.2;
-//    for (int i = 0; i < detectionMat.rows; i++) {
-//        float confidence = detectionMat.at<float>(i, 2);
-//        if (confidence > confidence_threshold) {
-//            size_t objIndex = (size_t)(detectionMat.at<float>(i, 1));
-//            float tl_x = detectionMat.at<float>(i, 3) * resize.cols;
-//            float tl_y = detectionMat.at<float>(i, 4) * resize.rows;
-//            float br_x = detectionMat.at<float>(i, 5) * resize.cols;
-//            float br_y = detectionMat.at<float>(i, 6) * resize.rows;
-
-//            Rect object_box((int)tl_x, (int)tl_y, (int)(br_x - tl_x), (int)(br_y - tl_y));
-//            rectangle(resize, object_box, Scalar(0, 0, 255), 2, 8, 0);
-//            putText(resize, format("%s", objNames[objIndex].c_str()), Point(tl_x, tl_y),
-//                    FONT_HERSHEY_SIMPLEX, 1.0, Scalar(255, 0, 0), 2);
-//        }
-//    }
-//    imshow("ssd-demo", resize);
-
-//////    waitKey(0);
-//    return inframe;
-    Mat frame = imread("D:/project/opencv_tutorial_data/images/gaoyy.png");
-
-    if (frame.empty()) {
-        printf("could not load image...\n");
-        return nullptr;
-    }
-    qDebug()<<"frame.type()"<<frame.type()<<frame.cols<<frame.rows<<frame.depth()
-           <<frame.elemSize();
-    namedWindow("input image", WINDOW_AUTOSIZE);
-
     Net net = readNetFromCaffe(model_text_file, modelFile);
-    Mat blobImage = blobFromImage(frame, 0.007843,
-        Size(300, 300),
-        Scalar(127.5, 127.5, 127.5), true, false);
-    qDebug()<<"blobImage width "<< blobImage.cols<<blobImage.rows;
-
     net.setInput(blobImage1, "data");
     Mat detection = net.forward("detection_out");
     Mat detectionMat(detection.size[2], detection.size[3], CV_32F, detection.ptr<float>());
@@ -99,19 +49,17 @@ cv::Mat * ssd_detect(cv::Mat *inframe) {
         float confidence = detectionMat.at<float>(i, 2);
         if (confidence > confidence_threshold) {
             size_t objIndex = (size_t)(detectionMat.at<float>(i, 1));
-            float tl_x = detectionMat.at<float>(i, 3) * image2.cols;
-            float tl_y = detectionMat.at<float>(i, 4) * image2.rows;
-            float br_x = detectionMat.at<float>(i, 5) * image2.cols;
-            float br_y = detectionMat.at<float>(i, 6) * image2.rows;
+            float tl_x = detectionMat.at<float>(i, 3) * image2->cols;
+            float tl_y = detectionMat.at<float>(i, 4) * image2->rows;
+            float br_x = detectionMat.at<float>(i, 5) * image2->cols;
+            float br_y = detectionMat.at<float>(i, 6) * image2->rows;
 
             Rect object_box((int)tl_x, (int)tl_y, (int)(br_x - tl_x), (int)(br_y - tl_y));
-            rectangle(image2, object_box, Scalar(0, 0, 255), 2, 8, 0);
-            putText(image2, format("%s", objNames[objIndex].c_str()), Point(tl_x, tl_y), FONT_HERSHEY_SIMPLEX, 1.0, Scalar(255, 0, 0), 2);
+            rectangle(*image2, object_box, Scalar(0, 0, 255), 2, 8, 0);
+            putText(*image2, format("%s", objNames[objIndex].c_str()), Point(tl_x, tl_y), FONT_HERSHEY_SIMPLEX, 1.0, Scalar(255, 0, 0), 2);
         }
     }
-    imshow("ssd-demo", image2);
-    waitKey(0);
-    return nullptr;
+    return image2;
 }
 
 
diff --git a/client/webrtc_capture/src/mainwindow.cpp b/client/webrtc_capture/src/mainwindow.cpp
index 7bab47c..e432fbe 100644
--- a/client/webrtc_capture/src/mainwindow.cpp
+++ b/client/webrtc_capture/src/mainwindow.cpp
@@ -9,6 +9,76 @@
 #include <opencv2/imgproc.hpp>
 #include "cvhelper.h"
 #include "cv_ssd.h"
+#include <list>
+#include <mutex>
+
+
+class  ASyncDetectAndRenderThread :public QSSASyncProcess{
+public:
+    typedef enum {
+        STATUS_STOP = 1,
+        STATUS_RUNNING = 2
+    }Status;
+    ASyncDetectAndRenderThread(QWidget * parent,CPlayWidget *render_ui,
+                               int width,int height){
+        m_render = render_ui;
+        m_parent = parent;
+        m_status = STATUS_RUNNING;
+        m_width = width;
+        m_height = height;
+    }
+    bool DetectStatus(){
+        if(m_mat.size() > 0)
+            return true;
+        else
+            return false;
+    }
+    void SetCvImage(uint8_t *frame){
+        cv::Mat yuv420p;
+        yuv420p.create(m_height*3/2,
+                       m_width, CV_8UC1);
+        memcpy(yuv420p.data, frame, m_height*m_width
+               *sizeof(unsigned char)*3/2);
+        cv::Mat *rgbImg = new cv::Mat;
+        cv::cvtColor(yuv420p, *rgbImg, cv::COLOR_YUV2BGR_I420);
+        std::lock_guard<std::mutex> guard(this->m_mutex);
+//        imshow("yuv420",yuv420p);
+        m_mat.push_back(rgbImg);
+    }
+
+    void Run(void *) override{
+        while(m_status == STATUS_RUNNING){
+            cv::Mat *c = takeLast();
+            if(nullptr != c){
+                cv::Mat *result =  ssd_detect(c);
+                qDebug()<<result->cols<<result->rows<<result->type();
+                cv::Mat yuvData;
+                cv::cvtColor(*result, yuvData, cv::COLOR_BGR2YUV_I420);
+                this->m_render->OnCameraData(yuvData.data);
+            }
+        }
+    }
+
+private:
+    cv::Mat *takeLast(){
+        std::lock_guard<std::mutex> guard(this->m_mutex);
+        if(m_mat.size() != 0){
+            auto ret = *m_mat.begin();
+            m_mat.pop_front();
+            return ret;
+        }
+        return nullptr;
+    }
+    std::list<cv::Mat*> m_mat;
+    std::mutex m_mutex;
+    CPlayWidget *m_render;
+    QWidget *m_parent;
+    Status m_status;
+    int m_width;
+    int m_height;
+};
+
+ASyncDetectAndRenderThread *gdetect = nullptr;
 
 class AsyncRennder :public QSSASyncProcess{
 public:
@@ -32,27 +102,13 @@ public:
 
         while(state){
             if(mfbs ->Size() > 0){
-                uint8_t *frame = this->mfbs->TakeLast();
-                cv::Mat yuv420p;
-                yuv420p.create(mUI->RenderHeight()*3/2,
-                               mUI->RenderWidth(), CV_8UC1);
-                memcpy(yuv420p.data, frame, mUI->RenderHeight()*mUI->RenderWidth()
-                       *sizeof(unsigned char)*3/2);
-                cv::Mat rgbImg;
-                cv::cvtColor(yuv420p, rgbImg, cv::COLOR_YUV2BGR_I420);
-
-//                cv::Mat dst;
-//                cv::rotate(rgbImg, dst, cv::ROTATE_90_CLOCKWISE);  // 绕x翻转，上下颠倒
-//                cv::imshow("img", dst);
-////                cv::Mat gray, dst;
-////                cvtColor(src, gray, COLOR_BGR2GRAY);
-////                imshow("input", gray);
-////                equalizeHist(gray, dst);
-////                imshow("eq", dst);
-//                cv::waitKey(1);
-
+                uint8_t *frame = mfbs->TakeLast();
                 mUI->OnCameraData(frame);
-//                qDebug()<<"dst size is "<<dst.size().width<<dst.size().height;
+                if(gdetect != nullptr){
+                    if(!gdetect->DetectStatus()){
+                        gdetect->SetCvImage(frame);
+                    }
+                }
                 eventLoop.exec(); // 渲染一次
                 delete frame;
             }
@@ -64,7 +120,9 @@ private:
     CPlayWidget *mUI;
 };
 
-AsyncRennder *gRender;
+
+AsyncRennder *gRender = nullptr;
+
 
 MainWindow::MainWindow(QWidget *parent)
     : QssMainWindow(parent)
@@ -92,6 +150,9 @@ MainWindow::MainWindow(QWidget *parent)
     origin_picture->setMaximumHeight(12000);
 
     ui->gridLayout->addWidget(processed_picture,1,0);
+    mDetectResut = new CPlayWidget(this);
+    ui->gridLayout->addWidget(mDetectResut,1,1);
+
 }
 
 MainWindow::~MainWindow()
@@ -106,6 +167,8 @@ void MainWindow::OnUpdateFrame( rtc::scoped_refptr<webrtc::I420BufferInterface>&
 
 void MainWindow::on_pushButton_clicked()
 {
+
+
     int id = ui->comboBox->currentData().toInt();
     webrtc::VideoCaptureCapability p;
 
@@ -131,6 +194,10 @@ void MainWindow::on_pushButton_clicked()
         gRender->Start(this);
         connect(gRender,&QSSASyncProcess::Done,this,&MainWindow::RenderDone);
     }
+    mDetectResut->SetImgSize(480,560);
+    mDetectResut->StartRender();
+
+
 }
 
 void MainWindow::RenderDone()
@@ -154,14 +221,15 @@ cv::Mat *QImage2cvMat(QImage image)
         memcpy(mat->data,image.bits(),image.bytesPerLine()*image.height());
         break;
     case QImage::Format_RGB888:
-        mat = new cv::Mat(image.height(), image.width(), CV_8UC3, (void*)image.bits(), image.bytesPerLine());
+        mat = new cv::Mat(image.height(), image.width(), CV_8UC3,
+                          (void*)image.bits(), image.bytesPerLine());
         break;
     case QImage::Format_Indexed8:
-        mat = new cv::Mat(image.height(), image.width(), CV_8UC1, (void*)image.bits(), image.bytesPerLine());
+        mat = new cv::Mat(image.height(), image.width(), CV_8UC1,
+                          (void*)image.bits(), image.bytesPerLine());
         break;
     }
     Save1BitImage(*mat,"d://tgest.png");
-
     return mat;
 }
 
@@ -181,13 +249,19 @@ void MainWindow::on_pushButton_3_clicked()
         cv::Mat pic =  *QImage2cvMat(image);
 
         cv::Mat *result =  ssd_detect(&pic);
-//        try{
-////            cv::imshow("img", *result);
-//        }catch (std::exception &e){
-//            qDebug()<<e.what();
-//        }
-//        cv::waitKey(1);
         this->processed_picture->setPixmap(QPixmap::fromImage(image));
     }
 }
 
+
+void MainWindow::on_pushButton_4_clicked()
+{
+    if(gdetect == nullptr){
+        gdetect = new ASyncDetectAndRenderThread(this,mDetectResut,ui->openGLWidget->RenderWidth(),
+                                                 ui->openGLWidget->RenderHeight());
+        gdetect->Start(this);
+        connect(gdetect,&QSSASyncProcess::Done,this,&MainWindow::RenderDone);
+    }
+}
+
+
diff --git a/client/webrtc_capture/src/mainwindow.h b/client/webrtc_capture/src/mainwindow.h
index bacffd4..a8350f7 100644
--- a/client/webrtc_capture/src/mainwindow.h
+++ b/client/webrtc_capture/src/mainwindow.h
@@ -4,6 +4,7 @@
 #include "rtc.h"
 #include "api/video/i420_buffer.h"
 #include "Qss.h"
+#include "cplaywidget.h"
 
 QT_BEGIN_NAMESPACE
 namespace Ui { class MainWindow; }
@@ -25,10 +26,13 @@ private slots:
 
     void on_pushButton_3_clicked();
 
+    void on_pushButton_4_clicked();
+
 private:
     Ui::MainWindow *ui;
     std::unique_ptr<CameraVideoSink> m_capturer;
     QLabel *origin_picture;
     QLabel *processed_picture;
+    CPlayWidget *mDetectResut;
 };
 #endif // MAINWINDOW_H