diff --git a/client/webrtc_capture/src/camera_video_sink.cpp b/client/webrtc_capture/src/camera_video_sink.cpp index ae7afac..179f666 100644 --- a/client/webrtc_capture/src/camera_video_sink.cpp +++ b/client/webrtc_capture/src/camera_video_sink.cpp @@ -82,14 +82,8 @@ void CameraVideoSink::OnFrame(const webrtc::VideoFrame& frame) { cnt++; int width = this->m_capability.width; int height = this->m_capability.height; - /* - qDebug()<Capability().height<Capability().width - <Capability().videoType) - <Capability().interlaced<GetI420()->type()); memcpy(data,frameBuffer->GetI420()->DataY(),width*height); memcpy(data + width*height ,frameBuffer->GetI420()->DataU(), diff --git a/client/webrtc_capture/src/camera_video_sink.h b/client/webrtc_capture/src/camera_video_sink.h index 4eb512f..9043a87 100644 --- a/client/webrtc_capture/src/camera_video_sink.h +++ b/client/webrtc_capture/src/camera_video_sink.h @@ -2,8 +2,6 @@ #define CAMERA_VIDEO_SINK_H // vcm_capturer_test.h - - #include #include #include "modules/video_capture/video_capture.h" diff --git a/client/webrtc_capture/src/cplaywidget.cpp b/client/webrtc_capture/src/cplaywidget.cpp index c2aca64..00c47d0 100644 --- a/client/webrtc_capture/src/cplaywidget.cpp +++ b/client/webrtc_capture/src/cplaywidget.cpp @@ -83,6 +83,28 @@ void CPlayWidget::OnPaintData(const rtc::scoped_refptr update(); } + + int CPlayWidget::OnCameraData(uint8_t *p) { memcpy(m_pBufYuv420p,p,m_nVideoH*m_nVideoW/2*3); diff --git a/client/webrtc_capture/src/cplaywidget.h b/client/webrtc_capture/src/cplaywidget.h index a257af5..8bd5a8a 100644 --- a/client/webrtc_capture/src/cplaywidget.h +++ b/client/webrtc_capture/src/cplaywidget.h @@ -36,6 +36,8 @@ public: TYPE_I420, TYPE_UNSET, }IMG_TYPE; + CPlayWidget(QWidget* parent,IMG_TYPE type); + CPlayWidget(QWidget* parent); ~CPlayWidget(); int SetDataType(IMG_TYPE); diff --git a/client/webrtc_capture/src/cv_ssd.cpp b/client/webrtc_capture/src/cv_ssd.cpp index cbed0bf..1a7e8c1 100644 --- a/client/webrtc_capture/src/cv_ssd.cpp +++ b/client/webrtc_capture/src/cv_ssd.cpp @@ -18,18 +18,13 @@ cv::Mat * ssd_detect(cv::Mat *inframe) { printf("could not load image...\n"); return inframe; } - namedWindow("input image", WINDOW_AUTOSIZE); cv::Mat resize; cv::resize(*inframe, resize, Size(480,560), 0, 0, INTER_LINEAR);// X Y各缩小一半 - qDebug()<<"frame.type()"<()); -// float confidence_threshold = 0.2; -// for (int i = 0; i < detectionMat.rows; i++) { -// float confidence = detectionMat.at(i, 2); -// if (confidence > confidence_threshold) { -// size_t objIndex = (size_t)(detectionMat.at(i, 1)); -// float tl_x = detectionMat.at(i, 3) * resize.cols; -// float tl_y = detectionMat.at(i, 4) * resize.rows; -// float br_x = detectionMat.at(i, 5) * resize.cols; -// float br_y = detectionMat.at(i, 6) * resize.rows; - -// Rect object_box((int)tl_x, (int)tl_y, (int)(br_x - tl_x), (int)(br_y - tl_y)); -// rectangle(resize, object_box, Scalar(0, 0, 255), 2, 8, 0); -// putText(resize, format("%s", objNames[objIndex].c_str()), Point(tl_x, tl_y), -// FONT_HERSHEY_SIMPLEX, 1.0, Scalar(255, 0, 0), 2); -// } -// } -// imshow("ssd-demo", resize); - -////// waitKey(0); -// return inframe; - Mat frame = imread("D:/project/opencv_tutorial_data/images/gaoyy.png"); - - if (frame.empty()) { - printf("could not load image...\n"); - return nullptr; - } - qDebug()<<"frame.type()"<()); @@ -99,19 +49,17 @@ cv::Mat * ssd_detect(cv::Mat *inframe) { float confidence = detectionMat.at(i, 2); if (confidence > confidence_threshold) { size_t objIndex = (size_t)(detectionMat.at(i, 1)); - float tl_x = detectionMat.at(i, 3) * image2.cols; - float tl_y = detectionMat.at(i, 4) * image2.rows; - float br_x = detectionMat.at(i, 5) * image2.cols; - float br_y = detectionMat.at(i, 6) * image2.rows; + float tl_x = detectionMat.at(i, 3) * image2->cols; + float tl_y = detectionMat.at(i, 4) * image2->rows; + float br_x = detectionMat.at(i, 5) * image2->cols; + float br_y = detectionMat.at(i, 6) * image2->rows; Rect object_box((int)tl_x, (int)tl_y, (int)(br_x - tl_x), (int)(br_y - tl_y)); - rectangle(image2, object_box, Scalar(0, 0, 255), 2, 8, 0); - putText(image2, format("%s", objNames[objIndex].c_str()), Point(tl_x, tl_y), FONT_HERSHEY_SIMPLEX, 1.0, Scalar(255, 0, 0), 2); + rectangle(*image2, object_box, Scalar(0, 0, 255), 2, 8, 0); + putText(*image2, format("%s", objNames[objIndex].c_str()), Point(tl_x, tl_y), FONT_HERSHEY_SIMPLEX, 1.0, Scalar(255, 0, 0), 2); } } - imshow("ssd-demo", image2); - waitKey(0); - return nullptr; + return image2; } diff --git a/client/webrtc_capture/src/mainwindow.cpp b/client/webrtc_capture/src/mainwindow.cpp index 7bab47c..e432fbe 100644 --- a/client/webrtc_capture/src/mainwindow.cpp +++ b/client/webrtc_capture/src/mainwindow.cpp @@ -9,6 +9,76 @@ #include #include "cvhelper.h" #include "cv_ssd.h" +#include +#include + + +class ASyncDetectAndRenderThread :public QSSASyncProcess{ +public: + typedef enum { + STATUS_STOP = 1, + STATUS_RUNNING = 2 + }Status; + ASyncDetectAndRenderThread(QWidget * parent,CPlayWidget *render_ui, + int width,int height){ + m_render = render_ui; + m_parent = parent; + m_status = STATUS_RUNNING; + m_width = width; + m_height = height; + } + bool DetectStatus(){ + if(m_mat.size() > 0) + return true; + else + return false; + } + void SetCvImage(uint8_t *frame){ + cv::Mat yuv420p; + yuv420p.create(m_height*3/2, + m_width, CV_8UC1); + memcpy(yuv420p.data, frame, m_height*m_width + *sizeof(unsigned char)*3/2); + cv::Mat *rgbImg = new cv::Mat; + cv::cvtColor(yuv420p, *rgbImg, cv::COLOR_YUV2BGR_I420); + std::lock_guard guard(this->m_mutex); +// imshow("yuv420",yuv420p); + m_mat.push_back(rgbImg); + } + + void Run(void *) override{ + while(m_status == STATUS_RUNNING){ + cv::Mat *c = takeLast(); + if(nullptr != c){ + cv::Mat *result = ssd_detect(c); + qDebug()<cols<rows<type(); + cv::Mat yuvData; + cv::cvtColor(*result, yuvData, cv::COLOR_BGR2YUV_I420); + this->m_render->OnCameraData(yuvData.data); + } + } + } + +private: + cv::Mat *takeLast(){ + std::lock_guard guard(this->m_mutex); + if(m_mat.size() != 0){ + auto ret = *m_mat.begin(); + m_mat.pop_front(); + return ret; + } + return nullptr; + } + std::list m_mat; + std::mutex m_mutex; + CPlayWidget *m_render; + QWidget *m_parent; + Status m_status; + int m_width; + int m_height; +}; + +ASyncDetectAndRenderThread *gdetect = nullptr; class AsyncRennder :public QSSASyncProcess{ public: @@ -32,27 +102,13 @@ public: while(state){ if(mfbs ->Size() > 0){ - uint8_t *frame = this->mfbs->TakeLast(); - cv::Mat yuv420p; - yuv420p.create(mUI->RenderHeight()*3/2, - mUI->RenderWidth(), CV_8UC1); - memcpy(yuv420p.data, frame, mUI->RenderHeight()*mUI->RenderWidth() - *sizeof(unsigned char)*3/2); - cv::Mat rgbImg; - cv::cvtColor(yuv420p, rgbImg, cv::COLOR_YUV2BGR_I420); - -// cv::Mat dst; -// cv::rotate(rgbImg, dst, cv::ROTATE_90_CLOCKWISE); // 绕x翻转,上下颠倒 -// cv::imshow("img", dst); -//// cv::Mat gray, dst; -//// cvtColor(src, gray, COLOR_BGR2GRAY); -//// imshow("input", gray); -//// equalizeHist(gray, dst); -//// imshow("eq", dst); -// cv::waitKey(1); - + uint8_t *frame = mfbs->TakeLast(); mUI->OnCameraData(frame); -// qDebug()<<"dst size is "<DetectStatus()){ + gdetect->SetCvImage(frame); + } + } eventLoop.exec(); // 渲染一次 delete frame; } @@ -64,7 +120,9 @@ private: CPlayWidget *mUI; }; -AsyncRennder *gRender; + +AsyncRennder *gRender = nullptr; + MainWindow::MainWindow(QWidget *parent) : QssMainWindow(parent) @@ -92,6 +150,9 @@ MainWindow::MainWindow(QWidget *parent) origin_picture->setMaximumHeight(12000); ui->gridLayout->addWidget(processed_picture,1,0); + mDetectResut = new CPlayWidget(this); + ui->gridLayout->addWidget(mDetectResut,1,1); + } MainWindow::~MainWindow() @@ -106,6 +167,8 @@ void MainWindow::OnUpdateFrame( rtc::scoped_refptr& void MainWindow::on_pushButton_clicked() { + + int id = ui->comboBox->currentData().toInt(); webrtc::VideoCaptureCapability p; @@ -131,6 +194,10 @@ void MainWindow::on_pushButton_clicked() gRender->Start(this); connect(gRender,&QSSASyncProcess::Done,this,&MainWindow::RenderDone); } + mDetectResut->SetImgSize(480,560); + mDetectResut->StartRender(); + + } void MainWindow::RenderDone() @@ -154,14 +221,15 @@ cv::Mat *QImage2cvMat(QImage image) memcpy(mat->data,image.bits(),image.bytesPerLine()*image.height()); break; case QImage::Format_RGB888: - mat = new cv::Mat(image.height(), image.width(), CV_8UC3, (void*)image.bits(), image.bytesPerLine()); + mat = new cv::Mat(image.height(), image.width(), CV_8UC3, + (void*)image.bits(), image.bytesPerLine()); break; case QImage::Format_Indexed8: - mat = new cv::Mat(image.height(), image.width(), CV_8UC1, (void*)image.bits(), image.bytesPerLine()); + mat = new cv::Mat(image.height(), image.width(), CV_8UC1, + (void*)image.bits(), image.bytesPerLine()); break; } Save1BitImage(*mat,"d://tgest.png"); - return mat; } @@ -181,13 +249,19 @@ void MainWindow::on_pushButton_3_clicked() cv::Mat pic = *QImage2cvMat(image); cv::Mat *result = ssd_detect(&pic); -// try{ -//// cv::imshow("img", *result); -// }catch (std::exception &e){ -// qDebug()<processed_picture->setPixmap(QPixmap::fromImage(image)); } } + +void MainWindow::on_pushButton_4_clicked() +{ + if(gdetect == nullptr){ + gdetect = new ASyncDetectAndRenderThread(this,mDetectResut,ui->openGLWidget->RenderWidth(), + ui->openGLWidget->RenderHeight()); + gdetect->Start(this); + connect(gdetect,&QSSASyncProcess::Done,this,&MainWindow::RenderDone); + } +} + + diff --git a/client/webrtc_capture/src/mainwindow.h b/client/webrtc_capture/src/mainwindow.h index bacffd4..a8350f7 100644 --- a/client/webrtc_capture/src/mainwindow.h +++ b/client/webrtc_capture/src/mainwindow.h @@ -4,6 +4,7 @@ #include "rtc.h" #include "api/video/i420_buffer.h" #include "Qss.h" +#include "cplaywidget.h" QT_BEGIN_NAMESPACE namespace Ui { class MainWindow; } @@ -25,10 +26,13 @@ private slots: void on_pushButton_3_clicked(); + void on_pushButton_4_clicked(); + private: Ui::MainWindow *ui; std::unique_ptr m_capturer; QLabel *origin_picture; QLabel *processed_picture; + CPlayWidget *mDetectResut; }; #endif // MAINWINDOW_H