From 74aa46fbd2072c43c8167e2eca5dbb223e46b0d1 Mon Sep 17 00:00:00 2001 From: zcy <29019252@qq.com> Date: Tue, 17 May 2022 23:35:55 +0800 Subject: [PATCH] add opencv demo --- client/webrtc_capture/src/cplaywidget.cpp | 35 ++++----- client/webrtc_capture/src/cplaywidget.h | 2 + client/webrtc_capture/src/mainwindow.cpp | 92 ++++++++++++++++++++--- client/webrtc_capture/src/mainwindow.h | 2 + client/webrtc_capture/src/mainwindow.ui | 2 +- client/webrtc_capture/webrtc_capture.pro | 15 +++- 6 files changed, 117 insertions(+), 31 deletions(-) diff --git a/client/webrtc_capture/src/cplaywidget.cpp b/client/webrtc_capture/src/cplaywidget.cpp index 12323f5..c2aca64 100644 --- a/client/webrtc_capture/src/cplaywidget.cpp +++ b/client/webrtc_capture/src/cplaywidget.cpp @@ -1,5 +1,3 @@ -#include "CPlayWidget.h" - #include #include #include @@ -61,10 +59,6 @@ void main(void) \ gl_FragColor = vec4(rgb, 1); \ }"; -// rgb??????????? -// ???MEDIASUBTYPE_RGB32 ??bgr????????????????????? - - const char *fsrcrgb = "varying vec2 textureOut; \ uniform sampler2D rgbdata; \ void main() \ @@ -82,7 +76,6 @@ void CPlayWidget::OnPaintData(const rtc::scoped_refptrm_nVideoW; +} + +int CPlayWidget::RenderHeight() +{ + return this->m_nVideoH; +} + /* @@ -176,7 +180,7 @@ U = - 0.1687 R - 0.3313 G + 0.5 B + 128 V = 0.5 R - 0.4187 G - 0.0813 B + 128 -????????RGB ?????????YUV (256????) ????: +RGB YUV (256) R = Y + 1.402 (Cr-128) @@ -190,7 +194,6 @@ void CPlayWidget::initializeGL() glEnable(GL_DEPTH_TEST); m_pVSHader = new QOpenGLShader(QOpenGLShader::Vertex, this); - bool bCompile = m_pVSHader->compileSourceCode(vsrcyuv); if(!bCompile) { @@ -372,7 +375,6 @@ int CPlayWidget::loadYuvTexture() glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); glActiveTexture(GL_TEXTURE1); glBindTexture(GL_TEXTURE_2D, id_u); -// glPixelStorei(GL_UNPACK_ROW_LENGTH, m_buffer->StrideU()); glTexImage2D(GL_TEXTURE_2D, 0, GL_RED, m_nVideoW/2, @@ -385,10 +387,9 @@ int CPlayWidget::loadYuvTexture() glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MIN_FILTER,GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - // v·ÖÁ¿ - glActiveTexture(GL_TEXTURE2);//???????????GL_TEXTURE2 + glActiveTexture(GL_TEXTURE2);// GL_TEXTURE2 glBindTexture(GL_TEXTURE_2D, id_v); -// glPixelStorei(GL_UNPACK_ROW_LENGTH, m_buffer->StrideV()); + glTexImage2D(GL_TEXTURE_2D, 0, GL_RED, m_nVideoW/2, @@ -412,7 +413,7 @@ int CPlayWidget::loadRtcI420Texture() if(nullptr == m_buffer) return 0; glActiveTexture(GL_TEXTURE0); - //???????y???????????? + glBindTexture(GL_TEXTURE_2D, id_y); glPixelStorei(GL_UNPACK_ROW_LENGTH, m_buffer->StrideY()); glTexImage2D(GL_TEXTURE_2D, @@ -430,7 +431,7 @@ int CPlayWidget::loadRtcI420Texture() glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MIN_FILTER,GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - //????u???????? + glActiveTexture(GL_TEXTURE1); glBindTexture(GL_TEXTURE_2D, id_u); glPixelStorei(GL_UNPACK_ROW_LENGTH, m_buffer->StrideU()); @@ -446,8 +447,8 @@ int CPlayWidget::loadRtcI420Texture() glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MIN_FILTER,GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - // v·ÖÁ¿ - glActiveTexture(GL_TEXTURE2);//???????????GL_TEXTURE2 + + glActiveTexture(GL_TEXTURE2);//GL_TEXTURE2 glBindTexture(GL_TEXTURE_2D, id_v); glPixelStorei(GL_UNPACK_ROW_LENGTH, m_buffer->StrideV()); glTexImage2D(GL_TEXTURE_2D, diff --git a/client/webrtc_capture/src/cplaywidget.h b/client/webrtc_capture/src/cplaywidget.h index 4d04020..a257af5 100644 --- a/client/webrtc_capture/src/cplaywidget.h +++ b/client/webrtc_capture/src/cplaywidget.h @@ -43,6 +43,8 @@ public: int StopRender(); int OnCameraData(uint8_t *); int SetImgSize(uint32_t width,uint32_t ); + int RenderWidth(); + int RenderHeight(); protected: QTimer tm; void initializeGL() override; diff --git a/client/webrtc_capture/src/mainwindow.cpp b/client/webrtc_capture/src/mainwindow.cpp index 13f4d29..df70662 100644 --- a/client/webrtc_capture/src/mainwindow.cpp +++ b/client/webrtc_capture/src/mainwindow.cpp @@ -1,6 +1,13 @@ #include "mainwindow.h" #include "ui_mainwindow.h" #include "camera_video_sink.h" +#include + +#include "opencv2/opencv.hpp" +#include +#include +#include +#include "cvhelper.h" class AsyncRennder :public QSSASyncProcess{ public: @@ -20,16 +27,34 @@ public: qtimer->setSingleShot(false); QObject::connect(qtimer, SIGNAL(timeout()), &eventLoop, SLOT(quit())); - qtimer->start(10); + qtimer->start(3); while(state){ if(mfbs ->Size() > 0){ uint8_t *frame = this->mfbs->TakeLast(); + cv::Mat yuv420p; + yuv420p.create(mUI->RenderHeight()*3/2, + mUI->RenderWidth(), CV_8UC1); + memcpy(yuv420p.data, frame, mUI->RenderHeight()*mUI->RenderWidth() + *sizeof(unsigned char)*3/2); + cv::Mat rgbImg; + cv::cvtColor(yuv420p, rgbImg, cv::COLOR_YUV2BGR_I420); + +// cv::Mat dst; +// cv::rotate(rgbImg, dst, cv::ROTATE_90_CLOCKWISE); // 绕x翻转，上下颠倒 +// cv::imshow("img", dst); +//// cv::Mat gray, dst; +//// cvtColor(src, gray, COLOR_BGR2GRAY); +//// imshow("input", gray); +//// equalizeHist(gray, dst); +//// imshow("eq", dst); +// cv::waitKey(1); + mUI->OnCameraData(frame); +// qDebug()<<"dst size is "<GetDeviceName(i,name,100,nullptr,0,nullptr,0); ui->comboBox->addItem(QString::asprintf("%s",name),i); } + origin_picture = new QLabel(this); + processed_picture = new QLabel(this); + + ui->gridLayout->addWidget(origin_picture,0,1); + origin_picture->setMaximumWidth(12000); + origin_picture->setMaximumHeight(12000); + + ui->gridLayout->addWidget(processed_picture,1,0); } MainWindow::~MainWindow() @@ -105,15 +138,54 @@ void MainWindow::RenderDone() } -void MainWindow::on_pushButton_2_clicked() + +cv::Mat *QImage2cvMat(QImage image) { - CPlayWidget *ptr = new CPlayWidget(this); - CPlayWidget *ptr1 = new CPlayWidget(this); - CPlayWidget *ptr2 = new CPlayWidget(this); - - ui->gridLayout->addWidget(ptr,0,1); - ui->gridLayout->addWidget(ptr1,1,0); - ui->gridLayout->addWidget(ptr2,1,1); + cv::Mat *mat; + switch(image.format()) + { + case QImage::Format_ARGB32: + case QImage::Format_RGB32: + case QImage::Format_ARGB32_Premultiplied: + mat = new cv::Mat(image.height(), + image.width(), + CV_8UC4, + image.bits(), + image.bytesPerLine()); + memcpy(mat->data,image.bits(),image.bytesPerLine()*image.height()); + break; + case QImage::Format_RGB888: + mat = new cv::Mat(image.height(), image.width(), CV_8UC3, (void*)image.bits(), image.bytesPerLine()); +// cv::cvtColor(mat, mat, CV_BGR2RGB); + break; + case QImage::Format_Indexed8: + mat = new cv::Mat(image.height(), image.width(), CV_8UC1, (void*)image.bits(), image.bytesPerLine()); + break; + } + Save1BitImage(*mat,"d://tgest.png"); + return mat; +} + + +void MainWindow::on_pushButton_2_clicked() +{ + QPixmap pix = ui->openGLWidget->grab(); + QImage image = pix.toImage(); + image = image.scaled(1280,720); + origin_picture->setPixmap(pix); + qDebug()<<"format is "<processed_picture->setPixmap(QPixmap::fromImage(image)); } diff --git a/client/webrtc_capture/src/mainwindow.h b/client/webrtc_capture/src/mainwindow.h index a6e215d..e9aec31 100644 --- a/client/webrtc_capture/src/mainwindow.h +++ b/client/webrtc_capture/src/mainwindow.h @@ -26,5 +26,7 @@ private slots: private: Ui::MainWindow *ui; std::unique_ptr m_capturer; + QLabel *origin_picture; + QLabel *processed_picture; }; #endif // MAINWINDOW_H diff --git a/client/webrtc_capture/src/mainwindow.ui b/client/webrtc_capture/src/mainwindow.ui index 3843a54..713bd32 100644 --- a/client/webrtc_capture/src/mainwindow.ui +++ b/client/webrtc_capture/src/mainwindow.ui @@ -59,7 +59,7 @@ - å¢žåŠ çª—å£ + æ‹ç…§ diff --git a/client/webrtc_capture/webrtc_capture.pro b/client/webrtc_capture/webrtc_capture.pro index ddeffaa..00cd386 100644 --- a/client/webrtc_capture/webrtc_capture.pro +++ b/client/webrtc_capture/webrtc_capture.pro @@ -1,6 +1,6 @@ QT += core gui -include(D:\\project\\c++qt\\qsswraper\\qsswraper.pri) +include(D:\\project\\qt_project\\qsswraper\\qsswraper.pri) RESOURCES += $$PWD/qss/qss.qrc greaterThan(QT_MAJOR_VERSION, 4): QT += widgets @@ -20,11 +20,10 @@ DEFINES += QT_DEPRECATED_WARNINGS NOMINMAX WEBRTC_WIN NOMINMAX WIN32_LEAN_AND_ME INCLUDEPATH += third/include/ -LIBS += -L$$PWD/third/lib libwebrtc.lib ole32.lib oleaut32.lib strmiids.lib advapi32.lib - SOURCES += \ src/camera_video_sink.cpp \ src/cplaywidget.cpp \ + src/cvhelper.cpp \ src/main.cpp \ src/mainwindow.cpp \ src/video_source_impl.cpp @@ -32,6 +31,7 @@ SOURCES += \ HEADERS += \ src/camera_video_sink.h \ src/cplaywidget.h \ + src/cvhelper.h \ src/mainwindow.h \ src/rtc.h \ src/video_source_impl.h @@ -43,3 +43,12 @@ FORMS += \ qnx: target.path = /tmp/$${TARGET}/bin else: unix:!android: target.path = /opt/$${TARGET}/bin !isEmpty(target.path): INSTALLS += target + +CONFIG(debug, debug|release){ + message("debug mode") + LIBS += -L$$PWD/third/lib libwebrtc.lib ole32.lib oleaut32.lib strmiids.lib advapi32.lib opencv_core455d.lib + LIBS +=opencv_stitching455d.lib opencv_objdetect455d.lib opencv_ml455d.lib opencv_imgcodecs455d.lib opencv_imgproc455d.lib + LIBS+= opencv_highgui455d.lib + DEFINES += DEBUG_FLAG + Qt += debug +}