From 74aa46fbd2072c43c8167e2eca5dbb223e46b0d1 Mon Sep 17 00:00:00 2001 From: zcy <29019252@qq.com> Date: Tue, 17 May 2022 23:35:55 +0800 Subject: [PATCH 1/9] add opencv demo --- client/webrtc_capture/src/cplaywidget.cpp | 35 ++++----- client/webrtc_capture/src/cplaywidget.h | 2 + client/webrtc_capture/src/mainwindow.cpp | 92 ++++++++++++++++++++--- client/webrtc_capture/src/mainwindow.h | 2 + client/webrtc_capture/src/mainwindow.ui | 2 +- client/webrtc_capture/webrtc_capture.pro | 15 +++- 6 files changed, 117 insertions(+), 31 deletions(-) diff --git a/client/webrtc_capture/src/cplaywidget.cpp b/client/webrtc_capture/src/cplaywidget.cpp index 12323f5..c2aca64 100644 --- a/client/webrtc_capture/src/cplaywidget.cpp +++ b/client/webrtc_capture/src/cplaywidget.cpp @@ -1,5 +1,3 @@ -#include "CPlayWidget.h" - #include #include #include @@ -61,10 +59,6 @@ void main(void) \ gl_FragColor = vec4(rgb, 1); \ }"; -// rgb??????????? -// ???MEDIASUBTYPE_RGB32 ??bgr????????????????????? - - const char *fsrcrgb = "varying vec2 textureOut; \ uniform sampler2D rgbdata; \ void main() \ @@ -82,7 +76,6 @@ void CPlayWidget::OnPaintData(const rtc::scoped_refptrm_nVideoW; +} + +int CPlayWidget::RenderHeight() +{ + return this->m_nVideoH; +} + /* @@ -176,7 +180,7 @@ U = - 0.1687 R - 0.3313 G + 0.5 B + 128 V = 0.5 R - 0.4187 G - 0.0813 B + 128 -????????RGB ?????????YUV (256????) ????: +RGB YUV (256) R = Y + 1.402 (Cr-128) @@ -190,7 +194,6 @@ void CPlayWidget::initializeGL() glEnable(GL_DEPTH_TEST); m_pVSHader = new QOpenGLShader(QOpenGLShader::Vertex, this); - bool bCompile = m_pVSHader->compileSourceCode(vsrcyuv); if(!bCompile) { @@ -372,7 +375,6 @@ int CPlayWidget::loadYuvTexture() glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); glActiveTexture(GL_TEXTURE1); glBindTexture(GL_TEXTURE_2D, id_u); -// glPixelStorei(GL_UNPACK_ROW_LENGTH, m_buffer->StrideU()); glTexImage2D(GL_TEXTURE_2D, 0, GL_RED, m_nVideoW/2, @@ -385,10 +387,9 @@ int CPlayWidget::loadYuvTexture() glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MIN_FILTER,GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - // v - glActiveTexture(GL_TEXTURE2);//???????????GL_TEXTURE2 + glActiveTexture(GL_TEXTURE2);// GL_TEXTURE2 glBindTexture(GL_TEXTURE_2D, id_v); -// glPixelStorei(GL_UNPACK_ROW_LENGTH, m_buffer->StrideV()); + glTexImage2D(GL_TEXTURE_2D, 0, GL_RED, m_nVideoW/2, @@ -412,7 +413,7 @@ int CPlayWidget::loadRtcI420Texture() if(nullptr == m_buffer) return 0; glActiveTexture(GL_TEXTURE0); - //???????y???????????? + glBindTexture(GL_TEXTURE_2D, id_y); glPixelStorei(GL_UNPACK_ROW_LENGTH, m_buffer->StrideY()); glTexImage2D(GL_TEXTURE_2D, @@ -430,7 +431,7 @@ int CPlayWidget::loadRtcI420Texture() glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MIN_FILTER,GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - //????u???????? + glActiveTexture(GL_TEXTURE1); glBindTexture(GL_TEXTURE_2D, id_u); glPixelStorei(GL_UNPACK_ROW_LENGTH, m_buffer->StrideU()); @@ -446,8 +447,8 @@ int CPlayWidget::loadRtcI420Texture() glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MIN_FILTER,GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - // v - glActiveTexture(GL_TEXTURE2);//???????????GL_TEXTURE2 + + glActiveTexture(GL_TEXTURE2);//GL_TEXTURE2 glBindTexture(GL_TEXTURE_2D, id_v); glPixelStorei(GL_UNPACK_ROW_LENGTH, m_buffer->StrideV()); glTexImage2D(GL_TEXTURE_2D, diff --git a/client/webrtc_capture/src/cplaywidget.h b/client/webrtc_capture/src/cplaywidget.h index 4d04020..a257af5 100644 --- a/client/webrtc_capture/src/cplaywidget.h +++ b/client/webrtc_capture/src/cplaywidget.h @@ -43,6 +43,8 @@ public: int StopRender(); int OnCameraData(uint8_t *); int SetImgSize(uint32_t width,uint32_t ); + int RenderWidth(); + int RenderHeight(); protected: QTimer tm; void initializeGL() override; diff --git a/client/webrtc_capture/src/mainwindow.cpp b/client/webrtc_capture/src/mainwindow.cpp index 13f4d29..df70662 100644 --- a/client/webrtc_capture/src/mainwindow.cpp +++ b/client/webrtc_capture/src/mainwindow.cpp @@ -1,6 +1,13 @@ #include "mainwindow.h" #include "ui_mainwindow.h" #include "camera_video_sink.h" +#include + +#include "opencv2/opencv.hpp" +#include +#include +#include +#include "cvhelper.h" class AsyncRennder :public QSSASyncProcess{ public: @@ -20,16 +27,34 @@ public: qtimer->setSingleShot(false); QObject::connect(qtimer, SIGNAL(timeout()), &eventLoop, SLOT(quit())); - qtimer->start(10); + qtimer->start(3); while(state){ if(mfbs ->Size() > 0){ uint8_t *frame = this->mfbs->TakeLast(); + cv::Mat yuv420p; + yuv420p.create(mUI->RenderHeight()*3/2, + mUI->RenderWidth(), CV_8UC1); + memcpy(yuv420p.data, frame, mUI->RenderHeight()*mUI->RenderWidth() + *sizeof(unsigned char)*3/2); + cv::Mat rgbImg; + cv::cvtColor(yuv420p, rgbImg, cv::COLOR_YUV2BGR_I420); + +// cv::Mat dst; +// cv::rotate(rgbImg, dst, cv::ROTATE_90_CLOCKWISE); // 绕x翻转，上下颠倒 +// cv::imshow("img", dst); +//// cv::Mat gray, dst; +//// cvtColor(src, gray, COLOR_BGR2GRAY); +//// imshow("input", gray); +//// equalizeHist(gray, dst); +//// imshow("eq", dst); +// cv::waitKey(1); + mUI->OnCameraData(frame); +// qDebug()<<"dst size is "<GetDeviceName(i,name,100,nullptr,0,nullptr,0); ui->comboBox->addItem(QString::asprintf("%s",name),i); } + origin_picture = new QLabel(this); + processed_picture = new QLabel(this); + + ui->gridLayout->addWidget(origin_picture,0,1); + origin_picture->setMaximumWidth(12000); + origin_picture->setMaximumHeight(12000); + + ui->gridLayout->addWidget(processed_picture,1,0); } MainWindow::~MainWindow() @@ -105,15 +138,54 @@ void MainWindow::RenderDone() } -void MainWindow::on_pushButton_2_clicked() + +cv::Mat *QImage2cvMat(QImage image) { - CPlayWidget *ptr = new CPlayWidget(this); - CPlayWidget *ptr1 = new CPlayWidget(this); - CPlayWidget *ptr2 = new CPlayWidget(this); - - ui->gridLayout->addWidget(ptr,0,1); - ui->gridLayout->addWidget(ptr1,1,0); - ui->gridLayout->addWidget(ptr2,1,1); + cv::Mat *mat; + switch(image.format()) + { + case QImage::Format_ARGB32: + case QImage::Format_RGB32: + case QImage::Format_ARGB32_Premultiplied: + mat = new cv::Mat(image.height(), + image.width(), + CV_8UC4, + image.bits(), + image.bytesPerLine()); + memcpy(mat->data,image.bits(),image.bytesPerLine()*image.height()); + break; + case QImage::Format_RGB888: + mat = new cv::Mat(image.height(), image.width(), CV_8UC3, (void*)image.bits(), image.bytesPerLine()); +// cv::cvtColor(mat, mat, CV_BGR2RGB); + break; + case QImage::Format_Indexed8: + mat = new cv::Mat(image.height(), image.width(), CV_8UC1, (void*)image.bits(), image.bytesPerLine()); + break; + } + Save1BitImage(*mat,"d://tgest.png"); + return mat; +} + + +void MainWindow::on_pushButton_2_clicked() +{ + QPixmap pix = ui->openGLWidget->grab(); + QImage image = pix.toImage(); + image = image.scaled(1280,720); + origin_picture->setPixmap(pix); + qDebug()<<"format is "<processed_picture->setPixmap(QPixmap::fromImage(image)); } diff --git a/client/webrtc_capture/src/mainwindow.h b/client/webrtc_capture/src/mainwindow.h index a6e215d..e9aec31 100644 --- a/client/webrtc_capture/src/mainwindow.h +++ b/client/webrtc_capture/src/mainwindow.h @@ -26,5 +26,7 @@ private slots: private: Ui::MainWindow *ui; std::unique_ptr m_capturer; + QLabel *origin_picture; + QLabel *processed_picture; }; #endif // MAINWINDOW_H diff --git a/client/webrtc_capture/src/mainwindow.ui b/client/webrtc_capture/src/mainwindow.ui index 3843a54..713bd32 100644 --- a/client/webrtc_capture/src/mainwindow.ui +++ b/client/webrtc_capture/src/mainwindow.ui @@ -59,7 +59,7 @@ - 增加窗口 + 拍照 diff --git a/client/webrtc_capture/webrtc_capture.pro b/client/webrtc_capture/webrtc_capture.pro index ddeffaa..00cd386 100644 --- a/client/webrtc_capture/webrtc_capture.pro +++ b/client/webrtc_capture/webrtc_capture.pro @@ -1,6 +1,6 @@ QT += core gui -include(D:\\project\\c++qt\\qsswraper\\qsswraper.pri) +include(D:\\project\\qt_project\\qsswraper\\qsswraper.pri) RESOURCES += $$PWD/qss/qss.qrc greaterThan(QT_MAJOR_VERSION, 4): QT += widgets @@ -20,11 +20,10 @@ DEFINES += QT_DEPRECATED_WARNINGS NOMINMAX WEBRTC_WIN NOMINMAX WIN32_LEAN_AND_ME INCLUDEPATH += third/include/ -LIBS += -L$$PWD/third/lib libwebrtc.lib ole32.lib oleaut32.lib strmiids.lib advapi32.lib - SOURCES += \ src/camera_video_sink.cpp \ src/cplaywidget.cpp \ + src/cvhelper.cpp \ src/main.cpp \ src/mainwindow.cpp \ src/video_source_impl.cpp @@ -32,6 +31,7 @@ SOURCES += \ HEADERS += \ src/camera_video_sink.h \ src/cplaywidget.h \ + src/cvhelper.h \ src/mainwindow.h \ src/rtc.h \ src/video_source_impl.h @@ -43,3 +43,12 @@ FORMS += \ qnx: target.path = /tmp/$${TARGET}/bin else: unix:!android: target.path = /opt/$${TARGET}/bin !isEmpty(target.path): INSTALLS += target + +CONFIG(debug, debug|release){ + message("debug mode") + LIBS += -L$$PWD/third/lib libwebrtc.lib ole32.lib oleaut32.lib strmiids.lib advapi32.lib opencv_core455d.lib + LIBS +=opencv_stitching455d.lib opencv_objdetect455d.lib opencv_ml455d.lib opencv_imgcodecs455d.lib opencv_imgproc455d.lib + LIBS+= opencv_highgui455d.lib + DEFINES += DEBUG_FLAG + Qt += debug +} From cd72125f29f43f71825d31382a3a0a97025394d8 Mon Sep 17 00:00:00 2001 From: zcy <29019252@qq.com> Date: Wed, 18 May 2022 02:19:46 +0800 Subject: [PATCH 2/9] no message --- .gitignore | 1 + client/webrtc_capture/src/mainwindow.cpp | 18 +++++++----------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index ef29265..74a106c 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,4 @@ client/janus_gateway_win/x64/ client/janus_gateway_win/janus_win/x64/ client/janus_gateway_win/.vs/ client/janus_gateway_win/usocket_test/x64/ +client/build-webrtc_capture-Desktop_Qt_5_15_2_MSVC2019_64bit-Debug/ diff --git a/client/webrtc_capture/src/mainwindow.cpp b/client/webrtc_capture/src/mainwindow.cpp index df70662..b65773c 100644 --- a/client/webrtc_capture/src/mainwindow.cpp +++ b/client/webrtc_capture/src/mainwindow.cpp @@ -149,14 +149,11 @@ cv::Mat *QImage2cvMat(QImage image) case QImage::Format_ARGB32_Premultiplied: mat = new cv::Mat(image.height(), image.width(), - CV_8UC4, - image.bits(), - image.bytesPerLine()); + CV_8UC4); memcpy(mat->data,image.bits(),image.bytesPerLine()*image.height()); break; case QImage::Format_RGB888: mat = new cv::Mat(image.height(), image.width(), CV_8UC3, (void*)image.bits(), image.bytesPerLine()); -// cv::cvtColor(mat, mat, CV_BGR2RGB); break; case QImage::Format_Indexed8: mat = new cv::Mat(image.height(), image.width(), CV_8UC1, (void*)image.bits(), image.bytesPerLine()); @@ -177,14 +174,13 @@ void MainWindow::on_pushButton_2_clicked() qDebug()<<"format is "<processed_picture->setPixmap(QPixmap::fromImage(image)); } From 7f6d765d28461fb5a04da7bf8a1066a3eb4b70a0 Mon Sep 17 00:00:00 2001 From: zcy <290198252@qq.com> Date: Wed, 18 May 2022 02:28:56 +0800 Subject: [PATCH 3/9] no message --- client/webrtc_capture/src/cvhelper.cpp | 162 +++++++++++++++++++++++++ client/webrtc_capture/src/cvhelper.h | 17 +++ 2 files changed, 179 insertions(+) create mode 100644 client/webrtc_capture/src/cvhelper.cpp create mode 100644 client/webrtc_capture/src/cvhelper.h diff --git a/client/webrtc_capture/src/cvhelper.cpp b/client/webrtc_capture/src/cvhelper.cpp new file mode 100644 index 0000000..2eec1db --- /dev/null +++ b/client/webrtc_capture/src/cvhelper.cpp @@ -0,0 +1,162 @@ +#include +#include "cvhelper.h" + +#include +#include + +/* +* 将一幅RGB图像转换YCrCb图像，并得到三个分离的通道。 +*/ +void TransferRGB2YCrCb(cv::Mat &input, std::vector &chls) +{ + //YCrCb色彩空间：Y为颜色的亮度（luma）成分、而CB和CR则为蓝色和红色的浓度偏移量成分。 + //只有Y成分的图基本等同于彩色图像的灰度图。 + cv::cvtColor(input, input, cv::COLOR_BGR2YCrCb); + cv::split(input, chls); +} + +/* +* 将Y、Cr、Cb三个分离的通道合并为YCrCb图像，再转换为RGB图像。 +*/ +void MergeYCrCb2RGB(cv::Mat &output, std::vector &chls) +{ + cv::merge(chls, output); + cv::cvtColor(output, output, cv::COLOR_YCrCb2BGR); +} + + +/* +*CLAHE. +*@ClipLimit:直方图的柱子高度大于计算后的ClipLimit的部分被裁剪掉，然后将其平均分配给整张直方图,从而提升*整个图像。 +*但是本人觉得这里解释的不对，更像是累计概率分布函数的斜率上限阈值，高于ClipLimit的部分被裁剪掉， +*然后将其平均分配给整张直方图,从而提升整个图像。 +*@TilesGridSize:// 将图像分为TilesGridSize*TilesGridSize大小的方块.。（滑动窗口大小为TilesGridSize*TilesGridSize） +*/ +cv::Mat clahe_deal(cv::Mat &src, double ClipLimit = 40.0, int TilesGridSize = 8) +{ + cv::Mat ycrcb = src.clone(); + std::vector channels; + +// TransferRGB2YCrCb(ycrcb, channels); + +// cv::Mat clahe_img; +// //opencv源码中原型： createCLAHE(double clipLimit = 40.0, Size tileGridSize = Size(8, 8)); +// cv::Ptr clahe = cv::createCLAHE(); + +// clahe->setClipLimit(ClipLimit); +// clahe->setTilesGridSize(Size(TilesGridSize, TilesGridSize)); +// clahe->apply(channels[0], clahe_img); +// channels[0].release(); +// clahe_img.copyTo(channels[0]); +// MergeYCrCb2RGB(ycrcb, channels); +// return ycrcb; + return ycrcb; +} + + +//将mat数组转换为二进制，255用1表示，0用0表示 +int Mat2Binary(const cv::Mat img, int line_byte, char* data) +{ + int width = img.cols; + int height = img.rows; + size_t line_size = line_byte * 8; + size_t bit_size = line_size * height; + + + char* p = data; int offset, v; unsigned char temp; + for (int row = height - 1; row >= 0; row--) + { + for (int col = 0; col < width; col++) + { + offset = col % 8; + //v = img.data[row * width + col]; + v = img.at(row, col); + temp = 1; + temp = temp << (8 - offset - 1); + if (v == 255) + { + *(p + col / 8) |= temp; + } + else + { + temp = ~temp; + *(p + col / 8) &= temp; + } + } + //注释掉一下代码，列数否则不能被4整除的图像最后4列会出现黑边 +/* for (int j = width / 8; j < line_byte + 1; j++) { + p[j] = 0; + } */ + p = p + line_byte; + } + return 0; +} + +//将Mat图像保存为1位bmp图 +int Save1BitImage(const cv::Mat img, std::string dst) +{ + int width = img.cols; + int height = img.rows; + const int biBitCount = 1; + + //待存储图像数据每行字节数为4的倍数，计算位图每行占多少个字节 + //int line_byte = (width * biBitCount >> 3 + 3) / 4 * 4; + int line_byte = (width * biBitCount / 8 + 5) / 4 * 4;//+3改为+5,否则无法向上对齐，造成崩溃 + char* p_data = (char*)malloc(line_byte * height + 1);//后面加1否则会报heap错误 + //memset(p_data, 0x01, line_byte * height+1); + + //将mat数组转换为二进制，255用1表示，0用0表示 + Mat2Binary(img, line_byte, p_data); + + //bmp位图颜色表大小，以字节为单位，灰度图像颜色表为256*4字节，彩色图像颜色表大小为0,二值图为2*4 + int color_type_num = 2; + int colorTablesize = color_type_num * sizeof(RGBQUAD); + RGBQUAD* pColorTable = new RGBQUAD[color_type_num]; + for (int i = 0; i < color_type_num; i++) { + pColorTable[i].rgbBlue = i * 255; + pColorTable[i].rgbRed = i * 255; + pColorTable[i].rgbGreen = i * 255; + pColorTable[i].rgbReserved = 0; + } + //申请位图文件头结构变量，填写文件头信息 + BITMAPFILEHEADER fileHead; + fileHead.bfType = 0x4D42; //bmp类型 + fileHead.bfSize = sizeof(BITMAPFILEHEADER) + sizeof(BITMAPINFOHEADER) + colorTablesize + line_byte * height; //bfSize是图像文件4个组成部分之和,图文件的大小，以字节为单位 + fileHead.bfReserved1 = 0;//位图文件保留字，必须为0 + fileHead.bfReserved2 = 0; + fileHead.bfOffBits = 54 + colorTablesize; //bfOffBits是图像文件前3个部分所需空间之和,位图数据的起始位置 + + //申请位图信息头结构变量，填写信息头信息 + BITMAPINFOHEADER head; + head.biBitCount = biBitCount;//每个像素所需的位数，必须是1（双色），（29-30字节）4(16色），8(256色）16(高彩色)或24（真彩色）之一 + head.biCompression = BI_RGB; //位图压缩类型,必须为BI_RGB否则图片打开会有错误 + head.biHeight = height;//位图的高度，以像素为单位 + head.biWidth = width; + //head.biSize = 40; + head.biSize = sizeof(BITMAPINFOHEADER);//本结构所占用字节数 + head.biSizeImage = line_byte * height;//位图的大小(其中包含了为了补齐行数是4的倍数而添加的空字节) + head.biPlanes = 1;//目标设备的级别，必须为1 + head.biXPelsPerMeter = 23622;//位图水平分辨率，每米像素数，分辨率设为600 + head.biYPelsPerMeter = 23622;//位图垂直分辨率，每米像素数 + head.biClrImportant = 0;//位图显示过程中重要的颜色数 + head.biClrUsed = 0;//位图实际使用的颜色表中的颜色数 + + std::ofstream fp(dst.c_str(), std::ios::binary | std::ios::out); + if (!fp.is_open()) { + return -1; + } + fp.write((char*)&fileHead, sizeof(BITMAPFILEHEADER)); //写文件头进文件 + //写位图信息头进内存 + fp.write((char*)&head, sizeof(BITMAPINFOHEADER)); + //颜色表，写入文件 + fp.write((char*)pColorTable, sizeof(RGBQUAD) * color_type_num); + //写位图数据进文件pBmpBuf + fp.write((char*)p_data, height * line_byte); + fp.close(); + + delete[]pColorTable; + delete[]p_data; + return 0; +} + + diff --git a/client/webrtc_capture/src/cvhelper.h b/client/webrtc_capture/src/cvhelper.h new file mode 100644 index 0000000..2f9d870 --- /dev/null +++ b/client/webrtc_capture/src/cvhelper.h @@ -0,0 +1,17 @@ +#ifndef CVHELPER_H +#define CVHELPER_H + +// 常见opencv 图像操作的一层wrap + +#include "opencv2/opencv.hpp" +#include +#include +#include + +void TransferRGB2YCrCb(cv::Mat &input, std::vector &chls); +void MergeYCrCb2RGB(cv::Mat &output, std::vector &chls); +cv::Mat clahe_deal(cv::Mat &src, double ClipLimit , int TilesGridSize ); +int Save1BitImage(const cv::Mat img, std::string dst); +int Mat2Binary(const cv::Mat img, int line_byte, char* data); + +#endif // CVHELPER_H From f748b3ded8a3904c8dba2216e9b2bb536474fef5 Mon Sep 17 00:00:00 2001 From: zcy <290198252@qq.com> Date: Thu, 19 May 2022 18:01:35 +0800 Subject: [PATCH 4/9] no message --- client/webrtc_capture/src/cv_ssd.cpp | 64 +++++++++ client/webrtc_capture/src/cv_ssd.h | 17 +++ client/webrtc_capture/src/cv_yolo.cpp | 168 +++++++++++++++++++++++ client/webrtc_capture/src/cv_yolo.h | 19 +++ client/webrtc_capture/src/main.cpp | 28 +++- client/webrtc_capture/src/mainwindow.cpp | 34 +++-- client/webrtc_capture/src/mainwindow.h | 2 + client/webrtc_capture/src/mainwindow.ui | 9 +- client/webrtc_capture/webrtc_capture.pro | 10 +- 9 files changed, 332 insertions(+), 19 deletions(-) create mode 100644 client/webrtc_capture/src/cv_ssd.cpp create mode 100644 client/webrtc_capture/src/cv_ssd.h create mode 100644 client/webrtc_capture/src/cv_yolo.cpp create mode 100644 client/webrtc_capture/src/cv_yolo.h diff --git a/client/webrtc_capture/src/cv_ssd.cpp b/client/webrtc_capture/src/cv_ssd.cpp new file mode 100644 index 0000000..2f04590 --- /dev/null +++ b/client/webrtc_capture/src/cv_ssd.cpp @@ -0,0 +1,64 @@ +#include "cv_ssd.h" +#include +const size_t width = 1280; +const size_t height = 720; +String labelFile = "ssd/labelmap_det.txt"; +String modelFile = "D:/project/multimedia/client/build-webrtc_capture-Desktop_Qt_5_15_2_MSVC2019_64bit-Debug/debug/ssd/MobileNetSSD_deploy.caffemodel"; +String model_text_file = "D:/project/multimedia/client/build-webrtc_capture-Desktop_Qt_5_15_2_MSVC2019_64bit-Debug/debug/ssd/MobileNetSSD_deploy.prototxt"; + +String objNames[] = { "background", +"aeroplane", "bicycle", "bird", "boat", +"bottle", "bus", "car", "cat", "chair", +"cow", "diningtable", "dog", "horse", +"motorbike", "person", "pottedplant", +"sheep", "sofa", "train", "tvmonitor" }; + + +cv::Mat * ssd_detect(cv::Mat *inframe) { + if (inframe->empty()) { + printf("could not load image...\n"); + return inframe; + } + namedWindow("input image", WINDOW_AUTOSIZE); + imshow("input image", *inframe); + + Net net = readNetFromCaffe(model_text_file, modelFile); + +// image：这个就是我们将要输入神经网络进行处理或者分类的图片。 +// mean：需要将图片整体减去的平均值，如果我们需要对RGB图片的三个通道分别减去不同的值，那么可以使用3组平均值，如果只使用一组，那么就默认对三个通道减去一样的值。减去平均值（mean）：为了消除同一场景下不同光照的图片，对我们最终的分类或者神经网络的影响，我们常常对图片的R、G、B通道的像素求一个平均值，然后将每个像素值减去我们的平均值，这样就可以得到像素之间的相对值，就可以排除光照的影响。 +// scalefactor：当我们将图片减去平均值之后，还可以对剩下的像素值进行一定的尺度缩放，它的默认值是1，如果希望减去平均像素之后的值，全部缩小一半，那么可以将scalefactor设为1/2。 +// size：这个参数是我们神经网络在训练的时候要求输入的图片尺寸。 +// swapRB：OpenCV中认为我们的图片通道顺序是BGR，但是我平均值假设的顺序是RGB，所以如果需要交换R和G，那么就要使swapRB=true + Mat blobImage = blobFromImage(*inframe, 0.007843, + Size(300, 300), + Scalar(127.5, 127.5, 127.5), true, false); + qDebug()<<"blobImage width : " << blobImage.cols + <<"blobImage.cols: " << blobImage.rows; + + imshow("ssd-demo", blobImage); + +// net.setInput(blobImage, "data"); +// Mat detection = net.forward("detection_out"); +// Mat detectionMat(detection.size[2], detection.size[3], CV_32F, detection.ptr()); +// float confidence_threshold = 0.2; +// for (int i = 0; i < detectionMat.rows; i++) { +// float confidence = detectionMat.at(i, 2); +// if (confidence > confidence_threshold) { +// size_t objIndex = (size_t)(detectionMat.at(i, 1)); +// float tl_x = detectionMat.at(i, 3) * frame->cols; +// float tl_y = detectionMat.at(i, 4) * frame->rows; +// float br_x = detectionMat.at(i, 5) * frame->cols; +// float br_y = detectionMat.at(i, 6) * frame->rows; + +// Rect object_box((int)tl_x, (int)tl_y, (int)(br_x - tl_x), (int)(br_y - tl_y)); +// rectangle(*frame, object_box, Scalar(0, 0, 255), 2, 8, 0); +// putText(*frame, format("%s", objNames[objIndex].c_str()), Point(tl_x, tl_y), +// FONT_HERSHEY_SIMPLEX, 1.0, Scalar(255, 0, 0), 2); +// } +// } + +// waitKey(0); + return inframe; +} + + diff --git a/client/webrtc_capture/src/cv_ssd.h b/client/webrtc_capture/src/cv_ssd.h new file mode 100644 index 0000000..66f20db --- /dev/null +++ b/client/webrtc_capture/src/cv_ssd.h @@ -0,0 +1,17 @@ +#ifndef CV_SSD_H +#define CV_SSD_H + +#include +#include +#include + +using namespace cv; +using namespace cv::dnn; +using namespace std; + +cv::Mat * ssd_detect(cv::Mat*); + + + + +#endif // CV_SSD_H diff --git a/client/webrtc_capture/src/cv_yolo.cpp b/client/webrtc_capture/src/cv_yolo.cpp new file mode 100644 index 0000000..c20e7ca --- /dev/null +++ b/client/webrtc_capture/src/cv_yolo.cpp @@ -0,0 +1,168 @@ +#include "cv_yolo.h" + +float confidenceThreshold = 0.25; + +void test_yolo() +{ + image_detection(); +} + +void video_detection() { + String modelConfiguration = "D:/vcprojects/images/dnn/yolov2-tiny-voc/yolov2-tiny-voc.cfg"; + String modelBinary = "D:/vcprojects/images/dnn/yolov2-tiny-voc/yolov2-tiny-voc.weights"; + dnn::Net net = readNetFromDarknet(modelConfiguration, modelBinary); + if (net.empty()) + { + printf("Could not load net...\n"); + return; + } + + vector classNamesVec; + ifstream classNamesFile("D:/vcprojects/images/dnn/yolov2-tiny-voc/voc.names"); + if (classNamesFile.is_open()) + { + string className = ""; + while (std::getline(classNamesFile, className)) + classNamesVec.push_back(className); + } + + // VideoCapture capture(0); + VideoCapture capture; + capture.open("D:/vcprojects/images/fbb.avi"); + if (!capture.isOpened()) { + printf("could not open the camera...\n"); + return; + } + + Mat frame; + while (capture.read(frame)) + { + if (frame.empty()) + if (frame.channels() == 4) + cvtColor(frame, frame, COLOR_BGRA2BGR); + Mat inputBlob = blobFromImage(frame, 1 / 255.F, Size(416, 416), Scalar(), true, false); + net.setInput(inputBlob, "data"); + Mat detectionMat = net.forward("detection_out"); + vector layersTimings; + double freq = getTickFrequency() / 1000; + double time = net.getPerfProfile(layersTimings) / freq; + ostringstream ss; + ss << "FPS: " << 1000 / time << " ; time: " << time << " ms"; + putText(frame, ss.str(), Point(20, 20), 0, 0.5, Scalar(0, 0, 255)); + + for (int i = 0; i < detectionMat.rows; i++) + { + const int probability_index = 5; + const int probability_size = detectionMat.cols - probability_index; + float *prob_array_ptr = &detectionMat.at(i, probability_index); + size_t objectClass = max_element(prob_array_ptr, prob_array_ptr + probability_size) - prob_array_ptr; + float confidence = detectionMat.at(i, (int)objectClass + probability_index); + if (confidence > confidenceThreshold) + { + float x = detectionMat.at(i, 0); + float y = detectionMat.at(i, 1); + float width = detectionMat.at(i, 2); + float height = detectionMat.at(i, 3); + int xLeftBottom = static_cast((x - width / 2) * frame.cols); + int yLeftBottom = static_cast((y - height / 2) * frame.rows); + int xRightTop = static_cast((x + width / 2) * frame.cols); + int yRightTop = static_cast((y + height / 2) * frame.rows); + Rect object(xLeftBottom, yLeftBottom, + xRightTop - xLeftBottom, + yRightTop - yLeftBottom); + rectangle(frame, object, Scalar(0, 255, 0)); + if (objectClass < classNamesVec.size()) + { + ss.str(""); + ss << confidence; + String conf(ss.str()); + String label = String(classNamesVec[objectClass]) + ": " + conf; + int baseLine = 0; + Size labelSize = getTextSize(label, FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine); + rectangle(frame, Rect(Point(xLeftBottom, yLeftBottom), + Size(labelSize.width, labelSize.height + baseLine)), + Scalar(255, 255, 255), FILLED); + putText(frame, label, Point(xLeftBottom, yLeftBottom + labelSize.height), + FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 0, 0)); + } + } + } + imshow("YOLOv3: Detections", frame); + if (waitKey(1) >= 0) break; + } +} + +void image_detection() { + String modelConfiguration = "D:/vcprojects/images/dnn/yolov2-tiny-voc/yolov2-tiny-voc.cfg"; + String modelBinary = "D:/vcprojects/images/dnn/yolov2-tiny-voc/yolov2-tiny-voc.weights"; + dnn::Net net = readNetFromDarknet(modelConfiguration, modelBinary); + if (net.empty()) + { + printf("Could not load net...\n"); + return; + } + vector classNamesVec; + ifstream classNamesFile("D:/vcprojects/images/dnn/yolov2-tiny-voc/voc.names"); + if (classNamesFile.is_open()) + { + string className = ""; + while (std::getline(classNamesFile, className)) + classNamesVec.push_back(className); + } + + // ͼ + Mat frame = imread("D:/vcprojects/images/fastrcnn.jpg"); + Mat inputBlob = blobFromImage(frame, 1 / 255.F, Size(416, 416), Scalar(), true, false); + net.setInput(inputBlob, "data"); + + // + Mat detectionMat = net.forward("detection_out"); + vector layersTimings; + double freq = getTickFrequency() / 1000; + double time = net.getPerfProfile(layersTimings) / freq; + ostringstream ss; + ss << "detection time: " << time << " ms"; + putText(frame, ss.str(), Point(20, 20), 0, 0.5, Scalar(0, 0, 255)); + + // + for (int i = 0; i < detectionMat.rows; i++) + { + const int probability_index = 5; + const int probability_size = detectionMat.cols - probability_index; + float *prob_array_ptr = &detectionMat.at(i, probability_index); + size_t objectClass = max_element(prob_array_ptr, prob_array_ptr + probability_size) - prob_array_ptr; + float confidence = detectionMat.at(i, (int)objectClass + probability_index); + if (confidence > confidenceThreshold) + { + float x = detectionMat.at(i, 0); + float y = detectionMat.at(i, 1); + float width = detectionMat.at(i, 2); + float height = detectionMat.at(i, 3); + int xLeftBottom = static_cast((x - width / 2) * frame.cols); + int yLeftBottom = static_cast((y - height / 2) * frame.rows); + int xRightTop = static_cast((x + width / 2) * frame.cols); + int yRightTop = static_cast((y + height / 2) * frame.rows); + Rect object(xLeftBottom, yLeftBottom, + xRightTop - xLeftBottom, + yRightTop - yLeftBottom); + rectangle(frame, object, Scalar(0, 0, 255), 2, 8); + if (objectClass < classNamesVec.size()) + { + ss.str(""); + ss << confidence; + String conf(ss.str()); + String label = String(classNamesVec[objectClass]) + ": " + conf; + int baseLine = 0; + Size labelSize = getTextSize(label, FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine); + rectangle(frame, Rect(Point(xLeftBottom, yLeftBottom), + Size(labelSize.width, labelSize.height + baseLine)), + Scalar(255, 255, 255), FILLED); + putText(frame, label, Point(xLeftBottom, yLeftBottom + labelSize.height), + FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 0, 0)); + } + } + } + imshow("YOLO-Detections", frame); + waitKey(0); + return; +} diff --git a/client/webrtc_capture/src/cv_yolo.h b/client/webrtc_capture/src/cv_yolo.h new file mode 100644 index 0000000..71fca6b --- /dev/null +++ b/client/webrtc_capture/src/cv_yolo.h @@ -0,0 +1,19 @@ +#ifndef CV_YOLO_H +#define CV_YOLO_H + +#include +#include + +#include +#include +#include +#include +using namespace std; +using namespace cv; +using namespace cv::dnn; + + +void video_detection(); +void image_detection(); +void test_yolo(); +#endif // CV_YOLO_H diff --git a/client/webrtc_capture/src/main.cpp b/client/webrtc_capture/src/main.cpp index ebbfe32..2bb2190 100644 --- a/client/webrtc_capture/src/main.cpp +++ b/client/webrtc_capture/src/main.cpp @@ -1,7 +1,7 @@ #include "mainwindow.h" #include "rtc.h" #include - +#include "cv_yolo.h" #include "modules/video_capture/video_capture.h" #include "video_source_impl.h" #include @@ -11,6 +11,7 @@ #include "camera_video_sink.h" #include "video_source_impl.h" #include +#include "cv_ssd.h" # pragma comment(lib, "secur32.lib") @@ -21,6 +22,21 @@ # pragma comment(lib, "Strmiids.lib") # pragma comment(lib, "User32.lib") + +int BubbleSort(int *p,int len){ + if(nullptr == p) + return -1; + for(int i = 0;i < len - 1;i++){ + for(int j = 0;j < len - 1;j++){ + if(p[j] < p[j + 1]){ + int tmp = p[j]; + p[j] = p[j+1]; + p[j+1] = tmp; + } + } + } +} + void EnumCapture() { rtc::WinsockInitializer winsock_init; @@ -29,7 +45,6 @@ void EnumCapture() rtc::ThreadManager::Instance()->SetCurrentThread(&w32_thread); rtc::InitializeSSL(); - std::unique_ptr info( webrtc::VideoCaptureFactory::CreateDeviceInfo()); @@ -56,6 +71,11 @@ void EnumCapture() int main(int argc, char *argv[]) { + int p[5] = {3,1,49,23,23}; + BubbleSort(p,5); + for(int i = 0;i< 5;i++){ + qDebug()<>("rtc::scoped_refptr"); qRegisterMetaType>("rtc::scoped_refptr&"); @@ -73,3 +93,7 @@ int main(int argc, char *argv[]) w.show(); return a.exec(); } + + + + diff --git a/client/webrtc_capture/src/mainwindow.cpp b/client/webrtc_capture/src/mainwindow.cpp index b65773c..7bab47c 100644 --- a/client/webrtc_capture/src/mainwindow.cpp +++ b/client/webrtc_capture/src/mainwindow.cpp @@ -8,6 +8,7 @@ #include #include #include "cvhelper.h" +#include "cv_ssd.h" class AsyncRennder :public QSSASyncProcess{ public: @@ -168,20 +169,25 @@ cv::Mat *QImage2cvMat(QImage image) void MainWindow::on_pushButton_2_clicked() { QPixmap pix = ui->openGLWidget->grab(); - QImage image = pix.toImage(); - image = image.scaled(1280,720); origin_picture->setPixmap(pix); - qDebug()<<"format is "<processed_picture->setPixmap(QPixmap::fromImage(image)); +} + +// 识别 +void MainWindow::on_pushButton_3_clicked() +{ + if(nullptr != origin_picture->pixmap()){ + QImage image = origin_picture->pixmap()->toImage(); + image = image.scaled(1280,720); + cv::Mat pic = *QImage2cvMat(image); + + cv::Mat *result = ssd_detect(&pic); +// try{ +//// cv::imshow("img", *result); +// }catch (std::exception &e){ +// qDebug()<processed_picture->setPixmap(QPixmap::fromImage(image)); + } } diff --git a/client/webrtc_capture/src/mainwindow.h b/client/webrtc_capture/src/mainwindow.h index e9aec31..bacffd4 100644 --- a/client/webrtc_capture/src/mainwindow.h +++ b/client/webrtc_capture/src/mainwindow.h @@ -23,6 +23,8 @@ private slots: void RenderDone(); void on_pushButton_2_clicked(); + void on_pushButton_3_clicked(); + private: Ui::MainWindow *ui; std::unique_ptr m_capturer; diff --git a/client/webrtc_capture/src/mainwindow.ui b/client/webrtc_capture/src/mainwindow.ui index 713bd32..47a59a7 100644 --- a/client/webrtc_capture/src/mainwindow.ui +++ b/client/webrtc_capture/src/mainwindow.ui @@ -22,7 +22,7 @@ - + @@ -63,6 +63,13 @@ + + + + 检测 + + + diff --git a/client/webrtc_capture/webrtc_capture.pro b/client/webrtc_capture/webrtc_capture.pro index 00cd386..0c0427a 100644 --- a/client/webrtc_capture/webrtc_capture.pro +++ b/client/webrtc_capture/webrtc_capture.pro @@ -23,6 +23,8 @@ INCLUDEPATH += third/include/ SOURCES += \ src/camera_video_sink.cpp \ src/cplaywidget.cpp \ + src/cv_ssd.cpp \ + src/cv_yolo.cpp \ src/cvhelper.cpp \ src/main.cpp \ src/mainwindow.cpp \ @@ -31,6 +33,8 @@ SOURCES += \ HEADERS += \ src/camera_video_sink.h \ src/cplaywidget.h \ + src/cv_ssd.h \ + src/cv_yolo.h \ src/cvhelper.h \ src/mainwindow.h \ src/rtc.h \ @@ -46,8 +50,10 @@ else: unix:!android: target.path = /opt/$${TARGET}/bin CONFIG(debug, debug|release){ message("debug mode") - LIBS += -L$$PWD/third/lib libwebrtc.lib ole32.lib oleaut32.lib strmiids.lib advapi32.lib opencv_core455d.lib - LIBS +=opencv_stitching455d.lib opencv_objdetect455d.lib opencv_ml455d.lib opencv_imgcodecs455d.lib opencv_imgproc455d.lib + LIBS += -L$$PWD/third/lib libwebrtc.lib ole32.lib oleaut32.lib strmiids.lib + LIBS += advapi32.lib opencv_core455d.lib + LIBS += opencv_stitching455d.lib opencv_objdetect455d.lib opencv_videoio455d.lib + LIBS += opencv_ml455d.lib opencv_imgcodecs455d.lib opencv_imgproc455d.lib opencv_dnn455d.lib LIBS+= opencv_highgui455d.lib DEFINES += DEBUG_FLAG Qt += debug From ed2d2e9e09a8e559ed09643f06faa0b0ddc64b4b Mon Sep 17 00:00:00 2001 From: zcy <290198252@qq.com> Date: Sat, 21 May 2022 20:44:46 +0800 Subject: [PATCH 5/9] no message --- .../webrtc_capture/src/camera_video_sink.cpp | 7 +- client/webrtc_capture/src/cv_ssd.cpp | 97 ++++++++++++++----- client/webrtc_capture/src/mainwindow.ui | 9 +- 3 files changed, 87 insertions(+), 26 deletions(-) diff --git a/client/webrtc_capture/src/camera_video_sink.cpp b/client/webrtc_capture/src/camera_video_sink.cpp index cd55558..ae7afac 100644 --- a/client/webrtc_capture/src/camera_video_sink.cpp +++ b/client/webrtc_capture/src/camera_video_sink.cpp @@ -76,20 +76,21 @@ void CameraVideoSink::OnFrame(const webrtc::VideoFrame& frame) { std::chrono::system_clock::now().time_since_epoch()).count(); static size_t cnt = 0; - qDebug()<type()); +// qDebug()<type()); rtc::scoped_refptr frameBuffer = frame.video_frame_buffer()->ToI420(); cnt++; int width = this->m_capability.width; int height = this->m_capability.height; + /* qDebug()<Capability().height<Capability().width <Capability().videoType) <Capability().interlaced<Capability().interlaced<GetI420()->type()); +// qDebug()<GetI420()->type()); memcpy(data,frameBuffer->GetI420()->DataY(),width*height); memcpy(data + width*height ,frameBuffer->GetI420()->DataU(), width*height/4); diff --git a/client/webrtc_capture/src/cv_ssd.cpp b/client/webrtc_capture/src/cv_ssd.cpp index 2f04590..cbed0bf 100644 --- a/client/webrtc_capture/src/cv_ssd.cpp +++ b/client/webrtc_capture/src/cv_ssd.cpp @@ -13,29 +13,44 @@ String objNames[] = { "background", "motorbike", "person", "pottedplant", "sheep", "sofa", "train", "tvmonitor" }; - cv::Mat * ssd_detect(cv::Mat *inframe) { if (inframe->empty()) { printf("could not load image...\n"); return inframe; } namedWindow("input image", WINDOW_AUTOSIZE); - imshow("input image", *inframe); - - Net net = readNetFromCaffe(model_text_file, modelFile); - -// image：这个就是我们将要输入神经网络进行处理或者分类的图片。 -// mean：需要将图片整体减去的平均值，如果我们需要对RGB图片的三个通道分别减去不同的值，那么可以使用3组平均值，如果只使用一组，那么就默认对三个通道减去一样的值。减去平均值（mean）：为了消除同一场景下不同光照的图片，对我们最终的分类或者神经网络的影响，我们常常对图片的R、G、B通道的像素求一个平均值，然后将每个像素值减去我们的平均值，这样就可以得到像素之间的相对值，就可以排除光照的影响。 -// scalefactor：当我们将图片减去平均值之后，还可以对剩下的像素值进行一定的尺度缩放，它的默认值是1，如果希望减去平均像素之后的值，全部缩小一半，那么可以将scalefactor设为1/2。 -// size：这个参数是我们神经网络在训练的时候要求输入的图片尺寸。 -// swapRB：OpenCV中认为我们的图片通道顺序是BGR，但是我平均值假设的顺序是RGB，所以如果需要交换R和G，那么就要使swapRB=true - Mat blobImage = blobFromImage(*inframe, 0.007843, + cv::Mat resize; + cv::resize(*inframe, resize, + Size(480,560), + 0, 0, INTER_LINEAR);// X Y各缩小一半 + qDebug()<<"frame.type()"<(i, 2); // if (confidence > confidence_threshold) { // size_t objIndex = (size_t)(detectionMat.at(i, 1)); -// float tl_x = detectionMat.at(i, 3) * frame->cols; -// float tl_y = detectionMat.at(i, 4) * frame->rows; -// float br_x = detectionMat.at(i, 5) * frame->cols; -// float br_y = detectionMat.at(i, 6) * frame->rows; +// float tl_x = detectionMat.at(i, 3) * resize.cols; +// float tl_y = detectionMat.at(i, 4) * resize.rows; +// float br_x = detectionMat.at(i, 5) * resize.cols; +// float br_y = detectionMat.at(i, 6) * resize.rows; // Rect object_box((int)tl_x, (int)tl_y, (int)(br_x - tl_x), (int)(br_y - tl_y)); -// rectangle(*frame, object_box, Scalar(0, 0, 255), 2, 8, 0); -// putText(*frame, format("%s", objNames[objIndex].c_str()), Point(tl_x, tl_y), +// rectangle(resize, object_box, Scalar(0, 0, 255), 2, 8, 0); +// putText(resize, format("%s", objNames[objIndex].c_str()), Point(tl_x, tl_y), // FONT_HERSHEY_SIMPLEX, 1.0, Scalar(255, 0, 0), 2); // } // } +// imshow("ssd-demo", resize); -// waitKey(0); - return inframe; +////// waitKey(0); +// return inframe; + Mat frame = imread("D:/project/opencv_tutorial_data/images/gaoyy.png"); + + if (frame.empty()) { + printf("could not load image...\n"); + return nullptr; + } + qDebug()<<"frame.type()"<()); + float confidence_threshold = 0.2; + for (int i = 0; i < detectionMat.rows; i++) { + float confidence = detectionMat.at(i, 2); + if (confidence > confidence_threshold) { + size_t objIndex = (size_t)(detectionMat.at(i, 1)); + float tl_x = detectionMat.at(i, 3) * image2.cols; + float tl_y = detectionMat.at(i, 4) * image2.rows; + float br_x = detectionMat.at(i, 5) * image2.cols; + float br_y = detectionMat.at(i, 6) * image2.rows; + + Rect object_box((int)tl_x, (int)tl_y, (int)(br_x - tl_x), (int)(br_y - tl_y)); + rectangle(image2, object_box, Scalar(0, 0, 255), 2, 8, 0); + putText(image2, format("%s", objNames[objIndex].c_str()), Point(tl_x, tl_y), FONT_HERSHEY_SIMPLEX, 1.0, Scalar(255, 0, 0), 2); + } + } + imshow("ssd-demo", image2); + waitKey(0); + return nullptr; } diff --git a/client/webrtc_capture/src/mainwindow.ui b/client/webrtc_capture/src/mainwindow.ui index 47a59a7..5c231b8 100644 --- a/client/webrtc_capture/src/mainwindow.ui +++ b/client/webrtc_capture/src/mainwindow.ui @@ -22,7 +22,7 @@ - + @@ -70,6 +70,13 @@ + + + + 动态检测 + + + From 1ba89778f5de940fe876a2dbd5af5be8408dd779 Mon Sep 17 00:00:00 2001 From: zcy <290198252@qq.com> Date: Mon, 23 May 2022 01:20:11 +0800 Subject: [PATCH 6/9] add dynamic detect --- .../webrtc_capture/src/camera_video_sink.cpp | 8 +- client/webrtc_capture/src/camera_video_sink.h | 2 - client/webrtc_capture/src/cplaywidget.cpp | 26 +++- client/webrtc_capture/src/cplaywidget.h | 2 + client/webrtc_capture/src/cv_ssd.cpp | 72 ++-------- client/webrtc_capture/src/mainwindow.cpp | 134 ++++++++++++++---- client/webrtc_capture/src/mainwindow.h | 4 + 7 files changed, 146 insertions(+), 102 deletions(-) diff --git a/client/webrtc_capture/src/camera_video_sink.cpp b/client/webrtc_capture/src/camera_video_sink.cpp index ae7afac..179f666 100644 --- a/client/webrtc_capture/src/camera_video_sink.cpp +++ b/client/webrtc_capture/src/camera_video_sink.cpp @@ -82,14 +82,8 @@ void CameraVideoSink::OnFrame(const webrtc::VideoFrame& frame) { cnt++; int width = this->m_capability.width; int height = this->m_capability.height; - /* - qDebug()<Capability().height<Capability().width - <Capability().videoType) - <Capability().interlaced<GetI420()->type()); memcpy(data,frameBuffer->GetI420()->DataY(),width*height); memcpy(data + width*height ,frameBuffer->GetI420()->DataU(), diff --git a/client/webrtc_capture/src/camera_video_sink.h b/client/webrtc_capture/src/camera_video_sink.h index 4eb512f..9043a87 100644 --- a/client/webrtc_capture/src/camera_video_sink.h +++ b/client/webrtc_capture/src/camera_video_sink.h @@ -2,8 +2,6 @@ #define CAMERA_VIDEO_SINK_H // vcm_capturer_test.h - - #include #include #include "modules/video_capture/video_capture.h" diff --git a/client/webrtc_capture/src/cplaywidget.cpp b/client/webrtc_capture/src/cplaywidget.cpp index c2aca64..00c47d0 100644 --- a/client/webrtc_capture/src/cplaywidget.cpp +++ b/client/webrtc_capture/src/cplaywidget.cpp @@ -83,6 +83,28 @@ void CPlayWidget::OnPaintData(const rtc::scoped_refptr update(); } + + int CPlayWidget::OnCameraData(uint8_t *p) { memcpy(m_pBufYuv420p,p,m_nVideoH*m_nVideoW/2*3); diff --git a/client/webrtc_capture/src/cplaywidget.h b/client/webrtc_capture/src/cplaywidget.h index a257af5..8bd5a8a 100644 --- a/client/webrtc_capture/src/cplaywidget.h +++ b/client/webrtc_capture/src/cplaywidget.h @@ -36,6 +36,8 @@ public: TYPE_I420, TYPE_UNSET, }IMG_TYPE; + CPlayWidget(QWidget* parent,IMG_TYPE type); + CPlayWidget(QWidget* parent); ~CPlayWidget(); int SetDataType(IMG_TYPE); diff --git a/client/webrtc_capture/src/cv_ssd.cpp b/client/webrtc_capture/src/cv_ssd.cpp index cbed0bf..1a7e8c1 100644 --- a/client/webrtc_capture/src/cv_ssd.cpp +++ b/client/webrtc_capture/src/cv_ssd.cpp @@ -18,18 +18,13 @@ cv::Mat * ssd_detect(cv::Mat *inframe) { printf("could not load image...\n"); return inframe; } - namedWindow("input image", WINDOW_AUTOSIZE); cv::Mat resize; cv::resize(*inframe, resize, Size(480,560), 0, 0, INTER_LINEAR);// X Y各缩小一半 - qDebug()<<"frame.type()"<()); -// float confidence_threshold = 0.2; -// for (int i = 0; i < detectionMat.rows; i++) { -// float confidence = detectionMat.at(i, 2); -// if (confidence > confidence_threshold) { -// size_t objIndex = (size_t)(detectionMat.at(i, 1)); -// float tl_x = detectionMat.at(i, 3) * resize.cols; -// float tl_y = detectionMat.at(i, 4) * resize.rows; -// float br_x = detectionMat.at(i, 5) * resize.cols; -// float br_y = detectionMat.at(i, 6) * resize.rows; - -// Rect object_box((int)tl_x, (int)tl_y, (int)(br_x - tl_x), (int)(br_y - tl_y)); -// rectangle(resize, object_box, Scalar(0, 0, 255), 2, 8, 0); -// putText(resize, format("%s", objNames[objIndex].c_str()), Point(tl_x, tl_y), -// FONT_HERSHEY_SIMPLEX, 1.0, Scalar(255, 0, 0), 2); -// } -// } -// imshow("ssd-demo", resize); - -////// waitKey(0); -// return inframe; - Mat frame = imread("D:/project/opencv_tutorial_data/images/gaoyy.png"); - - if (frame.empty()) { - printf("could not load image...\n"); - return nullptr; - } - qDebug()<<"frame.type()"<()); @@ -99,19 +49,17 @@ cv::Mat * ssd_detect(cv::Mat *inframe) { float confidence = detectionMat.at(i, 2); if (confidence > confidence_threshold) { size_t objIndex = (size_t)(detectionMat.at(i, 1)); - float tl_x = detectionMat.at(i, 3) * image2.cols; - float tl_y = detectionMat.at(i, 4) * image2.rows; - float br_x = detectionMat.at(i, 5) * image2.cols; - float br_y = detectionMat.at(i, 6) * image2.rows; + float tl_x = detectionMat.at(i, 3) * image2->cols; + float tl_y = detectionMat.at(i, 4) * image2->rows; + float br_x = detectionMat.at(i, 5) * image2->cols; + float br_y = detectionMat.at(i, 6) * image2->rows; Rect object_box((int)tl_x, (int)tl_y, (int)(br_x - tl_x), (int)(br_y - tl_y)); - rectangle(image2, object_box, Scalar(0, 0, 255), 2, 8, 0); - putText(image2, format("%s", objNames[objIndex].c_str()), Point(tl_x, tl_y), FONT_HERSHEY_SIMPLEX, 1.0, Scalar(255, 0, 0), 2); + rectangle(*image2, object_box, Scalar(0, 0, 255), 2, 8, 0); + putText(*image2, format("%s", objNames[objIndex].c_str()), Point(tl_x, tl_y), FONT_HERSHEY_SIMPLEX, 1.0, Scalar(255, 0, 0), 2); } } - imshow("ssd-demo", image2); - waitKey(0); - return nullptr; + return image2; } diff --git a/client/webrtc_capture/src/mainwindow.cpp b/client/webrtc_capture/src/mainwindow.cpp index 7bab47c..e432fbe 100644 --- a/client/webrtc_capture/src/mainwindow.cpp +++ b/client/webrtc_capture/src/mainwindow.cpp @@ -9,6 +9,76 @@ #include #include "cvhelper.h" #include "cv_ssd.h" +#include +#include + + +class ASyncDetectAndRenderThread :public QSSASyncProcess{ +public: + typedef enum { + STATUS_STOP = 1, + STATUS_RUNNING = 2 + }Status; + ASyncDetectAndRenderThread(QWidget * parent,CPlayWidget *render_ui, + int width,int height){ + m_render = render_ui; + m_parent = parent; + m_status = STATUS_RUNNING; + m_width = width; + m_height = height; + } + bool DetectStatus(){ + if(m_mat.size() > 0) + return true; + else + return false; + } + void SetCvImage(uint8_t *frame){ + cv::Mat yuv420p; + yuv420p.create(m_height*3/2, + m_width, CV_8UC1); + memcpy(yuv420p.data, frame, m_height*m_width + *sizeof(unsigned char)*3/2); + cv::Mat *rgbImg = new cv::Mat; + cv::cvtColor(yuv420p, *rgbImg, cv::COLOR_YUV2BGR_I420); + std::lock_guard guard(this->m_mutex); +// imshow("yuv420",yuv420p); + m_mat.push_back(rgbImg); + } + + void Run(void *) override{ + while(m_status == STATUS_RUNNING){ + cv::Mat *c = takeLast(); + if(nullptr != c){ + cv::Mat *result = ssd_detect(c); + qDebug()<cols<rows<type(); + cv::Mat yuvData; + cv::cvtColor(*result, yuvData, cv::COLOR_BGR2YUV_I420); + this->m_render->OnCameraData(yuvData.data); + } + } + } + +private: + cv::Mat *takeLast(){ + std::lock_guard guard(this->m_mutex); + if(m_mat.size() != 0){ + auto ret = *m_mat.begin(); + m_mat.pop_front(); + return ret; + } + return nullptr; + } + std::list m_mat; + std::mutex m_mutex; + CPlayWidget *m_render; + QWidget *m_parent; + Status m_status; + int m_width; + int m_height; +}; + +ASyncDetectAndRenderThread *gdetect = nullptr; class AsyncRennder :public QSSASyncProcess{ public: @@ -32,27 +102,13 @@ public: while(state){ if(mfbs ->Size() > 0){ - uint8_t *frame = this->mfbs->TakeLast(); - cv::Mat yuv420p; - yuv420p.create(mUI->RenderHeight()*3/2, - mUI->RenderWidth(), CV_8UC1); - memcpy(yuv420p.data, frame, mUI->RenderHeight()*mUI->RenderWidth() - *sizeof(unsigned char)*3/2); - cv::Mat rgbImg; - cv::cvtColor(yuv420p, rgbImg, cv::COLOR_YUV2BGR_I420); - -// cv::Mat dst; -// cv::rotate(rgbImg, dst, cv::ROTATE_90_CLOCKWISE); // 绕x翻转，上下颠倒 -// cv::imshow("img", dst); -//// cv::Mat gray, dst; -//// cvtColor(src, gray, COLOR_BGR2GRAY); -//// imshow("input", gray); -//// equalizeHist(gray, dst); -//// imshow("eq", dst); -// cv::waitKey(1); - + uint8_t *frame = mfbs->TakeLast(); mUI->OnCameraData(frame); -// qDebug()<<"dst size is "<DetectStatus()){ + gdetect->SetCvImage(frame); + } + } eventLoop.exec(); // 渲染一次 delete frame; } @@ -64,7 +120,9 @@ private: CPlayWidget *mUI; }; -AsyncRennder *gRender; + +AsyncRennder *gRender = nullptr; + MainWindow::MainWindow(QWidget *parent) : QssMainWindow(parent) @@ -92,6 +150,9 @@ MainWindow::MainWindow(QWidget *parent) origin_picture->setMaximumHeight(12000); ui->gridLayout->addWidget(processed_picture,1,0); + mDetectResut = new CPlayWidget(this); + ui->gridLayout->addWidget(mDetectResut,1,1); + } MainWindow::~MainWindow() @@ -106,6 +167,8 @@ void MainWindow::OnUpdateFrame( rtc::scoped_refptr& void MainWindow::on_pushButton_clicked() { + + int id = ui->comboBox->currentData().toInt(); webrtc::VideoCaptureCapability p; @@ -131,6 +194,10 @@ void MainWindow::on_pushButton_clicked() gRender->Start(this); connect(gRender,&QSSASyncProcess::Done,this,&MainWindow::RenderDone); } + mDetectResut->SetImgSize(480,560); + mDetectResut->StartRender(); + + } void MainWindow::RenderDone() @@ -154,14 +221,15 @@ cv::Mat *QImage2cvMat(QImage image) memcpy(mat->data,image.bits(),image.bytesPerLine()*image.height()); break; case QImage::Format_RGB888: - mat = new cv::Mat(image.height(), image.width(), CV_8UC3, (void*)image.bits(), image.bytesPerLine()); + mat = new cv::Mat(image.height(), image.width(), CV_8UC3, + (void*)image.bits(), image.bytesPerLine()); break; case QImage::Format_Indexed8: - mat = new cv::Mat(image.height(), image.width(), CV_8UC1, (void*)image.bits(), image.bytesPerLine()); + mat = new cv::Mat(image.height(), image.width(), CV_8UC1, + (void*)image.bits(), image.bytesPerLine()); break; } Save1BitImage(*mat,"d://tgest.png"); - return mat; } @@ -181,13 +249,19 @@ void MainWindow::on_pushButton_3_clicked() cv::Mat pic = *QImage2cvMat(image); cv::Mat *result = ssd_detect(&pic); -// try{ -//// cv::imshow("img", *result); -// }catch (std::exception &e){ -// qDebug()<processed_picture->setPixmap(QPixmap::fromImage(image)); } } + +void MainWindow::on_pushButton_4_clicked() +{ + if(gdetect == nullptr){ + gdetect = new ASyncDetectAndRenderThread(this,mDetectResut,ui->openGLWidget->RenderWidth(), + ui->openGLWidget->RenderHeight()); + gdetect->Start(this); + connect(gdetect,&QSSASyncProcess::Done,this,&MainWindow::RenderDone); + } +} + + diff --git a/client/webrtc_capture/src/mainwindow.h b/client/webrtc_capture/src/mainwindow.h index bacffd4..a8350f7 100644 --- a/client/webrtc_capture/src/mainwindow.h +++ b/client/webrtc_capture/src/mainwindow.h @@ -4,6 +4,7 @@ #include "rtc.h" #include "api/video/i420_buffer.h" #include "Qss.h" +#include "cplaywidget.h" QT_BEGIN_NAMESPACE namespace Ui { class MainWindow; } @@ -25,10 +26,13 @@ private slots: void on_pushButton_3_clicked(); + void on_pushButton_4_clicked(); + private: Ui::MainWindow *ui; std::unique_ptr m_capturer; QLabel *origin_picture; QLabel *processed_picture; + CPlayWidget *mDetectResut; }; #endif // MAINWINDOW_H From c54519de29aab1fbf82896659658ae58041d7127 Mon Sep 17 00:00:00 2001 From: zcy <290198252@qq.com> Date: Wed, 25 May 2022 21:34:18 +0800 Subject: [PATCH 7/9] =?UTF-8?q?=E6=94=B9=E6=88=90ssd=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E7=94=A8=E6=96=87=E4=BB=B6=E8=BE=93=E5=85=A5=E7=9A=84=E5=BD=A2?= =?UTF-8?q?=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../webrtc_capture/src/camera_video_sink.cpp | 2 + client/webrtc_capture/src/cv_ssd.cpp | 49 +++++-- client/webrtc_capture/src/cv_ssd.h | 3 +- client/webrtc_capture/src/mainwindow.cpp | 121 +++++++++++------- client/webrtc_capture/src/mainwindow.h | 2 + client/webrtc_capture/src/mainwindow.ui | 62 +++++++-- 6 files changed, 168 insertions(+), 71 deletions(-) diff --git a/client/webrtc_capture/src/camera_video_sink.cpp b/client/webrtc_capture/src/camera_video_sink.cpp index 179f666..f71f45d 100644 --- a/client/webrtc_capture/src/camera_video_sink.cpp +++ b/client/webrtc_capture/src/camera_video_sink.cpp @@ -93,9 +93,11 @@ void CameraVideoSink::OnFrame(const webrtc::VideoFrame& frame) { width*height/4); m_buf.PushFirst(data); + auto timestamp_curr = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()).count(); if(timestamp_curr - timestamp > 1000) { + qDebug()< -const size_t width = 1280; -const size_t height = 720; +#include + +using namespace std; + String labelFile = "ssd/labelmap_det.txt"; -String modelFile = "D:/project/multimedia/client/build-webrtc_capture-Desktop_Qt_5_15_2_MSVC2019_64bit-Debug/debug/ssd/MobileNetSSD_deploy.caffemodel"; -String model_text_file = "D:/project/multimedia/client/build-webrtc_capture-Desktop_Qt_5_15_2_MSVC2019_64bit-Debug/debug/ssd/MobileNetSSD_deploy.prototxt"; +String modelFile = "debug/ssd/MobileNetSSD_deploy.caffemodel"; +String model_text_file = "debug/ssd/MobileNetSSD_deploy.prototxt"; -String objNames[] = { "background", -"aeroplane", "bicycle", "bird", "boat", -"bottle", "bus", "car", "cat", "chair", -"cow", "diningtable", "dog", "horse", -"motorbike", "person", "pottedplant", -"sheep", "sofa", "train", "tvmonitor" }; +String objNames[] = { + "background", + "aeroplane", + "bicycle", + "bird", + "boat", + "bottle", + "bus", + "car", + "cat", + "chair", + "cow", + "diningtable", + "dog", + "horse", + "motorbike", + "person", + "pottedplant", + "sheep", + "sofa", + "train", + "tvmonitor" +}; -cv::Mat * ssd_detect(cv::Mat *inframe) { - if (inframe->empty()) { +cv::Mat * ssd_detect(cv::Mat *inframe,std::string modelFile,std::string model_text_file) { + if (inframe->empty() || (modelFile == "") || (model_text_file == "")) { printf("could not load image...\n"); return inframe; } @@ -28,7 +47,6 @@ cv::Mat * ssd_detect(cv::Mat *inframe) { Size(300, 300), Scalar(127.5, 127.5, 127.5), true, false); -// Net net = readNetFromCaffe(model_text_file, modelFile); //// image：这个就是我们将要输入神经网络进行处理或者分类的图片。 //// mean：需要将图片整体减去的平均值，如果我们需要对RGB图片的三个通道分别减去不同的值，那么可以使用3组平均值， @@ -40,6 +58,7 @@ cv::Mat * ssd_detect(cv::Mat *inframe) { //// size：这个参数是我们神经网络在训练的时候要求输入的图片尺寸。 //// swapRB：OpenCV中认为我们的图片通道顺序是BGR，但是我平均值假设的顺序是RGB， /// 所以如果需要交换R和G，那么就要使swapRB=true +/// Net net = readNetFromCaffe(model_text_file, modelFile); net.setInput(blobImage1, "data"); Mat detection = net.forward("detection_out"); @@ -56,9 +75,11 @@ cv::Mat * ssd_detect(cv::Mat *inframe) { Rect object_box((int)tl_x, (int)tl_y, (int)(br_x - tl_x), (int)(br_y - tl_y)); rectangle(*image2, object_box, Scalar(0, 0, 255), 2, 8, 0); - putText(*image2, format("%s", objNames[objIndex].c_str()), Point(tl_x, tl_y), FONT_HERSHEY_SIMPLEX, 1.0, Scalar(255, 0, 0), 2); + putText(*image2, format("%s", objNames[objIndex].c_str()), + Point(tl_x, tl_y), FONT_HERSHEY_SIMPLEX, 1.0, Scalar(255, 0, 0), 2); } } + cv::imshow("image2",*image2); return image2; } diff --git a/client/webrtc_capture/src/cv_ssd.h b/client/webrtc_capture/src/cv_ssd.h index 66f20db..5cdb93e 100644 --- a/client/webrtc_capture/src/cv_ssd.h +++ b/client/webrtc_capture/src/cv_ssd.h @@ -9,8 +9,7 @@ using namespace cv; using namespace cv::dnn; using namespace std; -cv::Mat * ssd_detect(cv::Mat*); - +cv::Mat * ssd_detect(cv::Mat *inframe,std::string modelFile,std::string model_text_file); diff --git a/client/webrtc_capture/src/mainwindow.cpp b/client/webrtc_capture/src/mainwindow.cpp index e432fbe..d4935fb 100644 --- a/client/webrtc_capture/src/mainwindow.cpp +++ b/client/webrtc_capture/src/mainwindow.cpp @@ -11,6 +11,7 @@ #include "cv_ssd.h" #include #include +#include class ASyncDetectAndRenderThread :public QSSASyncProcess{ @@ -20,19 +21,26 @@ public: STATUS_RUNNING = 2 }Status; ASyncDetectAndRenderThread(QWidget * parent,CPlayWidget *render_ui, - int width,int height){ + int width,int height, + std::string model_path, + std::string model_txt_path){ m_render = render_ui; m_parent = parent; m_status = STATUS_RUNNING; m_width = width; m_height = height; + this->m_model_path = model_path; + this->m_model_txt_path = model_txt_path; } - bool DetectStatus(){ + bool Detecting(){ if(m_mat.size() > 0) return true; else return false; } + Status DetectStatus(){ + return this->m_status; + } void SetCvImage(uint8_t *frame){ cv::Mat yuv420p; yuv420p.create(m_height*3/2, @@ -42,7 +50,6 @@ public: cv::Mat *rgbImg = new cv::Mat; cv::cvtColor(yuv420p, *rgbImg, cv::COLOR_YUV2BGR_I420); std::lock_guard guard(this->m_mutex); -// imshow("yuv420",yuv420p); m_mat.push_back(rgbImg); } @@ -50,12 +57,18 @@ public: while(m_status == STATUS_RUNNING){ cv::Mat *c = takeLast(); if(nullptr != c){ - cv::Mat *result = ssd_detect(c); + cv::Mat *result = ssd_detect(c, + this->m_model_path, + this->m_model_txt_path); + qDebug()<cols<rows<type(); cv::Mat yuvData; + cv::cvtColor(*result, yuvData, cv::COLOR_BGR2YUV_I420); this->m_render->OnCameraData(yuvData.data); + } + delete c; } } @@ -68,7 +81,7 @@ private: return ret; } return nullptr; - } + } std::list m_mat; std::mutex m_mutex; CPlayWidget *m_render; @@ -76,6 +89,8 @@ private: Status m_status; int m_width; int m_height; + std::string m_model_path; // 模型路径 + std::string m_model_txt_path; // 模型txt路径 }; ASyncDetectAndRenderThread *gdetect = nullptr; @@ -87,6 +102,7 @@ public: this->mfbs = p; mUI = render_ui; this->state = true; + } // 停止渲染 void StopRender(){ @@ -98,14 +114,14 @@ public: qtimer->setSingleShot(false); QObject::connect(qtimer, SIGNAL(timeout()), &eventLoop, SLOT(quit())); - qtimer->start(3); + qtimer->start(22); while(state){ if(mfbs ->Size() > 0){ uint8_t *frame = mfbs->TakeLast(); mUI->OnCameraData(frame); if(gdetect != nullptr){ - if(!gdetect->DetectStatus()){ + if(!gdetect->Detecting()){ gdetect->SetCvImage(frame); } } @@ -142,16 +158,8 @@ MainWindow::MainWindow(QWidget *parent) info->GetDeviceName(i,name,100,nullptr,0,nullptr,0); ui->comboBox->addItem(QString::asprintf("%s",name),i); } - origin_picture = new QLabel(this); - processed_picture = new QLabel(this); - - ui->gridLayout->addWidget(origin_picture,0,1); - origin_picture->setMaximumWidth(12000); - origin_picture->setMaximumHeight(12000); - - ui->gridLayout->addWidget(processed_picture,1,0); mDetectResut = new CPlayWidget(this); - ui->gridLayout->addWidget(mDetectResut,1,1); + ui->gridLayout->addWidget(mDetectResut,0,1); } @@ -167,37 +175,39 @@ void MainWindow::OnUpdateFrame( rtc::scoped_refptr& void MainWindow::on_pushButton_clicked() { + if(ui->pushButton->text() == QString("采集")) { + int id = ui->comboBox->currentData().toInt(); + webrtc::VideoCaptureCapability p; + std::unique_ptr info( + webrtc::VideoCaptureFactory::CreateDeviceInfo()); - int id = ui->comboBox->currentData().toInt(); - webrtc::VideoCaptureCapability p; + char ids[128]; + info->GetDeviceName(id,nullptr,0,ids,128,nullptr,0); + info->GetCapability(ids,id,p); + qDebug()< info( - webrtc::VideoCaptureFactory::CreateDeviceInfo()); + m_capturer.reset(CameraVideoSink::Create(p.width, p.height, 25, id)); + if (!m_capturer) { + qDebug()<<"error"; + } + ui->openGLWidget->SetDataType(CPlayWidget::TYPE_YUV420P); + ui->openGLWidget->SetImgSize(m_capturer->Capability().width, + m_capturer->Capability().height); + ui->openGLWidget->StartRender(); + // ui->openGLWidget->moveToThread(&gRender->Thread()); + if(gRender == nullptr){ + gRender = new AsyncRennder(this,m_capturer->VideoBuffer(),ui->openGLWidget); + gRender->Start(this); + connect(gRender,&QSSASyncProcess::Done,this,&MainWindow::RenderDone); + } + mDetectResut->SetImgSize(480,560); + mDetectResut->StartRender(); - char ids[128]; - info->GetDeviceName(id,nullptr,0,ids,128,nullptr,0); - info->GetCapability(ids,id,p); - qDebug()<pushButton->setText(QString::asprintf("正在采集")); + }else{ + ui->openGLWidget->StopRender(); } - ui->openGLWidget->SetDataType(CPlayWidget::TYPE_YUV420P); - ui->openGLWidget->SetImgSize(m_capturer->Capability().width, - m_capturer->Capability().height); - ui->openGLWidget->StartRender(); -// ui->openGLWidget->moveToThread(&gRender->Thread()); - if(gRender == nullptr){ - gRender = new AsyncRennder(this,m_capturer->VideoBuffer(),ui->openGLWidget); - gRender->Start(this); - connect(gRender,&QSSASyncProcess::Done,this,&MainWindow::RenderDone); - } - mDetectResut->SetImgSize(480,560); - mDetectResut->StartRender(); - - } void MainWindow::RenderDone() @@ -205,8 +215,6 @@ void MainWindow::RenderDone() } - - cv::Mat *QImage2cvMat(QImage image) { cv::Mat *mat; @@ -247,8 +255,13 @@ void MainWindow::on_pushButton_3_clicked() QImage image = origin_picture->pixmap()->toImage(); image = image.scaled(1280,720); cv::Mat pic = *QImage2cvMat(image); + if((ui->lineEdit->text() == "") || (ui->lineEdit_2->text() == "")){ - cv::Mat *result = ssd_detect(&pic); + } + cv::Mat *result = ssd_detect(&pic, + ui->lineEdit->text().toStdString(), + ui->lineEdit_2->text().toStdString() + ); this->processed_picture->setPixmap(QPixmap::fromImage(image)); } } @@ -258,10 +271,26 @@ void MainWindow::on_pushButton_4_clicked() { if(gdetect == nullptr){ gdetect = new ASyncDetectAndRenderThread(this,mDetectResut,ui->openGLWidget->RenderWidth(), - ui->openGLWidget->RenderHeight()); + ui->openGLWidget->RenderHeight(), + ui->lineEdit->text().toStdString(), + ui->lineEdit_2->text().toStdString()); gdetect->Start(this); connect(gdetect,&QSSASyncProcess::Done,this,&MainWindow::RenderDone); } } +// 导入文件 +void MainWindow::on_pushButton_5_clicked() +{ + QString label1 = ui->lineEdit_2->text(); + QString openFile = QFileDialog::getOpenFileName(this, "0.0", "", + "*.caffemodel *.prototxt",nullptr); + if (openFile.contains(".caffemodel")){ + ui->lineEdit->setText(openFile); + } + if (openFile.contains(".prototxt")){ + ui->lineEdit_2->setText(openFile); + } +} + diff --git a/client/webrtc_capture/src/mainwindow.h b/client/webrtc_capture/src/mainwindow.h index a8350f7..89a6d49 100644 --- a/client/webrtc_capture/src/mainwindow.h +++ b/client/webrtc_capture/src/mainwindow.h @@ -28,6 +28,8 @@ private slots: void on_pushButton_4_clicked(); + void on_pushButton_5_clicked(); + private: Ui::MainWindow *ui; std::unique_ptr m_capturer; diff --git a/client/webrtc_capture/src/mainwindow.ui b/client/webrtc_capture/src/mainwindow.ui index 5c231b8..86b6706 100644 --- a/client/webrtc_capture/src/mainwindow.ui +++ b/client/webrtc_capture/src/mainwindow.ui @@ -20,9 +20,9 @@ MainWindow - + - + @@ -70,13 +70,6 @@ - - - - 动态检测 - - - @@ -92,6 +85,57 @@ + + + + + + 模型文件: .caffemodel + + + + + + + + + + .prototxt: + + + + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + 导入 + + + + + + + 动态检测 + + + + + From 91115809937206dbfc27d2a732d3e70242d97090 Mon Sep 17 00:00:00 2001 From: zcy <290198252@qq.com> Date: Wed, 25 May 2022 21:40:29 +0800 Subject: [PATCH 8/9] no message --- client/webrtc_capture/src/mainwindow.cpp | 8 +++++++- client/webrtc_capture/src/mainwindow.ui | 16 +--------------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/client/webrtc_capture/src/mainwindow.cpp b/client/webrtc_capture/src/mainwindow.cpp index d4935fb..ee1f4f5 100644 --- a/client/webrtc_capture/src/mainwindow.cpp +++ b/client/webrtc_capture/src/mainwindow.cpp @@ -12,7 +12,7 @@ #include #include #include - +#include "Qss.h" class ASyncDetectAndRenderThread :public QSSASyncProcess{ public: @@ -270,6 +270,12 @@ void MainWindow::on_pushButton_3_clicked() void MainWindow::on_pushButton_4_clicked() { if(gdetect == nullptr){ + if((ui->lineEdit->text().toStdString() == "") + || + (ui->lineEdit_2->text().toStdString() == "")){ + QssMessageBox::warn("",nullptr,"请输入模型",QMessageBox::Ok); + + } gdetect = new ASyncDetectAndRenderThread(this,mDetectResut,ui->openGLWidget->RenderWidth(), ui->openGLWidget->RenderHeight(), ui->lineEdit->text().toStdString(), diff --git a/client/webrtc_capture/src/mainwindow.ui b/client/webrtc_capture/src/mainwindow.ui index 86b6706..4d1bfc9 100644 --- a/client/webrtc_capture/src/mainwindow.ui +++ b/client/webrtc_capture/src/mainwindow.ui @@ -22,7 +22,7 @@ - + @@ -56,20 +56,6 @@ - - - - 拍照 - - - - - - - 检测 - - - From ceaccb7db090d07a75d14c43e07a96773ac1e9a6 Mon Sep 17 00:00:00 2001 From: zcy <290198252@qq.com> Date: Thu, 26 May 2022 09:33:26 +0800 Subject: [PATCH 9/9] no message --- client/webrtc_capture/src/cv_ssd.cpp | 1 - client/webrtc_capture/src/mainwindow.cpp | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/client/webrtc_capture/src/cv_ssd.cpp b/client/webrtc_capture/src/cv_ssd.cpp index 517c98a..08e525f 100644 --- a/client/webrtc_capture/src/cv_ssd.cpp +++ b/client/webrtc_capture/src/cv_ssd.cpp @@ -79,7 +79,6 @@ cv::Mat * ssd_detect(cv::Mat *inframe,std::string modelFile,std::string model_te Point(tl_x, tl_y), FONT_HERSHEY_SIMPLEX, 1.0, Scalar(255, 0, 0), 2); } } - cv::imshow("image2",*image2); return image2; } diff --git a/client/webrtc_capture/src/mainwindow.cpp b/client/webrtc_capture/src/mainwindow.cpp index ee1f4f5..a7ac5c6 100644 --- a/client/webrtc_capture/src/mainwindow.cpp +++ b/client/webrtc_capture/src/mainwindow.cpp @@ -273,8 +273,8 @@ void MainWindow::on_pushButton_4_clicked() if((ui->lineEdit->text().toStdString() == "") || (ui->lineEdit_2->text().toStdString() == "")){ - QssMessageBox::warn("",nullptr,"请输入模型",QMessageBox::Ok); - + QssMessageBox::warn("please input model file",nullptr,"please input model file",QMessageBox::Ok); + return; } gdetect = new ASyncDetectAndRenderThread(this,mDetectResut,ui->openGLWidget->RenderWidth(), ui->openGLWidget->RenderHeight(),