zcy
/
multimedia
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add dynamic detect

master
zcy 2022-05-23 01:20:11 +08:00
parent ed2d2e9e09
commit 1ba89778f5
7 changed files with 146 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
client/webrtc_capture/src/camera_video_sink.cpp
View File

@ -82,14 +82,8 @@ void CameraVideoSink::OnFrame(const webrtc::VideoFrame& frame) {
cnt++;
int width = this->m_capability.width;
int height = this->m_capability.height;
/*
qDebug()<<this->Capability().height<<this->Capability().width
<<int(this->Capability().videoType)
<<int(webrtc::VideoType::kI420)
<<this->Capability().interlaced<<width * height*4;*/
uint8_t *data =
new uint8_t[width * height*4];
uint8_t *data = new uint8_t[width * height*4];
// qDebug()<<width*height<<int(frameBuffer->GetI420()->type());
memcpy(data,frameBuffer->GetI420()->DataY(),width*height);
memcpy(data + width*height ,frameBuffer->GetI420()->DataU(),

2
client/webrtc_capture/src/camera_video_sink.h
View File

@ -2,8 +2,6 @@
#define CAMERA_VIDEO_SINK_H
// vcm_capturer_test.h
#include <memory>
#include <QObject>
#include "modules/video_capture/video_capture.h"

26
client/webrtc_capture/src/cplaywidget.cpp
View File

@ -83,6 +83,28 @@ void CPlayWidget::OnPaintData(const rtc::scoped_refptr<webrtc::I420BufferInterfa
update();
}
CPlayWidget::CPlayWidget(QWidget *parent, IMG_TYPE type)
{
textureUniformY = 0;
textureUniformU = 0;
textureUniformV = 0;
id_y = 0;
id_u = 0;
id_v = 0;
m_pTextureRGB = nullptr;
m_pBufYuv420p = nullptr;
m_pVSHader = NULL;
m_pFSHader = NULL;
m_pShaderProgram = NULL;
m_pTextureY = NULL;
m_pTextureU = NULL;
m_pTextureV = NULL;
m_nVideoH = 0;
m_nVideoW = 0;
mType = type;
m_start_render = false;
}
CPlayWidget::CPlayWidget(QWidget *parent):QOpenGLWidget(parent) {
textureUniformY = 0;
textureUniformU = 0;
@ -100,7 +122,7 @@ CPlayWidget::CPlayWidget(QWidget *parent):QOpenGLWidget(parent) {
m_pTextureV = NULL;
m_nVideoH = 0;
m_nVideoW = 0;
mType = TYPE_I420;
mType = TYPE_YUV420P;
m_start_render = false;
}
@ -138,6 +160,8 @@ void CPlayWidget::OnCameraData( rtc::scoped_refptr<webrtc::I420BufferInterface>
update();
}
int CPlayWidget::OnCameraData(uint8_t *p)
{
memcpy(m_pBufYuv420p,p,m_nVideoH*m_nVideoW/2*3);

2
client/webrtc_capture/src/cplaywidget.h
View File

@ -36,6 +36,8 @@ public:
TYPE_I420,
TYPE_UNSET,
}IMG_TYPE;
CPlayWidget(QWidget* parent,IMG_TYPE type);
CPlayWidget(QWidget* parent);
~CPlayWidget();
int SetDataType(IMG_TYPE);

72
client/webrtc_capture/src/cv_ssd.cpp
View File

@ -18,18 +18,13 @@ cv::Mat * ssd_detect(cv::Mat *inframe) {
printf("could not load image...\n");
return inframe;
}
namedWindow("input image", WINDOW_AUTOSIZE);
cv::Mat resize;
cv::resize(*inframe, resize,
Size(480,560),
0, 0, INTER_LINEAR);// X Y各缩小一半
qDebug()<<"frame.type()"<<resize.type();
cv::Mat image2(480,640,CV_8UC3);
qDebug()<<"image2 ()"<<image2.type()<<CV_8UC3;
cv::cvtColor(resize,image2,COLOR_RGBA2RGB);
imshow("input image", image2);
qDebug()<<"image2 frame.type()"<<image2.type()<<CV_8UC3;
Mat blobImage1 = blobFromImage(image2, 0.007843,
cv::Mat *image2 = new cv::Mat(480,640,CV_8UC3);
cv::cvtColor(resize,*image2,COLOR_RGBA2RGB);
Mat blobImage1 = blobFromImage(*image2, 0.007843,
Size(300, 300),
Scalar(127.5, 127.5, 127.5), true, false);
@ -45,52 +40,7 @@ cv::Mat * ssd_detect(cv::Mat *inframe) {
//// size：这个参数是我们神经网络在训练的时候要求输入的图片尺寸。
//// swapRB：OpenCV中认为我们的图片通道顺序是BGR，但是我平均值假设的顺序是RGB，
/// 所以如果需要交换R和G，那么就要使swapRB=true
// cv::Mat blobImage = blobFromImage(resize,
// 0.007843,
// Size(300, 300),
// Scalar(127.5, 127.5, 127.5), true, false);
// qDebug()<<"blobImage width : " << blobImage.cols
// <<"blobImage.cols: " << blobImage.rows;
// net.setInput(blobImage, "data");
// Mat detection = net.forward("detection_out");
// Mat detectionMat(detection.size[2], detection.size[3], CV_32F, detection.ptr<float>());
// float confidence_threshold = 0.2;
// for (int i = 0; i < detectionMat.rows; i++) {
// float confidence = detectionMat.at<float>(i, 2);
// if (confidence > confidence_threshold) {
// size_t objIndex = (size_t)(detectionMat.at<float>(i, 1));
// float tl_x = detectionMat.at<float>(i, 3) * resize.cols;
// float tl_y = detectionMat.at<float>(i, 4) * resize.rows;
// float br_x = detectionMat.at<float>(i, 5) * resize.cols;
// float br_y = detectionMat.at<float>(i, 6) * resize.rows;
// Rect object_box((int)tl_x, (int)tl_y, (int)(br_x - tl_x), (int)(br_y - tl_y));
// rectangle(resize, object_box, Scalar(0, 0, 255), 2, 8, 0);
// putText(resize, format("%s", objNames[objIndex].c_str()), Point(tl_x, tl_y),
// FONT_HERSHEY_SIMPLEX, 1.0, Scalar(255, 0, 0), 2);
// }
// }
// imshow("ssd-demo", resize);
////// waitKey(0);
// return inframe;
Mat frame = imread("D:/project/opencv_tutorial_data/images/gaoyy.png");
if (frame.empty()) {
printf("could not load image...\n");
return nullptr;
}
qDebug()<<"frame.type()"<<frame.type()<<frame.cols<<frame.rows<<frame.depth()
<<frame.elemSize();
namedWindow("input image", WINDOW_AUTOSIZE);
Net net = readNetFromCaffe(model_text_file, modelFile);
Mat blobImage = blobFromImage(frame, 0.007843,
Size(300, 300),
Scalar(127.5, 127.5, 127.5), true, false);
qDebug()<<"blobImage width "<< blobImage.cols<<blobImage.rows;
net.setInput(blobImage1, "data");
Mat detection = net.forward("detection_out");
Mat detectionMat(detection.size[2], detection.size[3], CV_32F, detection.ptr<float>());
@ -99,19 +49,17 @@ cv::Mat * ssd_detect(cv::Mat *inframe) {
float confidence = detectionMat.at<float>(i, 2);
if (confidence > confidence_threshold) {
size_t objIndex = (size_t)(detectionMat.at<float>(i, 1));
float tl_x = detectionMat.at<float>(i, 3) * image2.cols;
float tl_y = detectionMat.at<float>(i, 4) * image2.rows;
float br_x = detectionMat.at<float>(i, 5) * image2.cols;
float br_y = detectionMat.at<float>(i, 6) * image2.rows;
float tl_x = detectionMat.at<float>(i, 3) * image2->cols;
float tl_y = detectionMat.at<float>(i, 4) * image2->rows;
float br_x = detectionMat.at<float>(i, 5) * image2->cols;
float br_y = detectionMat.at<float>(i, 6) * image2->rows;
Rect object_box((int)tl_x, (int)tl_y, (int)(br_x - tl_x), (int)(br_y - tl_y));
rectangle(image2, object_box, Scalar(0, 0, 255), 2, 8, 0);
putText(image2, format("%s", objNames[objIndex].c_str()), Point(tl_x, tl_y), FONT_HERSHEY_SIMPLEX, 1.0, Scalar(255, 0, 0), 2);
rectangle(*image2, object_box, Scalar(0, 0, 255), 2, 8, 0);
putText(*image2, format("%s", objNames[objIndex].c_str()), Point(tl_x, tl_y), FONT_HERSHEY_SIMPLEX, 1.0, Scalar(255, 0, 0), 2);
}
}
imshow("ssd-demo", image2);
waitKey(0);
return nullptr;
return image2;
}

134
client/webrtc_capture/src/mainwindow.cpp
View File

@ -9,6 +9,76 @@
#include <opencv2/imgproc.hpp>
#include "cvhelper.h"
#include "cv_ssd.h"
#include <list>
#include <mutex>
class ASyncDetectAndRenderThread :public QSSASyncProcess{
public:
typedef enum {
STATUS_STOP = 1,
STATUS_RUNNING = 2
}Status;
ASyncDetectAndRenderThread(QWidget * parent,CPlayWidget *render_ui,
int width,int height){
m_render = render_ui;
m_parent = parent;
m_status = STATUS_RUNNING;
m_width = width;
m_height = height;
}
bool DetectStatus(){
if(m_mat.size() > 0)
return true;
else
return false;
}
void SetCvImage(uint8_t *frame){
cv::Mat yuv420p;
yuv420p.create(m_height*3/2,
m_width, CV_8UC1);
memcpy(yuv420p.data, frame, m_height*m_width
*sizeof(unsigned char)*3/2);
cv::Mat *rgbImg = new cv::Mat;
cv::cvtColor(yuv420p, *rgbImg, cv::COLOR_YUV2BGR_I420);
std::lock_guard<std::mutex> guard(this->m_mutex);
// imshow("yuv420",yuv420p);
m_mat.push_back(rgbImg);
}
void Run(void *) override{
while(m_status == STATUS_RUNNING){
cv::Mat *c = takeLast();
if(nullptr != c){
cv::Mat *result = ssd_detect(c);
qDebug()<<result->cols<<result->rows<<result->type();
cv::Mat yuvData;
cv::cvtColor(*result, yuvData, cv::COLOR_BGR2YUV_I420);
this->m_render->OnCameraData(yuvData.data);
}
}
}
private:
cv::Mat *takeLast(){
std::lock_guard<std::mutex> guard(this->m_mutex);
if(m_mat.size() != 0){
auto ret = *m_mat.begin();
m_mat.pop_front();
return ret;
}
return nullptr;
}
std::list<cv::Mat*> m_mat;
std::mutex m_mutex;
CPlayWidget *m_render;
QWidget *m_parent;
Status m_status;
int m_width;
int m_height;
};
ASyncDetectAndRenderThread *gdetect = nullptr;
class AsyncRennder :public QSSASyncProcess{
public:
@ -32,27 +102,13 @@ public:
while(state){
if(mfbs ->Size() > 0){
uint8_t *frame = this->mfbs->TakeLast();
cv::Mat yuv420p;
yuv420p.create(mUI->RenderHeight()*3/2,
mUI->RenderWidth(), CV_8UC1);
memcpy(yuv420p.data, frame, mUI->RenderHeight()*mUI->RenderWidth()
*sizeof(unsigned char)*3/2);
cv::Mat rgbImg;
cv::cvtColor(yuv420p, rgbImg, cv::COLOR_YUV2BGR_I420);
// cv::Mat dst;
// cv::rotate(rgbImg, dst, cv::ROTATE_90_CLOCKWISE); // 绕x翻转，上下颠倒
// cv::imshow("img", dst);
//// cv::Mat gray, dst;
//// cvtColor(src, gray, COLOR_BGR2GRAY);
//// imshow("input", gray);
//// equalizeHist(gray, dst);
//// imshow("eq", dst);
// cv::waitKey(1);
uint8_t *frame = mfbs->TakeLast();
mUI->OnCameraData(frame);
// qDebug()<<"dst size is "<<dst.size().width<<dst.size().height;
if(gdetect != nullptr){
if(!gdetect->DetectStatus()){
gdetect->SetCvImage(frame);
}
}
eventLoop.exec(); // 渲染一次
delete frame;
}
@ -64,7 +120,9 @@ private:
CPlayWidget *mUI;
};
AsyncRennder *gRender;
AsyncRennder *gRender = nullptr;
MainWindow::MainWindow(QWidget *parent)
: QssMainWindow(parent)
@ -92,6 +150,9 @@ MainWindow::MainWindow(QWidget *parent)
origin_picture->setMaximumHeight(12000);
ui->gridLayout->addWidget(processed_picture,1,0);
mDetectResut = new CPlayWidget(this);
ui->gridLayout->addWidget(mDetectResut,1,1);
}
MainWindow::~MainWindow()
@ -106,6 +167,8 @@ void MainWindow::OnUpdateFrame( rtc::scoped_refptr<webrtc::I420BufferInterface>&
void MainWindow::on_pushButton_clicked()
{
int id = ui->comboBox->currentData().toInt();
webrtc::VideoCaptureCapability p;
@ -131,6 +194,10 @@ void MainWindow::on_pushButton_clicked()
gRender->Start(this);
connect(gRender,&QSSASyncProcess::Done,this,&MainWindow::RenderDone);
}
mDetectResut->SetImgSize(480,560);
mDetectResut->StartRender();
}
void MainWindow::RenderDone()
@ -154,14 +221,15 @@ cv::Mat *QImage2cvMat(QImage image)
memcpy(mat->data,image.bits(),image.bytesPerLine()*image.height());
break;
case QImage::Format_RGB888:
mat = new cv::Mat(image.height(), image.width(), CV_8UC3, (void*)image.bits(), image.bytesPerLine());
mat = new cv::Mat(image.height(), image.width(), CV_8UC3,
(void*)image.bits(), image.bytesPerLine());
break;
case QImage::Format_Indexed8:
mat = new cv::Mat(image.height(), image.width(), CV_8UC1, (void*)image.bits(), image.bytesPerLine());
mat = new cv::Mat(image.height(), image.width(), CV_8UC1,
(void*)image.bits(), image.bytesPerLine());
break;
}
Save1BitImage(*mat,"d://tgest.png");
return mat;
}
@ -181,13 +249,19 @@ void MainWindow::on_pushButton_3_clicked()
cv::Mat pic = *QImage2cvMat(image);
cv::Mat *result = ssd_detect(&pic);
// try{
//// cv::imshow("img", *result);
// }catch (std::exception &e){
// qDebug()<<e.what();
// }
// cv::waitKey(1);
this->processed_picture->setPixmap(QPixmap::fromImage(image));
}
}
void MainWindow::on_pushButton_4_clicked()
{
if(gdetect == nullptr){
gdetect = new ASyncDetectAndRenderThread(this,mDetectResut,ui->openGLWidget->RenderWidth(),
ui->openGLWidget->RenderHeight());
gdetect->Start(this);
connect(gdetect,&QSSASyncProcess::Done,this,&MainWindow::RenderDone);
}
}

4
client/webrtc_capture/src/mainwindow.h
View File

@ -4,6 +4,7 @@
#include "rtc.h"
#include "api/video/i420_buffer.h"
#include "Qss.h"
#include "cplaywidget.h"
QT_BEGIN_NAMESPACE
namespace Ui { class MainWindow; }
@ -25,10 +26,13 @@ private slots:
void on_pushButton_3_clicked();
void on_pushButton_4_clicked();
private:
Ui::MainWindow *ui;
std::unique_ptr<CameraVideoSink> m_capturer;
QLabel *origin_picture;
QLabel *processed_picture;
CPlayWidget *mDetectResut;
};
#endif // MAINWINDOW_H