zcy
/
multimedia
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

no message

master
zcy 2022-05-19 18:01:35 +08:00
parent 7f6d765d28
commit f748b3ded8
9 changed files with 332 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
client/webrtc_capture/src/cv_ssd.cpp Normal file
View File

@ -0,0 +1,64 @@
#include "cv_ssd.h"
#include <QDebug>
const size_t width = 1280;
const size_t height = 720;
String labelFile = "ssd/labelmap_det.txt";
String modelFile = "D:/project/multimedia/client/build-webrtc_capture-Desktop_Qt_5_15_2_MSVC2019_64bit-Debug/debug/ssd/MobileNetSSD_deploy.caffemodel";
String model_text_file = "D:/project/multimedia/client/build-webrtc_capture-Desktop_Qt_5_15_2_MSVC2019_64bit-Debug/debug/ssd/MobileNetSSD_deploy.prototxt";
String objNames[] = { "background",
"aeroplane", "bicycle", "bird", "boat",
"bottle", "bus", "car", "cat", "chair",
"cow", "diningtable", "dog", "horse",
"motorbike", "person", "pottedplant",
"sheep", "sofa", "train", "tvmonitor" };
cv::Mat * ssd_detect(cv::Mat *inframe) {
if (inframe->empty()) {
printf("could not load image...\n");
return inframe;
}
namedWindow("input image", WINDOW_AUTOSIZE);
imshow("input image", *inframe);
Net net = readNetFromCaffe(model_text_file, modelFile);
// image：这个就是我们将要输入神经网络进行处理或者分类的图片。
// mean：需要将图片整体减去的平均值，如果我们需要对RGB图片的三个通道分别减去不同的值，那么可以使用3组平均值，如果只使用一组，那么就默认对三个通道减去一样的值。减去平均值（mean）：为了消除同一场景下不同光照的图片，对我们最终的分类或者神经网络的影响，我们常常对图片的R、G、B通道的像素求一个平均值，然后将每个像素值减去我们的平均值，这样就可以得到像素之间的相对值，就可以排除光照的影响。
// scalefactor：当我们将图片减去平均值之后，还可以对剩下的像素值进行一定的尺度缩放，它的默认值是1，如果希望减去平均像素之后的值，全部缩小一半，那么可以将scalefactor设为1/2。
// size：这个参数是我们神经网络在训练的时候要求输入的图片尺寸。
// swapRB：OpenCV中认为我们的图片通道顺序是BGR，但是我平均值假设的顺序是RGB，所以如果需要交换R和G，那么就要使swapRB=true
Mat blobImage = blobFromImage(*inframe, 0.007843,
Size(300, 300),
Scalar(127.5, 127.5, 127.5), true, false);
qDebug()<<"blobImage width : " << blobImage.cols
<<"blobImage.cols: " << blobImage.rows;
imshow("ssd-demo", blobImage);
// net.setInput(blobImage, "data");
// Mat detection = net.forward("detection_out");
// Mat detectionMat(detection.size[2], detection.size[3], CV_32F, detection.ptr<float>());
// float confidence_threshold = 0.2;
// for (int i = 0; i < detectionMat.rows; i++) {
// float confidence = detectionMat.at<float>(i, 2);
// if (confidence > confidence_threshold) {
// size_t objIndex = (size_t)(detectionMat.at<float>(i, 1));
// float tl_x = detectionMat.at<float>(i, 3) * frame->cols;
// float tl_y = detectionMat.at<float>(i, 4) * frame->rows;
// float br_x = detectionMat.at<float>(i, 5) * frame->cols;
// float br_y = detectionMat.at<float>(i, 6) * frame->rows;
// Rect object_box((int)tl_x, (int)tl_y, (int)(br_x - tl_x), (int)(br_y - tl_y));
// rectangle(*frame, object_box, Scalar(0, 0, 255), 2, 8, 0);
// putText(*frame, format("%s", objNames[objIndex].c_str()), Point(tl_x, tl_y),
// FONT_HERSHEY_SIMPLEX, 1.0, Scalar(255, 0, 0), 2);
// }
// }
// waitKey(0);
return inframe;
}

17
client/webrtc_capture/src/cv_ssd.h Normal file
View File

@ -0,0 +1,17 @@
#ifndef CV_SSD_H
#define CV_SSD_H
#include <opencv2/opencv.hpp>
#include <opencv2/dnn.hpp>
#include <iostream>
using namespace cv;
using namespace cv::dnn;
using namespace std;
cv::Mat * ssd_detect(cv::Mat*);
#endif // CV_SSD_H

168
client/webrtc_capture/src/cv_yolo.cpp Normal file
View File

@ -0,0 +1,168 @@
#include "cv_yolo.h"
float confidenceThreshold = 0.25;
void test_yolo()
{
image_detection();
}
void video_detection() {
String modelConfiguration = "D:/vcprojects/images/dnn/yolov2-tiny-voc/yolov2-tiny-voc.cfg";
String modelBinary = "D:/vcprojects/images/dnn/yolov2-tiny-voc/yolov2-tiny-voc.weights";
dnn::Net net = readNetFromDarknet(modelConfiguration, modelBinary);
if (net.empty())
{
printf("Could not load net...\n");
return;
}
vector<string> classNamesVec;
ifstream classNamesFile("D:/vcprojects/images/dnn/yolov2-tiny-voc/voc.names");
if (classNamesFile.is_open())
{
string className = "";
while (std::getline(classNamesFile, className))
classNamesVec.push_back(className);
}
// VideoCapture capture(0);
VideoCapture capture;
capture.open("D:/vcprojects/images/fbb.avi");
if (!capture.isOpened()) {
printf("could not open the camera...\n");
return;
}
Mat frame;
while (capture.read(frame))
{
if (frame.empty())
if (frame.channels() == 4)
cvtColor(frame, frame, COLOR_BGRA2BGR);
Mat inputBlob = blobFromImage(frame, 1 / 255.F, Size(416, 416), Scalar(), true, false);
net.setInput(inputBlob, "data");
Mat detectionMat = net.forward("detection_out");
vector<double> layersTimings;
double freq = getTickFrequency() / 1000;
double time = net.getPerfProfile(layersTimings) / freq;
ostringstream ss;
ss << "FPS: " << 1000 / time << " ; time: " << time << " ms";
putText(frame, ss.str(), Point(20, 20), 0, 0.5, Scalar(0, 0, 255));
for (int i = 0; i < detectionMat.rows; i++)
{
const int probability_index = 5;
const int probability_size = detectionMat.cols - probability_index;
float *prob_array_ptr = &detectionMat.at<float>(i, probability_index);
size_t objectClass = max_element(prob_array_ptr, prob_array_ptr + probability_size) - prob_array_ptr;
float confidence = detectionMat.at<float>(i, (int)objectClass + probability_index);
if (confidence > confidenceThreshold)
{
float x = detectionMat.at<float>(i, 0);
float y = detectionMat.at<float>(i, 1);
float width = detectionMat.at<float>(i, 2);
float height = detectionMat.at<float>(i, 3);
int xLeftBottom = static_cast<int>((x - width / 2) * frame.cols);
int yLeftBottom = static_cast<int>((y - height / 2) * frame.rows);
int xRightTop = static_cast<int>((x + width / 2) * frame.cols);
int yRightTop = static_cast<int>((y + height / 2) * frame.rows);
Rect object(xLeftBottom, yLeftBottom,
xRightTop - xLeftBottom,
yRightTop - yLeftBottom);
rectangle(frame, object, Scalar(0, 255, 0));
if (objectClass < classNamesVec.size())
{
ss.str("");
ss << confidence;
String conf(ss.str());
String label = String(classNamesVec[objectClass]) + ": " + conf;
int baseLine = 0;
Size labelSize = getTextSize(label, FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine);
rectangle(frame, Rect(Point(xLeftBottom, yLeftBottom),
Size(labelSize.width, labelSize.height + baseLine)),
Scalar(255, 255, 255), FILLED);
putText(frame, label, Point(xLeftBottom, yLeftBottom + labelSize.height),
FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 0, 0));
}
}
}
imshow("YOLOv3: Detections", frame);
if (waitKey(1) >= 0) break;
}
}
void image_detection() {
String modelConfiguration = "D:/vcprojects/images/dnn/yolov2-tiny-voc/yolov2-tiny-voc.cfg";
String modelBinary = "D:/vcprojects/images/dnn/yolov2-tiny-voc/yolov2-tiny-voc.weights";
dnn::Net net = readNetFromDarknet(modelConfiguration, modelBinary);
if (net.empty())
{
printf("Could not load net...\n");
return;
}
vector<string> classNamesVec;
ifstream classNamesFile("D:/vcprojects/images/dnn/yolov2-tiny-voc/voc.names");
if (classNamesFile.is_open())
{
string className = "";
while (std::getline(classNamesFile, className))
classNamesVec.push_back(className);
}
// ͼ
Mat frame = imread("D:/vcprojects/images/fastrcnn.jpg");
Mat inputBlob = blobFromImage(frame, 1 / 255.F, Size(416, 416), Scalar(), true, false);
net.setInput(inputBlob, "data");
//
Mat detectionMat = net.forward("detection_out");
vector<double> layersTimings;
double freq = getTickFrequency() / 1000;
double time = net.getPerfProfile(layersTimings) / freq;
ostringstream ss;
ss << "detection time: " << time << " ms";
putText(frame, ss.str(), Point(20, 20), 0, 0.5, Scalar(0, 0, 255));
//
for (int i = 0; i < detectionMat.rows; i++)
{
const int probability_index = 5;
const int probability_size = detectionMat.cols - probability_index;
float *prob_array_ptr = &detectionMat.at<float>(i, probability_index);
size_t objectClass = max_element(prob_array_ptr, prob_array_ptr + probability_size) - prob_array_ptr;
float confidence = detectionMat.at<float>(i, (int)objectClass + probability_index);
if (confidence > confidenceThreshold)
{
float x = detectionMat.at<float>(i, 0);
float y = detectionMat.at<float>(i, 1);
float width = detectionMat.at<float>(i, 2);
float height = detectionMat.at<float>(i, 3);
int xLeftBottom = static_cast<int>((x - width / 2) * frame.cols);
int yLeftBottom = static_cast<int>((y - height / 2) * frame.rows);
int xRightTop = static_cast<int>((x + width / 2) * frame.cols);
int yRightTop = static_cast<int>((y + height / 2) * frame.rows);
Rect object(xLeftBottom, yLeftBottom,
xRightTop - xLeftBottom,
yRightTop - yLeftBottom);
rectangle(frame, object, Scalar(0, 0, 255), 2, 8);
if (objectClass < classNamesVec.size())
{
ss.str("");
ss << confidence;
String conf(ss.str());
String label = String(classNamesVec[objectClass]) + ": " + conf;
int baseLine = 0;
Size labelSize = getTextSize(label, FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine);
rectangle(frame, Rect(Point(xLeftBottom, yLeftBottom),
Size(labelSize.width, labelSize.height + baseLine)),
Scalar(255, 255, 255), FILLED);
putText(frame, label, Point(xLeftBottom, yLeftBottom + labelSize.height),
FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 0, 0));
}
}
}
imshow("YOLO-Detections", frame);
waitKey(0);
return;
}

19
client/webrtc_capture/src/cv_yolo.h Normal file
View File

@ -0,0 +1,19 @@
#ifndef CV_YOLO_H
#define CV_YOLO_H
#include <opencv2/opencv.hpp>
#include <opencv2/dnn.hpp>
#include <fstream>
#include <iostream>
#include <algorithm>
#include <cstdlib>
using namespace std;
using namespace cv;
using namespace cv::dnn;
void video_detection();
void image_detection();
void test_yolo();
#endif // CV_YOLO_H

28
client/webrtc_capture/src/main.cpp
View File

@ -1,7 +1,7 @@
#include "mainwindow.h"
#include "rtc.h"
#include <QApplication>
#include "cv_yolo.h"
#include "modules/video_capture/video_capture.h"
#include "video_source_impl.h"
#include <QString>
@ -11,6 +11,7 @@
#include "camera_video_sink.h"
#include "video_source_impl.h"
#include <QMetaType>
#include "cv_ssd.h"
# pragma comment(lib, "secur32.lib")
@ -21,6 +22,21 @@
# pragma comment(lib, "Strmiids.lib")
# pragma comment(lib, "User32.lib")
int BubbleSort(int *p,int len){
if(nullptr == p)
return -1;
for(int i = 0;i < len - 1;i++){
for(int j = 0;j < len - 1;j++){
if(p[j] < p[j + 1]){
int tmp = p[j];
p[j] = p[j+1];
p[j+1] = tmp;
}
}
}
}
void EnumCapture()
{
rtc::WinsockInitializer winsock_init;
@ -29,7 +45,6 @@ void EnumCapture()
rtc::ThreadManager::Instance()->SetCurrentThread(&w32_thread);
rtc::InitializeSSL();
std::unique_ptr<webrtc::VideoCaptureModule::DeviceInfo> info(
webrtc::VideoCaptureFactory::CreateDeviceInfo());
@ -56,6 +71,11 @@ void EnumCapture()
int main(int argc, char *argv[])
{
int p[5] = {3,1,49,23,23};
BubbleSort(p,5);
for(int i = 0;i< 5;i++){
qDebug()<<p[i]<<" ";
}
qRegisterMetaType<rtc::scoped_refptr<webrtc::I420BufferInterface>>("rtc::scoped_refptr<webrtc::I420BufferInterface>");
qRegisterMetaType<rtc::scoped_refptr<webrtc::I420BufferInterface>>("rtc::scoped_refptr<webrtc::I420BufferInterface>&");
@ -73,3 +93,7 @@ int main(int argc, char *argv[])
w.show();
return a.exec();
}

34
client/webrtc_capture/src/mainwindow.cpp
View File

@ -8,6 +8,7 @@
#include <opencv2/highgui.hpp>
#include <opencv2/imgproc.hpp>
#include "cvhelper.h"
#include "cv_ssd.h"
class AsyncRennder :public QSSASyncProcess{
public:
@ -168,20 +169,25 @@ cv::Mat *QImage2cvMat(QImage image)
void MainWindow::on_pushButton_2_clicked()
{
QPixmap pix = ui->openGLWidget->grab();
QImage image = pix.toImage();
image = image.scaled(1280,720);
origin_picture->setPixmap(pix);
qDebug()<<"format is "<<image.format();
// cv::Mat Img;
cv::Mat pic = *QImage2cvMat(image);
try{
cv::imshow("img", pic);
}catch (std::exception &e){
qDebug()<<e.what();
}
cv::waitKey(1);
this->processed_picture->setPixmap(QPixmap::fromImage(image));
}
// 识别
void MainWindow::on_pushButton_3_clicked()
{
if(nullptr != origin_picture->pixmap()){
QImage image = origin_picture->pixmap()->toImage();
image = image.scaled(1280,720);
cv::Mat pic = *QImage2cvMat(image);
cv::Mat *result = ssd_detect(&pic);
// try{
//// cv::imshow("img", *result);
// }catch (std::exception &e){
// qDebug()<<e.what();
// }
// cv::waitKey(1);
this->processed_picture->setPixmap(QPixmap::fromImage(image));
}
}

2
client/webrtc_capture/src/mainwindow.h
View File

@ -23,6 +23,8 @@ private slots:
void RenderDone();
void on_pushButton_2_clicked();
void on_pushButton_3_clicked();
private:
Ui::MainWindow *ui;
std::unique_ptr<CameraVideoSink> m_capturer;

9
client/webrtc_capture/src/mainwindow.ui
View File

@ -22,7 +22,7 @@
<widget class="QWidget" name="centralwidget">
<layout class="QVBoxLayout" name="verticalLayout_2" stretch="1,9">
<item>
<layout class="QHBoxLayout" name="horizontalLayout" stretch="1,1,1,1,1,0,3">
<layout class="QHBoxLayout" name="horizontalLayout" stretch="1,1,1,1,1,0,0,3">
<item>
<widget class="QLabel" name="label">
<property name="text">
@ -63,6 +63,13 @@
</property>
</widget>
</item>
<item>
<widget class="QPushButton" name="pushButton_3">
<property name="text">
<string>检测</string>
</property>
</widget>
</item>
<item>
<spacer name="horizontalSpacer">
<property name="orientation">

10
client/webrtc_capture/webrtc_capture.pro
View File

@ -23,6 +23,8 @@ INCLUDEPATH += third/include/
SOURCES += \
src/camera_video_sink.cpp \
src/cplaywidget.cpp \
src/cv_ssd.cpp \
src/cv_yolo.cpp \
src/cvhelper.cpp \
src/main.cpp \
src/mainwindow.cpp \
@ -31,6 +33,8 @@ SOURCES += \
HEADERS += \
src/camera_video_sink.h \
src/cplaywidget.h \
src/cv_ssd.h \
src/cv_yolo.h \
src/cvhelper.h \
src/mainwindow.h \
src/rtc.h \
@ -46,8 +50,10 @@ else: unix:!android: target.path = /opt/$${TARGET}/bin
CONFIG(debug, debug|release){
message("debug mode")
LIBS += -L$$PWD/third/lib libwebrtc.lib ole32.lib oleaut32.lib strmiids.lib advapi32.lib opencv_core455d.lib
LIBS +=opencv_stitching455d.lib opencv_objdetect455d.lib opencv_ml455d.lib opencv_imgcodecs455d.lib opencv_imgproc455d.lib
LIBS += -L$$PWD/third/lib libwebrtc.lib ole32.lib oleaut32.lib strmiids.lib
LIBS += advapi32.lib opencv_core455d.lib
LIBS += opencv_stitching455d.lib opencv_objdetect455d.lib opencv_videoio455d.lib
LIBS += opencv_ml455d.lib opencv_imgcodecs455d.lib opencv_imgproc455d.lib opencv_dnn455d.lib
LIBS+= opencv_highgui455d.lib
DEFINES += DEBUG_FLAG
Qt += debug