/*
* Copyright (C) 2021 刘臣轩
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see .
*/
#include "widget.h"
#include "ui_widget.h"
Widget::Widget(QWidget* parent)
: QWidget(parent)
, ui(new Ui::Widget)
{
ui->setupUi(this);
connect(ui->pushButton, SIGNAL(clicked()), this, SLOT(close()));
ui->frame->setStyleSheet("#frame {border-image: url(:/new/prefix1/image/主.png);}");
ui->textEdit->append("开始初始化设备");
// Time
QTimer* timer = new QTimer(this);
connect(timer, SIGNAL(timeout()), this, SLOT(timerUpdate()));
timer->start(500);
initSerial();
#ifdef Q_OS_WIN
initCamera();
#else
connect(this, SIGNAL(imageCaptured(int, QImage)), this, SLOT(onImageCaptured(int, QImage)));
ui->textEdit->append("摄像头初始化成功");
#endif
// Network
networkManager = new QNetworkAccessManager();
networkRequest = new QNetworkRequest();
networkRequest->setHeader(QNetworkRequest::ContentTypeHeader, " application/json;charset=UTF-8");
url = new QUrl("https://aiapi.jd.com/jdai/garbageImageSearch");
connect(networkManager, SIGNAL(finished(QNetworkReply*)), this, SLOT(onRequestFinished(QNetworkReply*)));
ui->label_3->setText("工训大赛");
ui->label_4->setStyleSheet("border-image: url(:/new/prefix1/image/工训大赛.png);\nborder-radius: 10px;\n");
ui->label_4->setVisible(true);
ui->label_5->setVisible(false);
ui->textEdit->append("设备初始化成功√");
number = 0;
// Video
player = new QMediaPlayer;
videoWidget = new QVideoWidget(this);
playList = new QMediaPlaylist;
#ifdef Q_OS_WIN
playList->addMedia(QUrl::fromLocalFile("../WasteSorting/test.mp4"));
#else
playList->addMedia(QUrl::fromLocalFile("/home/pi/WasteSorting/test.mp4"));
#endif
playList->setPlaybackMode(QMediaPlaylist::CurrentItemInLoop);
player->setPlaylist(playList);
player->setVideoOutput(videoWidget);
ui->verticalLayout->addWidget(videoWidget);
videoWidget->setVisible(false);
videoTimer = new QTimer(this);
connect(videoTimer, SIGNAL(timeout()), this, SLOT(videoTimerUpdate()));
videoTimer->setSingleShot(true);
videoTimer->start(10000);
#ifdef Q_OS_WIN
#else
// Tensorflow
model = tflite::FlatBufferModel::BuildFromFile(model_file.c_str());
tflite::InterpreterBuilder(*model, resolver)(&interpreter);
interpreter->SetNumThreads(4);
interpreter->AllocateTensors();
input_tensor = interpreter->tensor(interpreter->inputs()[0]);
TfLiteIntArray* output_dims = interpreter->tensor(interpreter->outputs()[0])->dims;
output_size = output_dims->data[output_dims->size - 1];
#endif
//captureImage();
}
Widget::~Widget()
{
delete ui;
}
void Widget::timerUpdate()
{
QString str = QDateTime::currentDateTime().toString("yyyy年MM月dd日 hh:mm:ss");
ui->label->setText(str);
}
void Widget::videoTimerUpdate()
{
ui->label_4->setVisible(false);
player->play();
videoWidget->setVisible(true);
ui->label_3->setText("播放视频");
ui->textEdit->append("播放视频");
}
void Widget::initSerial()
{
ui->textEdit->append("开始初始化串口");
serialPort = new QSerialPort();
connect(serialPort, SIGNAL(readyRead()), this, SLOT(serialRead()));
if (QSerialPortInfo::availablePorts().length() == 0) {
QMessageBox::critical(this, "错误", "无可用串口设备,请检查硬件连接后重试");
exit(0);
}
#ifdef Q_OS_WIN
QString portName = QSerialPortInfo::availablePorts()[1].portName();
qDebug() << portName;
ui->textEdit->append("尝试连接串口" + portName);
serialPort->setPortName(portName);
#else
QString portName = "ttyUSB0";
ui->textEdit->append("尝试连接串口ttyUSB0");
serialPort->setPortName("ttyUSB0");
#endif
if (serialPort->open(QIODevice::ReadWrite)) {
ui->textEdit->append("串口连接成功");
serialPort->setBaudRate(115200);
serialPort->setDataBits(QSerialPort::Data8);
serialPort->setParity(QSerialPort::NoParity);
serialPort->setStopBits(QSerialPort::OneStop);
} else {
QMessageBox::critical(this, "错误", "串口设备" + portName + "无法打开,请检查硬件连接后重试");
exit(0);
}
ui->textEdit->append("串口初始化成功");
serialWrite('\xCC');
}
void Widget::initCamera()
{
// for (QCameraInfo& info : QCameraInfo::availableCameras()) {
// qDebug() << info.deviceName();
// }
int camNum = QCameraInfo::availableCameras().length();
if (!camNum) {
QMessageBox::critical(this, "错误", "无可用摄像头,请检查硬件连接后重试");
exit(0);
}
ui->textEdit->append("开始初始化摄像头");
camera = new QCamera(QCameraInfo::availableCameras()[0], this);
imageCapture = new QCameraImageCapture(camera);
connect(imageCapture, SIGNAL(imageCaptured(int, QImage)), this, SLOT(onImageCaptured(int, QImage)));
camera->setCaptureMode(QCamera::CaptureStillImage);
imageCapture->setCaptureDestination(QCameraImageCapture::CaptureToBuffer);
camera->start();
ui->textEdit->append("摄像头初始化成功");
}
void Widget::serialRead()
{
QByteArray buffer = serialPort->readAll();
// qDebug() << buffer;
if (buffer[0] == '\x03' && buffer[1] == '\xFC'
&& buffer[3] == '\xFC' && buffer[4] == '\x03') {
switch (buffer[2]) {
case '\x00':
ui->textEdit->append("取消警报");
ui->label_3->setText("取消警报");
ui->label_4->setVisible(true);
ui->label_5->setVisible(false);
ui->frame->setStyleSheet("#frame {border-image: url(:/new/prefix1/image/主.png);}");
videoTimer->start(10000);
break;
case '\x01':
ui->textEdit->append("触发拍照信号");
ui->label_3->setText("触发拍照");
videoTimer->stop();
videoWidget->setVisible(false);
player->stop();
#ifdef Q_OS_WIN
imageCapture->capture();
#else
captureImage();
#endif
break;
case '\x02':
ui->textEdit->append("投递完毕");
ui->label_3->setText("投递完毕");
ui->label_4->setVisible(true);
ui->label_5->setVisible(false);
ui->frame->setStyleSheet("#frame {border-image: url(:/new/prefix1/image/主.png);}");
videoTimer->start(10000);
break;
case '\x04':
ui->textEdit->append("满载警报");
ui->label_3->setText("满载警报");
ui->label_4->setVisible(false);
videoTimer->stop();
videoWidget->setVisible(false);
player->stop();
ui->frame->setStyleSheet("#frame {border-image: url(:/new/prefix1/image/满载警报.png);}");
break;
case '\x08':
// qDebug() << "倾倒警报";
ui->textEdit->append("倾倒警报");
ui->label_3->setText("倾倒警报");
ui->label_4->setVisible(false);
videoTimer->stop();
videoWidget->setVisible(false);
player->stop();
ui->frame->setStyleSheet("#frame {border-image: url(:/new/prefix1/image/倾倒警报.png);}");
break;
case '\xFF':
break;
}
}
}
void Widget::serialWrite(const char data)
{
QByteArray buffer("\x30\xCF\x0F\xCF\x30");
buffer[2] = data;
serialPort->write(buffer);
}
QImage Widget::cvMat2QImage(const cv::Mat& mat)
{
// 8-bits unsigned, NO. OF CHANNELS = 1
if(mat.type() == CV_8UC1)
{
QImage image(mat.cols, mat.rows, QImage::Format_Indexed8);
// Set the color table (used to translate colour indexes to qRgb values)
image.setColorCount(256);
for(int i = 0; i < 256; i++)
{
image.setColor(i, qRgb(i, i, i));
}
// Copy input Mat
uchar *pSrc = mat.data;
for(int row = 0; row < mat.rows; row ++)
{
uchar *pDest = image.scanLine(row);
memcpy(pDest, pSrc, mat.cols);
pSrc += mat.step;
}
return image;
}
// 8-bits unsigned, NO. OF CHANNELS = 3
else if(mat.type() == CV_8UC3)
{
// Copy input Mat
const uchar *pSrc = (const uchar*)mat.data;
// Create QImage with same dimensions as input Mat
QImage image(pSrc, mat.cols, mat.rows, mat.step, QImage::Format_RGB888);
return image.rgbSwapped();
}
else if(mat.type() == CV_8UC4)
{
qDebug() << "CV_8UC4";
// Copy input Mat
const uchar *pSrc = (const uchar*)mat.data;
// Create QImage with same dimensions as input Mat
QImage image(pSrc, mat.cols, mat.rows, mat.step, QImage::Format_ARGB32);
return image.copy();
}
else
{
qDebug() << "ERROR: Mat could not be converted to QImage.";
return QImage();
}
}
void Widget::captureImage()
{
// system("raspistill -o ../WasteSorting/WasteSorting.jpg -t 1 -br 60 -hf -awb sun");
// system("python3 ../WasteSorting/capture.py");
//system("rm -rf /home/pi/WasteSorting/WasteSorting.jpg");
capture = cv::VideoCapture(0);
cv::Mat frame;
capture >> frame;
QImage image = cvMat2QImage(frame);
capture.release();
//cv::imwrite("/home/pi/WasteSorting/WasteSorting.jpg", frame);
//QImage image("../WasteSorting/WasteSorting.jpg");
emit(imageCaptured(0, image));
}
void Widget::onImageCaptured(int, QImage image)
{
// 显示图片
ui->label_4->setVisible(false);
ui->label_5->setPixmap(QPixmap::fromImage(image).scaled(405, 306));
ui->label_5->setVisible(true);
ui->label_3->setText("识别中");
ui->frame->setStyleSheet("#frame {border-image: url(:/new/prefix1/image/识别中.png);}");
/* 使用京东垃圾识别 API
// Base64编码
QByteArray ba;
QBuffer buf(&ba);
image.save(&buf, "JPG", -1);
QByteArray imageBase64 = ba.toBase64();
// qDebug() << "data:image/jpg;base64," + imageBase64;
// Request
sendRequest(imageBase64);
*/
/* TensorFlow Lite Python
#ifdef Q_OS_WIN
image.save("../WasteSorting/WasteSorting.jpg");
FILE* fp = _popen("python ../WasteSorting/tensorflow/label_image.py", "rt");
char buf[255] = { 0 };
fscanf(fp, "%s", buf);
_pclose(fp);
#else
FILE* fp = popen("python3 ../WasteSorting/tensorflow/label_image.py", "r");
char buf[255] = { 0 };
printf("%s", buf);
fscanf(fp, "%s", buf);
pclose(fp);
#endif
QString cate_name = QString::fromLocal8Bit(buf);
qDebug() << cate_name;
classifyFinished(cate_name);
*/
/* Tensorflow Lite C++ */
image.save("../WasteSorting/WasteSorting.jpg");
image = image.convertToFormat(QImage::Format_RGB888).mirrored(true, false);
formatImageTFLite(interpreter->typed_tensor(interpreter->inputs()[0]), image.bits(),
image.height(), image.width(), 3, 224, 224, 3, false);
interpreter->Invoke();
std::vector> top_results;
get_top_n(interpreter->typed_output_tensor(0),
output_size, 1, 0.01f, &top_results, kTfLiteUInt8);
int index = top_results[0].second;
QString cate_name;
switch(index) {
case 0:
cate_name = "识别失败";
break;
case 1:case 2:case 10:
cate_name = "有害垃圾";
break;
case 3:case 4:case 5:case 11:
cate_name = "可回收垃圾";
break;
case 6:case 7:case 12:case 13:
cate_name = "厨余垃圾";
break;
case 8:case 9:
cate_name = "其他垃圾";
break;
}
qDebug() << cate_name;
classifyFinished(cate_name);
}
template
void Widget::formatImageTFLite(T* out, const uint8_t* in, int image_height, int image_width, int image_channels, int wanted_height, int wanted_width, int wanted_channels, bool input_floating)
{
const float input_mean = 127.5f;
const float input_std = 127.5f;
int number_of_pixels = image_height * image_width * image_channels;
std::unique_ptr interpreter(new tflite::Interpreter);
int base_index = 0;
// two inputs: input and new_sizes
interpreter->AddTensors(2, &base_index);
// one output
interpreter->AddTensors(1, &base_index);
// set input and output tensors
interpreter->SetInputs({0, 1});
interpreter->SetOutputs({2});
// set parameters of tensors
TfLiteQuantizationParams quant;
interpreter->SetTensorParametersReadWrite(0, kTfLiteFloat32, "input", {1, image_height, image_width, image_channels}, quant);
interpreter->SetTensorParametersReadWrite(1, kTfLiteInt32, "new_size", {2},quant);
interpreter->SetTensorParametersReadWrite(2, kTfLiteFloat32, "output", {1, wanted_height, wanted_width, wanted_channels}, quant);
tflite::ops::builtin::BuiltinOpResolver resolver;
const TfLiteRegistration *resize_op = resolver.FindOp(tflite::BuiltinOperator_RESIZE_BILINEAR,1);
auto* params = reinterpret_cast(malloc(sizeof(TfLiteResizeBilinearParams)));
params->align_corners = false;
interpreter->AddNodeWithParameters({0, 1}, {2}, nullptr, 0, params, resize_op, nullptr);
interpreter->AllocateTensors();
// fill input image
// in[] are integers, cannot do memcpy() directly
auto input = interpreter->typed_tensor(0);
for (int i = 0; i < number_of_pixels; i++)
input[i] = in[i];
// fill new_sizes
interpreter->typed_tensor(1)[0] = wanted_height;
interpreter->typed_tensor(1)[1] = wanted_width;
interpreter->Invoke();
auto output = interpreter->typed_tensor(2);
auto output_number_of_pixels = wanted_height * wanted_height * wanted_channels;
for (int i = 0; i < output_number_of_pixels; i++)
{
if (input_floating)
out[i] = (output[i] - input_mean) / input_std;
else
out[i] = (uint8_t)output[i];
}
}
template
void Widget::get_top_n(T* prediction, int prediction_size, size_t num_results,
float threshold, std::vector>* top_results,
TfLiteType input_type) {
// Will contain top N results in ascending order.
std::priority_queue, std::vector>,
std::greater>>
top_result_pq;
const long count = prediction_size; // NOLINT(runtime/int)
float value = 0.0;
for (int i = 0; i < count; ++i) {
switch (input_type) {
case kTfLiteFloat32:
value = prediction[i];
break;
case kTfLiteInt8:
value = (prediction[i] + 128) / 256.0;
break;
case kTfLiteUInt8:
value = prediction[i] / 255.0;
break;
default:
break;
}
// Only add it if it beats the threshold and has a chance at being in
// the top N.
if (value < threshold) {
continue;
}
top_result_pq.push(std::pair(value, i));
// If at capacity, kick the smallest value out.
if (top_result_pq.size() > num_results) {
top_result_pq.pop();
}
}
// Copy to output vector and reverse into descending order.
while (!top_result_pq.empty()) {
top_results->push_back(top_result_pq.top());
top_result_pq.pop();
}
std::reverse(top_results->begin(), top_results->end());
}
void Widget::sendRequest(QByteArray& imageBase64)
{
QUrlQuery query;
query.addQueryItem("appkey", "3a24b33468565b633d25d426eb0c660c");
qint64 timestamp = QDateTime::currentDateTime().toMSecsSinceEpoch();
query.addQueryItem("timestamp", QString::number(timestamp));
QString sign = "58125e5985e6ef2d385ebfaa646987ba" + QString::number(timestamp);
QByteArray ba = QCryptographicHash::hash(sign.toUtf8(), QCryptographicHash::Md5);
query.addQueryItem("sign", ba.toHex());
url->setQuery(query);
networkRequest->setUrl(*url);
// qDebug() << query.toString();
QJsonObject json;
json.insert("imgBase64", QString(imageBase64));
json.insert("cityId", "440300");
QJsonDocument document;
document.setObject(json);
QByteArray data = document.toJson(QJsonDocument::Compact);
// qDebug() << QString(data);
networkManager->post(*networkRequest, data);
}
void Widget::onRequestFinished(QNetworkReply* reply)
{
QVariant statusCode = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute);
if (!statusCode.isValid()) {
QMessageBox::critical(this, "错误", "网络连接不可用,请检查后重试");
exit(0);
}
QByteArray replyData = reply->readAll();
QJsonParseError jsonError;
QJsonDocument document = QJsonDocument::fromJson(replyData, &jsonError);
if (!document.isNull() && (jsonError.error == QJsonParseError::NoError)) {
QJsonObject object = document.object().value("result").toObject();
QJsonArray array = object.value("garbage_info").toArray();
QJsonObject max = std::max_element(array.begin(), array.end(),
[](QJsonValue const& a, QJsonValue const& b) { return a.toObject().value("confidence").toDouble()
< b.toObject().value("confidence").toDouble(); })
->toObject();
QString cate_name = max.value("cate_name").toString();
QString garbage_name = max.value("garbage_name").toString();
double confidence = max.value("confidence").toDouble();
// qDebug() << "cate_name:" << cate_name;
// qDebug() << "garbage_name:" << garbage_name;
// qDebug() << "confidence:" << confidence;
// ui->textEdit->append("cate_name: " + cate_name);
// ui->textEdit->append("garbage_name: " + garbage_name);
// ui->textEdit->append("confidence: " + QString::number(confidence));
classifyFinished(cate_name);
} else {
serialWrite('\xFD');
//ui->textEdit->append("识别失败,请重试");
//ui->label_3->setText("识别失败");
ui->label_4->setVisible(true);
ui->label_5->setVisible(false);
ui->frame->setStyleSheet("#frame {border-image: url(:/new/prefix1/image/主.png);}");
}
}
void Widget::classifyFinished(QString cate_name)
{
ui->frame->setStyleSheet("#frame {border-image: url(:/new/prefix1/image/" + cate_name + ".PNG);}");
ui->label_3->setText("投递中");
if (cate_name == "识别失败") {
serialWrite('\xFD');
//ui->textEdit->append("识别失败,请重试");
//ui->label_3->setText("识别失败");
ui->label_4->setVisible(true);
ui->label_5->setVisible(false);
ui->frame->setStyleSheet("#frame {border-image: url(:/new/prefix1/image/主.png);}");
}else {
number += 1;
ui->textEdit->append(QString::number(number) + " " + cate_name + " 1 OK!");
if (cate_name == "可回收垃圾")
serialWrite('\x01');
else if (cate_name == "厨余垃圾")
serialWrite('\x02');
else if (cate_name == "有害垃圾")
serialWrite('\x04');
else
serialWrite('\x08');
}
}