/* * Copyright (C) 2021 刘臣轩 * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ #include "widget.h" #include "ui_widget.h" Widget::Widget(QWidget* parent) : QWidget(parent) , ui(new Ui::Widget) { ui->setupUi(this); connect(ui->pushButton, SIGNAL(clicked()), this, SLOT(close())); ui->frame->setStyleSheet("#frame {border-image: url(:/new/prefix1/image/主.png);}"); ui->textEdit->append("开始初始化设备"); // Time QTimer* timer = new QTimer(this); connect(timer, SIGNAL(timeout()), this, SLOT(timerUpdate())); timer->start(500); initSerial(); #ifdef Q_OS_WIN initCamera(); #else connect(this, SIGNAL(imageCaptured(int, QImage)), this, SLOT(onImageCaptured(int, QImage))); ui->textEdit->append("摄像头初始化成功"); #endif // Network networkManager = new QNetworkAccessManager(); networkRequest = new QNetworkRequest(); networkRequest->setHeader(QNetworkRequest::ContentTypeHeader, " application/json;charset=UTF-8"); url = new QUrl("https://aiapi.jd.com/jdai/garbageImageSearch"); connect(networkManager, SIGNAL(finished(QNetworkReply*)), this, SLOT(onRequestFinished(QNetworkReply*))); ui->label_3->setText("工训大赛"); ui->label_4->setStyleSheet("border-image: url(:/new/prefix1/image/工训大赛.png);\nborder-radius: 10px;\n"); ui->label_4->setVisible(true); ui->label_5->setVisible(false); ui->textEdit->append("设备初始化成功√"); number = 0; // Video player = new QMediaPlayer; videoWidget = new QVideoWidget(this); playList = new QMediaPlaylist; #ifdef Q_OS_WIN playList->addMedia(QUrl::fromLocalFile("../WasteSorting/test.mp4")); #else playList->addMedia(QUrl::fromLocalFile("/home/pi/WasteSorting/test.mp4")); #endif playList->setPlaybackMode(QMediaPlaylist::CurrentItemInLoop); player->setPlaylist(playList); player->setVideoOutput(videoWidget); ui->verticalLayout->addWidget(videoWidget); videoWidget->setVisible(false); videoTimer = new QTimer(this); connect(videoTimer, SIGNAL(timeout()), this, SLOT(videoTimerUpdate())); videoTimer->setSingleShot(true); videoTimer->start(10000); #ifdef Q_OS_WIN #else // Tensorflow model = tflite::FlatBufferModel::BuildFromFile(model_file.c_str()); tflite::InterpreterBuilder(*model, resolver)(&interpreter); interpreter->SetNumThreads(4); interpreter->AllocateTensors(); input_tensor = interpreter->tensor(interpreter->inputs()[0]); TfLiteIntArray* output_dims = interpreter->tensor(interpreter->outputs()[0])->dims; output_size = output_dims->data[output_dims->size - 1]; #endif //captureImage(); } Widget::~Widget() { delete ui; } void Widget::timerUpdate() { QString str = QDateTime::currentDateTime().toString("yyyy年MM月dd日 hh:mm:ss"); ui->label->setText(str); } void Widget::videoTimerUpdate() { ui->label_4->setVisible(false); player->play(); videoWidget->setVisible(true); ui->label_3->setText("播放视频"); ui->textEdit->append("播放视频"); } void Widget::initSerial() { ui->textEdit->append("开始初始化串口"); serialPort = new QSerialPort(); connect(serialPort, SIGNAL(readyRead()), this, SLOT(serialRead())); if (QSerialPortInfo::availablePorts().length() == 0) { QMessageBox::critical(this, "错误", "无可用串口设备,请检查硬件连接后重试"); exit(0); } #ifdef Q_OS_WIN QString portName = QSerialPortInfo::availablePorts()[1].portName(); qDebug() << portName; ui->textEdit->append("尝试连接串口" + portName); serialPort->setPortName(portName); #else QString portName = "ttyUSB0"; ui->textEdit->append("尝试连接串口ttyUSB0"); serialPort->setPortName("ttyUSB0"); #endif if (serialPort->open(QIODevice::ReadWrite)) { ui->textEdit->append("串口连接成功"); serialPort->setBaudRate(115200); serialPort->setDataBits(QSerialPort::Data8); serialPort->setParity(QSerialPort::NoParity); serialPort->setStopBits(QSerialPort::OneStop); } else { QMessageBox::critical(this, "错误", "串口设备" + portName + "无法打开,请检查硬件连接后重试"); exit(0); } ui->textEdit->append("串口初始化成功"); serialWrite('\xCC'); } void Widget::initCamera() { // for (QCameraInfo& info : QCameraInfo::availableCameras()) { // qDebug() << info.deviceName(); // } int camNum = QCameraInfo::availableCameras().length(); if (!camNum) { QMessageBox::critical(this, "错误", "无可用摄像头,请检查硬件连接后重试"); exit(0); } ui->textEdit->append("开始初始化摄像头"); camera = new QCamera(QCameraInfo::availableCameras()[0], this); imageCapture = new QCameraImageCapture(camera); connect(imageCapture, SIGNAL(imageCaptured(int, QImage)), this, SLOT(onImageCaptured(int, QImage))); camera->setCaptureMode(QCamera::CaptureStillImage); imageCapture->setCaptureDestination(QCameraImageCapture::CaptureToBuffer); camera->start(); ui->textEdit->append("摄像头初始化成功"); } void Widget::serialRead() { QByteArray buffer = serialPort->readAll(); // qDebug() << buffer; if (buffer[0] == '\x03' && buffer[1] == '\xFC' && buffer[3] == '\xFC' && buffer[4] == '\x03') { switch (buffer[2]) { case '\x00': ui->textEdit->append("取消警报"); ui->label_3->setText("取消警报"); ui->label_4->setVisible(true); ui->label_5->setVisible(false); ui->frame->setStyleSheet("#frame {border-image: url(:/new/prefix1/image/主.png);}"); videoTimer->start(10000); break; case '\x01': ui->textEdit->append("触发拍照信号"); ui->label_3->setText("触发拍照"); videoTimer->stop(); videoWidget->setVisible(false); player->stop(); #ifdef Q_OS_WIN imageCapture->capture(); #else captureImage(); #endif break; case '\x02': ui->textEdit->append("投递完毕"); ui->label_3->setText("投递完毕"); ui->label_4->setVisible(true); ui->label_5->setVisible(false); ui->frame->setStyleSheet("#frame {border-image: url(:/new/prefix1/image/主.png);}"); videoTimer->start(10000); break; case '\x04': ui->textEdit->append("满载警报"); ui->label_3->setText("满载警报"); ui->label_4->setVisible(false); videoTimer->stop(); videoWidget->setVisible(false); player->stop(); ui->frame->setStyleSheet("#frame {border-image: url(:/new/prefix1/image/满载警报.png);}"); break; case '\x08': // qDebug() << "倾倒警报"; ui->textEdit->append("倾倒警报"); ui->label_3->setText("倾倒警报"); ui->label_4->setVisible(false); videoTimer->stop(); videoWidget->setVisible(false); player->stop(); ui->frame->setStyleSheet("#frame {border-image: url(:/new/prefix1/image/倾倒警报.png);}"); break; case '\xFF': break; } } } void Widget::serialWrite(const char data) { QByteArray buffer("\x30\xCF\x0F\xCF\x30"); buffer[2] = data; serialPort->write(buffer); } QImage Widget::cvMat2QImage(const cv::Mat& mat) { // 8-bits unsigned, NO. OF CHANNELS = 1 if(mat.type() == CV_8UC1) { QImage image(mat.cols, mat.rows, QImage::Format_Indexed8); // Set the color table (used to translate colour indexes to qRgb values) image.setColorCount(256); for(int i = 0; i < 256; i++) { image.setColor(i, qRgb(i, i, i)); } // Copy input Mat uchar *pSrc = mat.data; for(int row = 0; row < mat.rows; row ++) { uchar *pDest = image.scanLine(row); memcpy(pDest, pSrc, mat.cols); pSrc += mat.step; } return image; } // 8-bits unsigned, NO. OF CHANNELS = 3 else if(mat.type() == CV_8UC3) { // Copy input Mat const uchar *pSrc = (const uchar*)mat.data; // Create QImage with same dimensions as input Mat QImage image(pSrc, mat.cols, mat.rows, mat.step, QImage::Format_RGB888); return image.rgbSwapped(); } else if(mat.type() == CV_8UC4) { qDebug() << "CV_8UC4"; // Copy input Mat const uchar *pSrc = (const uchar*)mat.data; // Create QImage with same dimensions as input Mat QImage image(pSrc, mat.cols, mat.rows, mat.step, QImage::Format_ARGB32); return image.copy(); } else { qDebug() << "ERROR: Mat could not be converted to QImage."; return QImage(); } } void Widget::captureImage() { // system("raspistill -o ../WasteSorting/WasteSorting.jpg -t 1 -br 60 -hf -awb sun"); // system("python3 ../WasteSorting/capture.py"); //system("rm -rf /home/pi/WasteSorting/WasteSorting.jpg"); capture = cv::VideoCapture(0); cv::Mat frame; capture >> frame; QImage image = cvMat2QImage(frame); capture.release(); //cv::imwrite("/home/pi/WasteSorting/WasteSorting.jpg", frame); //QImage image("../WasteSorting/WasteSorting.jpg"); emit(imageCaptured(0, image)); } void Widget::onImageCaptured(int, QImage image) { // 显示图片 ui->label_4->setVisible(false); ui->label_5->setPixmap(QPixmap::fromImage(image).scaled(405, 306)); ui->label_5->setVisible(true); ui->label_3->setText("识别中"); ui->frame->setStyleSheet("#frame {border-image: url(:/new/prefix1/image/识别中.png);}"); /* 使用京东垃圾识别 API // Base64编码 QByteArray ba; QBuffer buf(&ba); image.save(&buf, "JPG", -1); QByteArray imageBase64 = ba.toBase64(); // qDebug() << "data:image/jpg;base64," + imageBase64; // Request sendRequest(imageBase64); */ /* TensorFlow Lite Python #ifdef Q_OS_WIN image.save("../WasteSorting/WasteSorting.jpg"); FILE* fp = _popen("python ../WasteSorting/tensorflow/label_image.py", "rt"); char buf[255] = { 0 }; fscanf(fp, "%s", buf); _pclose(fp); #else FILE* fp = popen("python3 ../WasteSorting/tensorflow/label_image.py", "r"); char buf[255] = { 0 }; printf("%s", buf); fscanf(fp, "%s", buf); pclose(fp); #endif QString cate_name = QString::fromLocal8Bit(buf); qDebug() << cate_name; classifyFinished(cate_name); */ /* Tensorflow Lite C++ */ image.save("../WasteSorting/WasteSorting.jpg"); image = image.convertToFormat(QImage::Format_RGB888).mirrored(true, false); formatImageTFLite(interpreter->typed_tensor(interpreter->inputs()[0]), image.bits(), image.height(), image.width(), 3, 224, 224, 3, false); interpreter->Invoke(); std::vector> top_results; get_top_n(interpreter->typed_output_tensor(0), output_size, 1, 0.01f, &top_results, kTfLiteUInt8); int index = top_results[0].second; QString cate_name; switch(index) { case 0: cate_name = "识别失败"; break; case 1:case 2:case 10: cate_name = "有害垃圾"; break; case 3:case 4:case 5:case 11: cate_name = "可回收垃圾"; break; case 6:case 7:case 12:case 13: cate_name = "厨余垃圾"; break; case 8:case 9: cate_name = "其他垃圾"; break; } qDebug() << cate_name; classifyFinished(cate_name); } template void Widget::formatImageTFLite(T* out, const uint8_t* in, int image_height, int image_width, int image_channels, int wanted_height, int wanted_width, int wanted_channels, bool input_floating) { const float input_mean = 127.5f; const float input_std = 127.5f; int number_of_pixels = image_height * image_width * image_channels; std::unique_ptr interpreter(new tflite::Interpreter); int base_index = 0; // two inputs: input and new_sizes interpreter->AddTensors(2, &base_index); // one output interpreter->AddTensors(1, &base_index); // set input and output tensors interpreter->SetInputs({0, 1}); interpreter->SetOutputs({2}); // set parameters of tensors TfLiteQuantizationParams quant; interpreter->SetTensorParametersReadWrite(0, kTfLiteFloat32, "input", {1, image_height, image_width, image_channels}, quant); interpreter->SetTensorParametersReadWrite(1, kTfLiteInt32, "new_size", {2},quant); interpreter->SetTensorParametersReadWrite(2, kTfLiteFloat32, "output", {1, wanted_height, wanted_width, wanted_channels}, quant); tflite::ops::builtin::BuiltinOpResolver resolver; const TfLiteRegistration *resize_op = resolver.FindOp(tflite::BuiltinOperator_RESIZE_BILINEAR,1); auto* params = reinterpret_cast(malloc(sizeof(TfLiteResizeBilinearParams))); params->align_corners = false; interpreter->AddNodeWithParameters({0, 1}, {2}, nullptr, 0, params, resize_op, nullptr); interpreter->AllocateTensors(); // fill input image // in[] are integers, cannot do memcpy() directly auto input = interpreter->typed_tensor(0); for (int i = 0; i < number_of_pixels; i++) input[i] = in[i]; // fill new_sizes interpreter->typed_tensor(1)[0] = wanted_height; interpreter->typed_tensor(1)[1] = wanted_width; interpreter->Invoke(); auto output = interpreter->typed_tensor(2); auto output_number_of_pixels = wanted_height * wanted_height * wanted_channels; for (int i = 0; i < output_number_of_pixels; i++) { if (input_floating) out[i] = (output[i] - input_mean) / input_std; else out[i] = (uint8_t)output[i]; } } template void Widget::get_top_n(T* prediction, int prediction_size, size_t num_results, float threshold, std::vector>* top_results, TfLiteType input_type) { // Will contain top N results in ascending order. std::priority_queue, std::vector>, std::greater>> top_result_pq; const long count = prediction_size; // NOLINT(runtime/int) float value = 0.0; for (int i = 0; i < count; ++i) { switch (input_type) { case kTfLiteFloat32: value = prediction[i]; break; case kTfLiteInt8: value = (prediction[i] + 128) / 256.0; break; case kTfLiteUInt8: value = prediction[i] / 255.0; break; default: break; } // Only add it if it beats the threshold and has a chance at being in // the top N. if (value < threshold) { continue; } top_result_pq.push(std::pair(value, i)); // If at capacity, kick the smallest value out. if (top_result_pq.size() > num_results) { top_result_pq.pop(); } } // Copy to output vector and reverse into descending order. while (!top_result_pq.empty()) { top_results->push_back(top_result_pq.top()); top_result_pq.pop(); } std::reverse(top_results->begin(), top_results->end()); } void Widget::sendRequest(QByteArray& imageBase64) { QUrlQuery query; query.addQueryItem("appkey", "3a24b33468565b633d25d426eb0c660c"); qint64 timestamp = QDateTime::currentDateTime().toMSecsSinceEpoch(); query.addQueryItem("timestamp", QString::number(timestamp)); QString sign = "58125e5985e6ef2d385ebfaa646987ba" + QString::number(timestamp); QByteArray ba = QCryptographicHash::hash(sign.toUtf8(), QCryptographicHash::Md5); query.addQueryItem("sign", ba.toHex()); url->setQuery(query); networkRequest->setUrl(*url); // qDebug() << query.toString(); QJsonObject json; json.insert("imgBase64", QString(imageBase64)); json.insert("cityId", "440300"); QJsonDocument document; document.setObject(json); QByteArray data = document.toJson(QJsonDocument::Compact); // qDebug() << QString(data); networkManager->post(*networkRequest, data); } void Widget::onRequestFinished(QNetworkReply* reply) { QVariant statusCode = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute); if (!statusCode.isValid()) { QMessageBox::critical(this, "错误", "网络连接不可用,请检查后重试"); exit(0); } QByteArray replyData = reply->readAll(); QJsonParseError jsonError; QJsonDocument document = QJsonDocument::fromJson(replyData, &jsonError); if (!document.isNull() && (jsonError.error == QJsonParseError::NoError)) { QJsonObject object = document.object().value("result").toObject(); QJsonArray array = object.value("garbage_info").toArray(); QJsonObject max = std::max_element(array.begin(), array.end(), [](QJsonValue const& a, QJsonValue const& b) { return a.toObject().value("confidence").toDouble() < b.toObject().value("confidence").toDouble(); }) ->toObject(); QString cate_name = max.value("cate_name").toString(); QString garbage_name = max.value("garbage_name").toString(); double confidence = max.value("confidence").toDouble(); // qDebug() << "cate_name:" << cate_name; // qDebug() << "garbage_name:" << garbage_name; // qDebug() << "confidence:" << confidence; // ui->textEdit->append("cate_name: " + cate_name); // ui->textEdit->append("garbage_name: " + garbage_name); // ui->textEdit->append("confidence: " + QString::number(confidence)); classifyFinished(cate_name); } else { serialWrite('\xFD'); //ui->textEdit->append("识别失败,请重试"); //ui->label_3->setText("识别失败"); ui->label_4->setVisible(true); ui->label_5->setVisible(false); ui->frame->setStyleSheet("#frame {border-image: url(:/new/prefix1/image/主.png);}"); } } void Widget::classifyFinished(QString cate_name) { ui->frame->setStyleSheet("#frame {border-image: url(:/new/prefix1/image/" + cate_name + ".PNG);}"); ui->label_3->setText("投递中"); if (cate_name == "识别失败") { serialWrite('\xFD'); //ui->textEdit->append("识别失败,请重试"); //ui->label_3->setText("识别失败"); ui->label_4->setVisible(true); ui->label_5->setVisible(false); ui->frame->setStyleSheet("#frame {border-image: url(:/new/prefix1/image/主.png);}"); }else { number += 1; ui->textEdit->append(QString::number(number) + " " + cate_name + " 1 OK!"); if (cate_name == "可回收垃圾") serialWrite('\x01'); else if (cate_name == "厨余垃圾") serialWrite('\x02'); else if (cate_name == "有害垃圾") serialWrite('\x04'); else serialWrite('\x08'); } }