#include #include #include #include #include #include #include #include // Normalize class to preprocess image channels class Normalize { public: Normalize() { mean = {0.485, 0.456, 0.406}; std = {0.229, 0.224, 0.225}; } // Function to normalize the image cv::Mat operator()(const cv::Mat& image) { cv::Mat normalizedImage; image.convertTo(normalizedImage, CV_32F, 1.0/255.0); // Convert to float32 and scale // Normalize each channel for (int i = 0; i < 3; ++i) { normalizedImage.forEach([&](cv::Vec3f &pixel, const int * position) -> void { pixel[i] = (pixel[i] - mean[i]) / std[i]; }); } return normalizedImage; } private: std::vector mean; std::vector std; }; namespace cv { typedef Vec Vec5f; } // Assuming you have a constant MODEL_SCALE defined somewhere const float MODEL_SCALE = 4; // Replace with your actual value std::pair pred2box(const cv::Mat& hm, const torch::Tensor& offset, const torch::Tensor& regr, const torch::Tensor& cos_sin_hm, float thresh = 0.99) { // Get center points where heatmap value is above threshold cv::Mat pred; cv::compare(hm, thresh, pred, cv::CMP_GT); // pred is a binary mask std::vector pred_center_points; cv::findNonZero(pred, pred_center_points); // Extract regressions and angles at predicted centers std::vector pred_r; std::vector pred_angles; for (auto& center : pred_center_points) { float regr_val1 = regr.index({0,center.y, center.x}).item(); float regr_val2 = regr.index({1,center.y, center.x}).item(); float cos_sin_val1 = cos_sin_hm.index({0,center.y, center.x}).item(); float cos_sin_val2 = cos_sin_hm.index({1,center.y, center.x}).item(); pred_angles.push_back(cv::Vec2f(cos_sin_val1, cos_sin_val2)); pred_r.push_back(cv::Vec2f(regr_val1, regr_val2)); } // Create bounding boxes std::vector boxes; cv::Mat scores; hm.convertTo(scores, CV_32F); // Convert scores to float for later use for (size_t i = 0; i < pred_center_points.size(); ++i) { auto& center = pred_center_points[i]; const auto& b = pred_r[i]; const auto& pred_angle = pred_angles[i]; float offsetx = offset.index({0,center.y, center.x}).item(); float offsety = offset.index({1,center.y, center.x}).item(); cv::Vec2f offset_xy = cv::Vec2f(offsetx, offsety); float angle = std::atan2(pred_angle[1], pred_angle[0]); cv::Vec5f arr = { (center.x + offset_xy[0]) * MODEL_SCALE, (center.y + offset_xy[1]) * MODEL_SCALE, b[0] * MODEL_SCALE, b[1] * MODEL_SCALE, angle }; boxes.push_back(arr); } // Convert to cv::Mat for output cv::Mat boxes_mat(boxes.size(), 1, CV_32FC(5), boxes.data()); cv::Mat scores_mat; for (const auto& center : pred_center_points) { scores_mat.push_back(scores.at(center)); } return {boxes_mat, scores_mat}; } cv::Mat select(cv::Mat& hm, float threshold) { cv::Mat pred; cv::compare(hm, threshold, pred, cv::CMP_GT); std::vector pred_centers; cv::findNonZero(pred, pred_centers); for (size_t i = 0; i < pred_centers.size(); ++i) { for (size_t j = i + 1; j < pred_centers.size(); ++j) { const cv::Point& ci = pred_centers[i]; const cv::Point& cj = pred_centers[j]; float distance = cv::norm(ci - cj); if (distance <= 2) { float score_i = hm.at(ci); float score_j = hm.at(cj); if (score_i > score_j) { hm.at(cj) = 0; } else { hm.at(ci) = 0; } } } } return hm; } std::tuple resize_and_pad(const cv::Mat& image, const cv::Size& target_size = cv::Size(512, 512)) { int original_height = image.rows; int original_width = image.cols; int target_width = target_size.width; int target_height = target_size.height; // Calculate the scaling factor float scale = std::min(static_cast(target_width) / original_width, static_cast(target_height) / original_height); // Calculate new dimensions int new_width = static_cast(original_width * scale); int new_height = static_cast(original_height * scale); // Resize the image cv::Mat resized_image; cv::resize(image, resized_image, cv::Size(new_width, new_height)); // Pad the image to the target size int delta_w = target_width - new_width; int delta_h = target_height - new_height; int top = delta_h / 2; int bottom = delta_h - top; int left = delta_w / 2; int right = delta_w - left; cv::Mat padded_image; cv::copyMakeBorder(resized_image, padded_image, top, bottom, left, right, cv::BORDER_CONSTANT, cv::Scalar(0, 0, 0)); return std::make_tuple(padded_image, scale, left, top); } std::vector pred4corner(const cv::Mat& hm, float thresh = 0.99) { float threshold = 0.2; cv::Mat thresholded_heatmap; cv::threshold(hm, thresholded_heatmap, threshold, 1, cv::THRESH_BINARY); cv::Mat thresholded_heatmap_8u; thresholded_heatmap.convertTo(thresholded_heatmap_8u, CV_8UC1); std::vector> contours; cv::findContours(thresholded_heatmap_8u, contours, cv::RETR_EXTERNAL, cv::CHAIN_APPROX_SIMPLE); std::vector keypoints; for (const auto& cnt : contours) { cv::Moments M = cv::moments(cnt); if (M.m00 != 0) { // Avoid division by zero int cx = static_cast(M.m10 / M.m00); int cy = static_cast(M.m01 / M.m00); keypoints.push_back(cv::Point(cx, cy)); } } return keypoints; } cv::Mat showbox(const cv::Mat& img, const cv::Mat& hm, const torch::Tensor& offset, const torch::Tensor& regr, const torch::Tensor& cos_sin_hm, float thresh = 0.9) { auto [boxes, _] = pred2box(hm, offset, regr, cos_sin_hm, thresh); // Get predicted boxes cv::Mat sample = img.clone(); // Create a copy to draw on for (int i = 0; i < boxes.rows; ++i) { const float* box_data = boxes.ptr(i); // Access box data cv::Point center(box_data[0], box_data[1]); float cos_angle = std::cos(box_data[4]); float sin_angle = std::sin(box_data[4]); cv::Mat rot = (cv::Mat_(2, 2) << cos_angle, sin_angle, -sin_angle, cos_angle); cv::Mat half_size = (cv::Mat_(2, 1) << box_data[2] / 2, box_data[3] / 2); cv::Mat bottom_right = rot * half_size; cv::Mat top_right = rot * (cv::Mat_(2, 1) << box_data[2] / 2, -box_data[3] / 2); cv::Mat top_left = rot * (cv::Mat_(2, 1) << -box_data[2] / 2, -box_data[3] / 2); cv::Mat bottom_left = rot * (cv::Mat_(2, 1) << -box_data[2] / 2, box_data[3] / 2); int thickness = 3; cv::line(sample, center + cv::Point(bottom_right), center + cv::Point(top_right), cv::Scalar(0, 220, 0), thickness); cv::line(sample, center + cv::Point(bottom_right), center + cv::Point(bottom_left), cv::Scalar(220, 220, 0), thickness); cv::line(sample, center + cv::Point(top_left), center + cv::Point(bottom_left), cv::Scalar(220, 220, 0), thickness); cv::line(sample, center + cv::Point(top_left), center + cv::Point(top_right), cv::Scalar(220, 220, 0), thickness); } return sample; } namespace fs = std::filesystem; int main() { // Load your PyTorch model torch::jit::script::Module model; try { model = torch::jit::load("C:/Users/John/Desktop/CPP_Centernet/hardnet_angle_4c_centernet_jit.pth"); // Replace with the actual path model.to(torch::kCUDA); // Move model to GPU if available } catch (const c10::Error& e) { std::cerr << "Error loading the model: " << e.what() << std::endl; return -1; } bool image = false; std::string test_folder = "C:/Users/John/Desktop/rotated_barcode/roboflow_barcode/test/images/"; float threshold = 0.2; bool half = false; // Set to true if you want to use half precision if (image) { // Iterate through image files in the test folder for (const auto& entry : fs::directory_iterator(test_folder)) { if (entry.is_regular_file() && entry.path().extension() == ".jpg") { cv::Mat image = cv::imread(entry.path().string()); // Preprocess the image auto [resized_image, scale, left, top] = resize_and_pad(image); cv::Mat img = resized_image.clone(); cv::Mat imgshow = img.clone(); img = Normalize()(img); torch::Tensor tensor = torch::from_blob(img.data, {1, img.rows, img.cols, 3}, torch::kFloat).permute({0, 3, 1, 2}); if (half) { tensor = tensor.to(torch::kCUDA).to(torch::kHalf); //.unsqueeze(0); } else { tensor = tensor.to(torch::kCUDA); //.unsqueeze(0); } // Perform inference torch::NoGradGuard no_grad; auto outputs = model.forward({tensor}).toTuple(); torch::Tensor hm = outputs->elements()[0].toTensor().to(torch::kCPU).squeeze(0); torch::Tensor offset_tensor = outputs->elements()[1].toTensor().to(torch::kCPU).squeeze(0); torch::Tensor wh_tensor = outputs->elements()[2].toTensor().to(torch::kCPU).squeeze(0); torch::Tensor angle_tensor = outputs->elements()[3].toTensor().to(torch::kCPU).squeeze(0); // Postprocess the outputs hm = torch::sigmoid(hm); cv::Mat hm_mat(hm.size(1), hm.size(2), CV_32F, hm[0].data_ptr()); cv::Mat hm_mat_corner(hm.size(1), hm.size(2), CV_32F, hm[1].data_ptr()); cv::imshow("heatmap",hm_mat); cv::imshow("heatmap corner",hm_mat_corner); hm_mat = select(hm_mat, threshold); // Visualize the results cv::Mat sample = showbox(imgshow, hm_mat, offset_tensor, wh_tensor, angle_tensor, threshold); cv::resize(hm_mat_corner,hm_mat_corner,cv::Size(512,512)); auto kpoints = pred4corner(hm_mat_corner,threshold); for (auto element: kpoints) { cv::circle(sample,cv::Point(element.x,element.y),5,cv::Scalar(255,255,255),-1); } cv::imshow("output", sample); // Wait for a key press char ch = cv::waitKey(0); if (ch == 'q') { cv::destroyAllWindows(); break; } } } } else { cv::VideoCapture cap(0); cap.set(cv::CAP_PROP_FOURCC,1196444237); cap.set(cv::CAP_PROP_FRAME_WIDTH,1280); cap.set(cv::CAP_PROP_FRAME_HEIGHT,720); cap.set(cv::CAP_PROP_AUTO_EXPOSURE, 3); //auto cap.set(cv::CAP_PROP_AUTO_EXPOSURE, 1); //manual //cap.set(cv::CAP_PROP_EXPOSURE,25); cap.set(cv::CAP_PROP_EXPOSURE,-5); cap.set(cv::CAP_PROP_FPS,30); if (!cap.isOpened()) { std::cerr << "Error opening webcam" << std::endl; return -1; } while (true) { cv::Mat frame; cap >> frame; cv::Mat image = frame.clone(); // Preprocess the image auto [resized_image, scale, left, top] = resize_and_pad(image); cv::Mat img = resized_image.clone(); cv::Mat imgshow = img.clone(); img = Normalize()(img); torch::Tensor tensor = torch::from_blob(img.data, {1, img.rows, img.cols, 3}, torch::kFloat).permute({0, 3, 1, 2}); if (half) { tensor = tensor.to(torch::kCUDA).to(torch::kHalf); //.unsqueeze(0); } else { tensor = tensor.to(torch::kCUDA); //.unsqueeze(0); } // Perform inference torch::NoGradGuard no_grad; auto start_time = std::chrono::high_resolution_clock::now(); auto outputs = model.forward({tensor}).toTuple(); auto end_time = std::chrono::high_resolution_clock::now(); std::chrono::duration elapsed_seconds = end_time - start_time; double fps = 1.0 / elapsed_seconds.count(); torch::Tensor hm = outputs->elements()[0].toTensor().to(torch::kCPU).squeeze(0); torch::Tensor offset_tensor = outputs->elements()[1].toTensor().to(torch::kCPU).squeeze(0); torch::Tensor wh_tensor = outputs->elements()[2].toTensor().to(torch::kCPU).squeeze(0); torch::Tensor angle_tensor = outputs->elements()[3].toTensor().to(torch::kCPU).squeeze(0); // Postprocess the outputs hm = torch::sigmoid(hm); cv::Mat hm_mat(hm.size(1), hm.size(2), CV_32F, hm[0].data_ptr()); cv::Mat hm_mat_corner(hm.size(1), hm.size(2), CV_32F, hm[1].data_ptr()); //cv::imshow("heatmap",hm_mat); //cv::imshow("heatmap corner",hm_mat_corner); hm_mat = select(hm_mat, threshold); // Visualize the results cv::Mat sample = showbox(imgshow, hm_mat, offset_tensor, wh_tensor, angle_tensor, threshold); cv::putText(sample, "FPS: " + std::to_string(fps), cv::Point(10, 30), cv::FONT_HERSHEY_SIMPLEX, 1, cv::Scalar(0, 255, 0), 2); cv::resize(hm_mat_corner,hm_mat_corner,cv::Size(512,512)); auto kpoints = pred4corner(hm_mat_corner,threshold); for (auto element: kpoints) { cv::circle(sample,cv::Point(element.x,element.y),5,cv::Scalar(255,255,255),-1); } cv::imshow("output", sample); // Wait for a key press char ch = cv::waitKey(1); if (ch == 'q') { cv::destroyAllWindows(); break; } } } return 0; }