diff --git a/.github/workflows/cpp-ci-serial-programs-base.yml b/.github/workflows/cpp-ci-serial-programs-base.yml index 193e5ca06c..102c2203f6 100644 --- a/.github/workflows/cpp-ci-serial-programs-base.yml +++ b/.github/workflows/cpp-ci-serial-programs-base.yml @@ -133,6 +133,26 @@ jobs: repository: 'PokemonAutomation/CommandLineTests' path: 'CommandLineTests' + # The unit tests assume the downloadable resources they need are already downloaded + # (see Packages/Resources/ResourceDownloadList.json). Unzip them from the Packages + # checkout into where the program looks for them, DOWNLOADED_RESOURCE_PATH(): + # the working directory on Windows/Linux, and ~/Library/Application Support/SerialPrograms + # on macOS. + - name: Install downloadable resources for tests + if: inputs.run-tests + shell: bash + run: | + if [[ "${{inputs.os}}" == mac* ]]; then + DOWNLOADED_RESOURCES="$HOME/Library/Application Support/SerialPrograms/DownloadedResources" + else + DOWNLOADED_RESOURCES="Arduino-Source/SerialPrograms/bin/DownloadedResources" + fi + for RESOURCE in PaddleOCRDetection; do + mkdir -p "$DOWNLOADED_RESOURCES/$RESOURCE" + (cd "$DOWNLOADED_RESOURCES/$RESOURCE" && cmake -E tar xf "${{github.workspace}}/Arduino-Source/Packages/DownloadedResources/$RESOURCE.zip") + ls -la "$DOWNLOADED_RESOURCES/$RESOURCE" + done + - name: Run tests (Windows) if: startsWith(inputs.os, 'windows') && inputs.run-tests run: | diff --git a/SerialPrograms/Source/CommonTools/OCR/OCR_RawPaddleOCR.cpp b/SerialPrograms/Source/CommonTools/OCR/OCR_PaddleOCR.cpp similarity index 86% rename from SerialPrograms/Source/CommonTools/OCR/OCR_RawPaddleOCR.cpp rename to SerialPrograms/Source/CommonTools/OCR/OCR_PaddleOCR.cpp index 508befad36..5713425ec4 100644 --- a/SerialPrograms/Source/CommonTools/OCR/OCR_RawPaddleOCR.cpp +++ b/SerialPrograms/Source/CommonTools/OCR/OCR_PaddleOCR.cpp @@ -9,7 +9,7 @@ #include "Common/Cpp/Concurrency/SpinLock.h" #include "CommonFramework/ImageTypes/ImageViewRGB32.h" #include "ML/Inference/ML_PaddleOCRPipeline.h" -#include "OCR_RawPaddleOCR.h" +#include "OCR_PaddleOCR.h" namespace PokemonAutomation{ namespace OCR{ @@ -97,7 +97,7 @@ ML::PaddleOCRPipeline& ensure_paddle_ocr_instance(Language language){ } -std::string paddle_ocr_read(Language language, const ImageViewRGB32& image){ +std::string paddle_ocr_read(Language language, const ImageViewRGB32& image, PageSegMode psm){ // static size_t c = 0; // image.save("ocr-" + std::to_string(c++) + ".png"); @@ -105,7 +105,18 @@ std::string paddle_ocr_read(Language language, const ImageViewRGB32& image){ // Run inference with the paddle model. // PaddleOCR with Onnx is threadsafe, so a single instance can be called by multiple threads. - std::string ret = paddle_instance.recognize(image); + std::string ret; + switch (psm){ + case PageSegMode::AUTO: + case PageSegMode::SINGLE_BLOCK: + case PageSegMode::SINGLE_COLUMN: + // Multi-line: run text detection first, then recognize each line. + ret = paddle_instance.recognize_multiline(image); + break; + default: + // Single line: the image is already cropped to the text, skip detection. + ret = paddle_instance.recognize(image); + } // global_logger_tagged().log(ret); diff --git a/SerialPrograms/Source/CommonTools/OCR/OCR_RawPaddleOCR.h b/SerialPrograms/Source/CommonTools/OCR/OCR_PaddleOCR.h similarity index 62% rename from SerialPrograms/Source/CommonTools/OCR/OCR_RawPaddleOCR.h rename to SerialPrograms/Source/CommonTools/OCR/OCR_PaddleOCR.h index 6d9e9df795..7ff241287d 100644 --- a/SerialPrograms/Source/CommonTools/OCR/OCR_RawPaddleOCR.h +++ b/SerialPrograms/Source/CommonTools/OCR/OCR_PaddleOCR.h @@ -8,11 +8,12 @@ // This file should not be inluded in other files. // Use OCR_Routines.h instead -#ifndef PokemonAutomation_CommonTools_OCR_RawPaddleOCR_H -#define PokemonAutomation_CommonTools_OCR_RawPaddleOCR_H +#ifndef PokemonAutomation_CommonTools_OCR_PaddleOCR_H +#define PokemonAutomation_CommonTools_OCR_PaddleOCR_H #include #include "CommonFramework/Language.h" +#include "OCR_RawTesseractOCR.h" namespace PokemonAutomation{ class ImageViewRGB32; @@ -36,9 +37,18 @@ ML::PaddleOCRPipeline& ensure_paddle_ocr_instance(Language language); // It creates one PaddleOCR instance for each language. You can // call `ensure_instances()` to pre-warm to pool with a given number of instances. // +// psm: Page segmentation mode, same meaning as in `tesseract_ocr_read()`. +// Defaults to SINGLE_LINE. +// - AUTO, SINGLE_BLOCK, SINGLE_COLUMN: multi-line OCR. Runs the PaddleOCR text +// detection model to find every line of text in the image, then recognizes each +// line. Lines are returned top to bottom, separated by '\n'. Use this when the +// image is not pre-cropped to a single line, e.g. a whole dialog box. +// - Any other mode: single-line OCR. Assumes the image is already cropped to one +// line of text and skips the detection model. std::string paddle_ocr_read( Language language, - const ImageViewRGB32& image + const ImageViewRGB32& image, + PageSegMode psm = PageSegMode::SINGLE_LINE ); diff --git a/SerialPrograms/Source/CommonTools/OCR/OCR_Routines.cpp b/SerialPrograms/Source/CommonTools/OCR/OCR_Routines.cpp index afea8343a6..3288422956 100644 --- a/SerialPrograms/Source/CommonTools/OCR/OCR_Routines.cpp +++ b/SerialPrograms/Source/CommonTools/OCR/OCR_Routines.cpp @@ -9,7 +9,7 @@ #include "CommonFramework/GlobalSettingsPanel.h" #include "CommonFramework/Exceptions/OperationFailedException.h" #include "CommonTools/Images/ImageFilter.h" -#include "OCR_RawPaddleOCR.h" +#include "OCR_PaddleOCR.h" #include "OCR_RawTesseractOCR.h" #include "OCR_DictionaryMatcher.h" #include "OCR_Routines.h" @@ -34,31 +34,15 @@ bool ocr_language_available(Language language){ std::string ocr_read(Language language, const ImageViewRGB32& image, PageSegMode psm){ - std::string ocr_text = ""; - if (psm == PageSegMode::AUTO || psm == PageSegMode::SINGLE_BLOCK || psm == PageSegMode::SINGLE_COLUMN){ - // if using multiline detection, force Tesseract - ocr_text = OCR::tesseract_ocr_read(language, image, psm); + if (GlobalSettings::instance().OCR_LIBRARY == OcrLibrary::PADDLE_OCR){ + return OCR::paddle_ocr_read(language, image, psm); }else{ - if (GlobalSettings::instance().OCR_LIBRARY == OcrLibrary::PADDLE_OCR){ - ocr_text = OCR::paddle_ocr_read(language, image); - }else{ - ocr_text = OCR::tesseract_ocr_read(language, image, psm); - } + return OCR::tesseract_ocr_read(language, image, psm); } - return ocr_text; } bool allow_parallel_ocr(PageSegMode psm){ - if (psm == PageSegMode::AUTO || psm == PageSegMode::SINGLE_BLOCK || psm == PageSegMode::SINGLE_COLUMN){ - // if using multiline detection, force Tesseract - return true; - }else{ - if (GlobalSettings::instance().OCR_LIBRARY == OcrLibrary::PADDLE_OCR){ - return false; - }else{ - return true; - } - } + return GlobalSettings::instance().OCR_LIBRARY != OcrLibrary::PADDLE_OCR; } void ensure_ocr_instances(Language language, size_t instances){ diff --git a/SerialPrograms/Source/CommonTools/OCR/OCR_Routines.h b/SerialPrograms/Source/CommonTools/OCR/OCR_Routines.h index ecb1f75a0c..dc49f41cee 100644 --- a/SerialPrograms/Source/CommonTools/OCR/OCR_Routines.h +++ b/SerialPrograms/Source/CommonTools/OCR/OCR_Routines.h @@ -32,9 +32,15 @@ struct TextColorRange{ bool ocr_language_available(Language language); -// will use the OCR library as per the dropdown in GlobalSettings -// if the preferred library's resource is missing (i.e. it has not been downloaded), +// Main OCR function for Pokemon automation. +// It uses the OCR library as per the dropdown in GlobalSettings +// If the preferred library's resource is missing (i.e. it has not been downloaded), // an error will be thrown within OCR initialization infra. +// psm: text layout mode to help improve OCR model accuracy. +// with PaddleOCR, SINGLE_LINE (and other single-line modes) assume the image is +// already cropped to one line of text. AUTO, SINGLE_BLOCK and SINGLE_COLUMN run the +// PaddleOCR text detection model to find each line first, and return the lines +// separated by '\n'. std::string ocr_read(Language language, const ImageViewRGB32& image, PageSegMode psm = PageSegMode::SINGLE_LINE); // diff --git a/SerialPrograms/Source/CommonTools/OCR/OCR_Tests.cpp b/SerialPrograms/Source/CommonTools/OCR/OCR_Tests.cpp index 593701bcea..1c93087f02 100644 --- a/SerialPrograms/Source/CommonTools/OCR/OCR_Tests.cpp +++ b/SerialPrograms/Source/CommonTools/OCR/OCR_Tests.cpp @@ -28,6 +28,7 @@ void add_tests(UnitTestDatabase& database){ add_tests_raw_OCR(database); add_tests_number_waterfill_OCR(database); Pokemon::add_tests_PokemonNameReader(database); + add_tests_multiline_OCR(database); } class Test_RawOCR : public UnitTest{ @@ -128,5 +129,97 @@ void add_tests_number_waterfill_OCR(UnitTestDatabase& database){ +// Crop `box` out of a screenshot, run multi-line OCR on it, and check that it finds +// exactly the expected lines in order. Each line is compared after +// `normalize_utf32()`, which ignores punctuation and whitespace. +class Test_MultilineOCR : public UnitTest{ +public: + Test_MultilineOCR( + const std::string& image, + const ImagePixelBox& box, + Language language, + std::vector expected_lines + ) + : UnitTest("OCR::MultilineOCR - " + image) + , m_image(UNIT_TEST_RESOURCE_PATH() + image) + , m_box(box) + , m_language(language) + , m_expected_lines(std::move(expected_lines)) + {} + + virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override{ + ImageRGB32 image(m_image); + ImageViewRGB32 cropped = extract_box_reference(image, m_box); + + std::string result = ocr_read(m_language, cropped, PageSegMode::SINGLE_BLOCK); + logger.log("Multi-line OCR read: " + result); + + std::vector lines; + size_t start = 0; + while (true){ + size_t end = result.find('\n', start); + lines.emplace_back(result.substr(start, end - start)); + if (end == std::string::npos){ + break; + } + start = end + 1; + } + + if (lines.size() != m_expected_lines.size()){ + logger.log( + "Expected " + std::to_string(m_expected_lines.size()) + + " lines, got " + std::to_string(lines.size()) + ); + return false; + } + for (size_t c = 0; c < lines.size(); c++){ + if (normalize_utf32(lines[c]) != normalize_utf32(m_expected_lines[c])){ + logger.log("Line " + std::to_string(c) + " mismatch. Expected: " + m_expected_lines[c]); + return false; + } + } + return true; + }; + +private: + std::string m_image; + ImagePixelBox m_box; + Language m_language; + std::vector m_expected_lines; +}; + +void add_tests_multiline_OCR(UnitTestDatabase& database){ + // Already cropped to a single line, but with lots of empty space around it. + database.add( + "OCR/sentence-1-1-tall.jpg", ImagePixelBox(0, 0, 564, 500), Language::English, + std::vector{"You hurry to the Pokemon Center, shielding your"} + ); + database.add( + "OCR/sentence-1-1-wide.jpg", ImagePixelBox(0, 0, 1000, 41), Language::English, + std::vector{"You hurry to the Pokemon Center, shielding your"} + ); + + // Two-line FRLG dialog boxes: blue text on white. + database.add( + "PokemonFRLG/AdvanceWhiteDialogDetector/English-Bulba_True.png", + ImagePixelBox(270, 790, 1650, 1000), Language::English, + std::vector{"I see! BULBASAUR is your choice.", "It's very easy to raise."} + ); + database.add( + "PokemonFRLG/AdvanceWhiteDialogDetector/Ger-KennyExile-Char_True.png", + ImagePixelBox(270, 790, 1650, 1000), Language::German, + std::vector{"Ah! GLUMANDA soll es sein!", "Trainiere es geduldig!"} + ); + + // SV dialog box: speaker name (white on dark blue) above the dialog text (dark on white). + database.add( + "PokemonSV/DialogBoxDetector/French_True.png", + ImagePixelBox(430, 740, 1470, 975), Language::French, + std::vector{"Clavel", "Poussacha, le Pokémon Chat Plante."} + ); +} + + + } } diff --git a/SerialPrograms/Source/CommonTools/OCR/OCR_Tests.h b/SerialPrograms/Source/CommonTools/OCR/OCR_Tests.h index a0189ae1eb..bc6c950b5e 100644 --- a/SerialPrograms/Source/CommonTools/OCR/OCR_Tests.h +++ b/SerialPrograms/Source/CommonTools/OCR/OCR_Tests.h @@ -20,6 +20,10 @@ void add_tests_raw_OCR(UnitTestDatabase& database); void add_tests_number_waterfill_OCR(UnitTestDatabase& database); +// Multi-line OCR tests. These exercise the PaddleOCR detection + recognition pipeline +// through `ocr_read()` with a multi-line page segmentation mode. +void add_tests_multiline_OCR(UnitTestDatabase& database); + } diff --git a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRDetector.cpp b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRDetector.cpp new file mode 100644 index 0000000000..3d83bad3c3 --- /dev/null +++ b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRDetector.cpp @@ -0,0 +1,550 @@ +/* ML Paddle OCR Detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Exceptions.h" +#include "Common/Cpp/Rectangle.tpp" +#include "CommonFramework/Logging/Logger.h" +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/StaticGlobals.h" +#include "CommonFramework/Options/Environment/PerformanceOptions.h" +#include "ML/Models/ML_ONNXRuntimeHelpers.h" +#include "ML_PaddleOCRPipeline.h" +#include "ML_PaddleOCRDetector.h" + +namespace PokemonAutomation{ +namespace ML{ + + + +Rectangle PaddleOCRTextBox::extent() const{ + return Rectangle( + std::min({corners[0].x, corners[1].x, corners[2].x, corners[3].x}), + std::min({corners[0].y, corners[1].y, corners[2].y, corners[3].y}), + std::max({corners[0].x, corners[1].x, corners[2].x, corners[3].x}), + std::max({corners[0].y, corners[1].y, corners[2].y, corners[3].y}) + ); +} +ImagePixelBox PaddleOCRTextBox::pixel_box(size_t image_width, size_t image_height) const{ + const Rectangle e = extent(); + auto to_pixel = [](float v){ + return (size_t)std::max(0.0f, v); + }; + ImagePixelBox box( + to_pixel(std::floor(e.min_x)), + to_pixel(std::floor(e.min_y)), + to_pixel(std::ceil(e.max_x)), + to_pixel(std::ceil(e.max_y)) + ); + box.clip(image_width, image_height); + box.min_x = std::min(box.min_x, box.max_x); + box.min_y = std::min(box.min_y, box.max_y); + return box; +} + +namespace{ + +cv::Rect to_cv_rect(const ImagePixelBox& box){ + return cv::Rect((int)box.min_x, (int)box.min_y, (int)box.width(), (int)box.height()); +} + +cv::Point2f to_cv_point(const PaddleOCRPoint& p){ + return cv::Point2f(p.x, p.y); +} + +} + + + +namespace{ + +// Order the four corners of a rotated rectangle as top-left, top-right, bottom-right, +// bottom-left. This is PaddleOCR's `get_mini_boxes()` ordering: sort by x, then the two +// leftmost points are split into top/bottom by y, and likewise for the two rightmost. +std::array ordered_corners(const cv::RotatedRect& rect){ + cv::Point2f pts[4]; + rect.points(pts); + std::sort(pts, pts + 4, [](const cv::Point2f& a, const cv::Point2f& b){ + return a.x < b.x; + }); + cv::Point2f tl = pts[0], bl = pts[1]; + if (tl.y > bl.y){ + std::swap(tl, bl); + } + cv::Point2f tr = pts[2], br = pts[3]; + if (tr.y > br.y){ + std::swap(tr, br); + } + return { + PaddleOCRPoint{tl.x, tl.y}, + PaddleOCRPoint{tr.x, tr.y}, + PaddleOCRPoint{br.x, br.y}, + PaddleOCRPoint{bl.x, bl.y}, + }; +} + +// Mean of the probability map inside the polygon `corners`. This is PaddleOCR's +// `box_score_fast()`: rasterize the polygon as a mask over its bounding rectangle and +// average the probabilities under the mask. +float box_score(const cv::Mat& prob, const std::array& corners){ + float min_x = corners[0].x, max_x = corners[0].x; + float min_y = corners[0].y, max_y = corners[0].y; + for (const PaddleOCRPoint& p : corners){ + min_x = std::min(min_x, p.x); + max_x = std::max(max_x, p.x); + min_y = std::min(min_y, p.y); + max_y = std::max(max_y, p.y); + } + int x0 = std::clamp((int)std::floor(min_x), 0, prob.cols - 1); + int x1 = std::clamp((int)std::ceil(max_x), 0, prob.cols - 1); + int y0 = std::clamp((int)std::floor(min_y), 0, prob.rows - 1); + int y1 = std::clamp((int)std::ceil(max_y), 0, prob.rows - 1); + + cv::Mat mask = cv::Mat::zeros(y1 - y0 + 1, x1 - x0 + 1, CV_8UC1); + std::vector poly; + for (const PaddleOCRPoint& p : corners){ + poly.emplace_back((int)std::round(p.x) - x0, (int)std::round(p.y) - y0); + } + cv::fillPoly(mask, std::vector>{poly}, cv::Scalar(1)); + + cv::Mat roi = prob(cv::Rect(x0, y0, x1 - x0 + 1, y1 - y0 + 1)); + return (float)cv::mean(roi, mask)[0]; +} + +// Expand a rotated rectangle outward by `distance = area * unclip_ratio / perimeter`. +// PaddleOCR does this with a polygon offset (pyclipper) followed by `minAreaRect()`. +// For a rectangle input, offsetting every edge outward by `distance` and taking the +// minimum bounding rectangle of the rounded result is exactly a rectangle with both +// sides grown by `2 * distance`, so we can compute it directly. +cv::RotatedRect unclip(const cv::RotatedRect& rect, float unclip_ratio){ + float w = rect.size.width; + float h = rect.size.height; + float area = w * h; + float perimeter = 2 * (w + h); + if (perimeter <= 0){ + return rect; + } + float distance = area * unclip_ratio / perimeter; + return cv::RotatedRect(rect.center, cv::Size2f(w + 2 * distance, h + 2 * distance), rect.angle); +} + +// Angle of the top edge of the box in degrees. 0 for axis-aligned text. +double box_angle_degrees(const PaddleOCRTextBox& box){ + const PaddleOCRPoint& tl = box.corners[0]; + const PaddleOCRPoint& tr = box.corners[1]; + return std::atan2(tr.y - tl.y, tr.x - tl.x) * 180.0 / CV_PI; +} + +// Boxes rotated by less than this are treated as axis-aligned. +constexpr double AXIS_ALIGNED_MAX_ANGLE = 5.0; + +// Round `x` to the nearest multiple of 32 (DBNet downsamples by 32), at least 32. +int round_to_32(double x){ + return std::max(32, (int)std::round(x / 32.0) * 32); +} + +} // anonymous namespace + + + +PaddleOCRDetector::PaddleOCRDetector(const std::string& model_path, PaddleOCRDetectorParams params) + : m_params(params) + , m_session( + create_session( + model_path, + ML_MODEL_CACHE_PATH() + "PaddleOCRDetector/", + PerformanceOptions::instance().ONNX_OPTIONS.USE_GPU + ) + ) + , m_input_name(m_session.GetInputNameAllocated(0, Ort::AllocatorWithDefaultOptions{}).get()) + , m_output_name(m_session.GetOutputNameAllocated(0, Ort::AllocatorWithDefaultOptions{}).get()) +{} + + +std::vector PaddleOCRDetector::detect(const cv::Mat& image_bgr) const{ + if (image_bgr.empty() || image_bgr.type() != CV_8UC3){ + return {}; + } + + const int image_w = image_bgr.cols; + const int image_h = image_bgr.rows; + + // 1. Pad the border with the background color so text touching the edge of a + // pre-cropped box is still detected. + cv::Mat padded; + const int pad = std::max(0, m_params.border_padding); + if (pad > 0){ + cv::copyMakeBorder( + image_bgr, padded, pad, pad, pad, pad, + cv::BORDER_CONSTANT, estimate_background_color(image_bgr) + ); + }else{ + padded = image_bgr; + } + + // 2. Resize so that the shorter side >= min_side_len and the longer side <= + // max_side_len, with both sides rounded to multiples of 32. + double ratio = 1.0; + const int short_side = std::min(padded.cols, padded.rows); + const int long_side = std::max(padded.cols, padded.rows); + if (short_side < m_params.min_side_len){ + ratio = (double)m_params.min_side_len / short_side; + } + if (long_side * ratio > m_params.max_side_len){ + ratio = (double)m_params.max_side_len / long_side; + } + const int input_w = round_to_32(padded.cols * ratio); + const int input_h = round_to_32(padded.rows * ratio); + + cv::Mat resized; + cv::resize(padded, resized, cv::Size(input_w, input_h), 0, 0, cv::INTER_LINEAR); + + // 3. Normalize exactly the way PaddleOCR prepared images when it trained this model: + // BGR channel order, scaled to [0, 1], then the ImageNet mean/std per channel. + // + // NOTE: The channel order mismatch below is INTENTIONAL. The mean/std are the + // standard ImageNet values, which are defined in RGB order (R: 0.485/0.229, + // G: 0.456/0.224, B: 0.406/0.225). But PaddleOCR decodes images with OpenCV, + // which gives BGR, never converts them to RGB, and then applies these values to + // the channels in the order listed. So during training the blue channel was + // normalized with the red mean/std and vice versa. The model's weights were + // learned from inputs prepared that way, so we must do the same: our input image + // is already BGR, and we keep the mean/std in the listed order. "Fixing" it to + // proper RGB normalization would feed the model inputs it never saw during training. + // (The effect is small because the three means/stds are close to each other.) + // + // Evidence (links pinned to the versions that were checked): + // - PP-OCRv5 det training and eval config: `DecodeImage` with `img_mode: BGR`, + // then `NormalizeImage` with mean [0.485, 0.456, 0.406], std + // [0.229, 0.224, 0.225] and `order: hwc`: + // https://github.com/PaddlePaddle/PaddleOCR/blob/dab3fe35379033fdcb2d0e9572fac0b36c9a9ebf/configs/det/PP-OCRv5/PP-OCRv5_server_det.yml#L78-L123 + // - `DecodeImage` decodes with `cv2.imdecode()` (BGR) and only reverses the + // channels when `img_mode` is RGB. `NormalizeImage` computes + // `(img * scale - mean) / std` with the mean/std laid out per channel (HWC): + // https://github.com/PaddlePaddle/PaddleOCR/blob/dab3fe35379033fdcb2d0e9572fac0b36c9a9ebf/ppocr/data/imaug/operators.py#L31-L94 + // - The inference config shipped with the official model has the same + // `img_mode: BGR` and mean/std: + // https://huggingface.co/PaddlePaddle/PP-OCRv5_server_det/blob/ca867c897ecbca8873081573a802ad70d499cb94/inference.yml + // - PaddleX, the runtime PaddleOCR 3.x uses, replaces its default RGB reader with + // `ReadImage(format="BGR")` from that inference.yml, then normalizes channel c + // with mean[c] and std[c]: + // https://github.com/PaddlePaddle/PaddleX/blob/c50f5da858020db473a2285f089bb8c7bbd6afdc/paddlex/inference/models/text_detection/predictor.py#L107-L164 + // https://github.com/PaddlePaddle/PaddleX/blob/c50f5da858020db473a2285f089bb8c7bbd6afdc/paddlex/inference/models/text_detection/processors.py#L236-L274 + // + // The config.json next to the ONNX model we use (monkt/paddleocr-onnx) describes + // the preprocessing as "RGB image, normalized to [0, 1]". That's inaccurate: the + // ONNX conversion doesn't change what input the model expects, so PaddleOCR's own + // configs above are the authority. + // + // Normalization and the HWC -> NCHW conversion are done together, straight into + // the input buffer: split the image into its B, G, R planes, then `convertTo()` + // each plane into its slice of `input_values` as ((pixel / 255) - mean) / std. + // `convertTo()` writes into a destination that already has the right size and + // type instead of allocating a new one. + static constexpr double MEAN[3] = {0.485, 0.456, 0.406}; + static constexpr double STD[3] = {0.229, 0.224, 0.225}; + const size_t plane_size = (size_t)input_h * input_w; + std::vector input_values(3 * plane_size); + cv::Mat channels[3]; + cv::split(resized, channels); + for (int c = 0; c < 3; c++){ + float* plane_data = input_values.data() + c * plane_size; + cv::Mat plane(input_h, input_w, CV_32FC1, plane_data); + channels[c].convertTo(plane, CV_32F, 1.0 / (255.0 * STD[c]), -MEAN[c] / STD[c]); + if (plane.data != (uchar*)plane_data){ + // OpenCV allocated a new destination, so `input_values` was not written. + throw InternalProgramError( + nullptr, PA_CURRENT_FUNCTION, + "PaddleOCRDetector::detect(): convertTo() did not write into the input buffer." + ); + } + } + + // 4. Run the model. + std::vector input_shape = {1, 3, input_h, input_w}; + + Ort::MemoryInfo memory_info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); + Ort::Value input_tensor = create_tensor(memory_info, input_values, input_shape); + + const char* input_names[] = {m_input_name.c_str()}; + const char* output_names[] = {m_output_name.c_str()}; + + std::vector outputs; + try{ + outputs = m_session.Run( + Ort::RunOptions{nullptr}, + input_names, &input_tensor, 1, + output_names, 1 + ); + }catch (Ort::Exception& e){ + throw InternalProgramError( + nullptr, PA_CURRENT_FUNCTION, + "PaddleOCRDetector::detect(): Failed. " + std::string(e.what()) + ); + } + + // 5. The output is a text probability map of shape {1, 1, H, W}. + std::vector output_shape = outputs[0].GetTensorTypeAndShapeInfo().GetShape(); + if (output_shape.size() != 4 || output_shape[2] != input_h || output_shape[3] != input_w){ + throw InternalProgramError( + nullptr, PA_CURRENT_FUNCTION, + "PaddleOCRDetector::detect(): Unexpected output shape " + to_string(output_shape) + + " for input shape " + to_string(input_shape) + ); + } + cv::Mat prob(input_h, input_w, CV_32FC1, outputs[0].GetTensorMutableData()); + + // Map model input pixels -> padded image pixels -> original image pixels. + const float scale_x = (float)padded.cols / input_w; + const float scale_y = (float)padded.rows / input_h; + std::vector boxes = postprocess( + prob, scale_x, scale_y, + (float)pad, + image_w, image_h + ); + + if (STATIC_GLOBALS.PADDLE_OCR_DEBUG){ + global_logger_tagged().log( + "[OCR-DET-DEBUG] Input " + std::to_string(image_w) + "x" + std::to_string(image_h) + + " -> model " + std::to_string(input_w) + "x" + std::to_string(input_h) + + ", found " + std::to_string(boxes.size()) + " text boxes." + ); + } + return boxes; +} + + +std::vector PaddleOCRDetector::postprocess( + const cv::Mat& prob, + float scale_x, float scale_y, + float offset, + int image_width, int image_height +) const{ + // 1. Binarize the probability map. + cv::Mat bitmap; + cv::threshold(prob, bitmap, m_params.binary_threshold, 255, cv::THRESH_BINARY); + bitmap.convertTo(bitmap, CV_8UC1); + + // 2. Each connected text blob becomes a candidate. + std::vector> contours; + cv::findContours(bitmap, contours, cv::RETR_LIST, cv::CHAIN_APPROX_SIMPLE); + + std::vector boxes; + const size_t num_candidates = std::min(contours.size(), m_params.max_candidates); + for (size_t i = 0; i < num_candidates; i++){ + const std::vector& contour = contours[i]; + if (contour.size() < 3){ + continue; + } + + // 3. Fit a rotated rectangle and score it against the probability map. + cv::RotatedRect rect = cv::minAreaRect(contour); + if (std::min(rect.size.width, rect.size.height) < m_params.min_box_size){ + continue; + } + float score = box_score(prob, ordered_corners(rect)); + if (score < m_params.box_threshold){ + continue; + } + + // 4. Expand the shrunk text kernel to cover the full text. + cv::RotatedRect expanded = unclip(rect, m_params.unclip_ratio); + if (std::min(expanded.size.width, expanded.size.height) < m_params.min_box_size + 2){ + continue; + } + + // 5. Map back to the original image and clip. + PaddleOCRTextBox box; + box.score = score; + std::array corners = ordered_corners(expanded); + for (size_t c = 0; c < 4; c++){ + float x = corners[c].x * scale_x - offset; + float y = corners[c].y * scale_y - offset; + box.corners[c].x = std::clamp(x, 0.0f, (float)image_width); + box.corners[c].y = std::clamp(y, 0.0f, (float)image_height); + } + + // Skip boxes that ended up entirely inside the padding. + ImagePixelBox bounds = box.pixel_box(image_width, image_height); + if (bounds.width() < 2 || bounds.height() < 2){ + continue; + } + boxes.emplace_back(box); + } + return boxes; +} + + + +std::vector> group_text_boxes_into_lines(const std::vector& boxes){ + std::vector order(boxes.size()); + std::iota(order.begin(), order.end(), 0); + std::sort(order.begin(), order.end(), [&](size_t a, size_t b){ + Rectangle ea = boxes[a].extent(); + Rectangle eb = boxes[b].extent(); + return ea.min_y + ea.max_y < eb.min_y + eb.max_y; + }); + + struct Line{ + float min_y; + float max_y; + std::vector members; + }; + std::vector lines; + for (size_t index : order){ + const PaddleOCRTextBox& box = boxes[index]; + const Rectangle extent = box.extent(); + const float box_min = extent.min_y; + const float box_max = extent.max_y; + const float box_h = box_max - box_min; + + // Find the line with the most vertical overlap. + Line* best = nullptr; + float best_overlap = 0; + for (Line& line : lines){ + float overlap = std::min(box_max, line.max_y) - std::max(box_min, line.min_y); + float min_h = std::min(box_h, line.max_y - line.min_y); + if (min_h > 0 && overlap >= 0.5f * min_h && overlap > best_overlap){ + best = &line; + best_overlap = overlap; + } + } + if (best == nullptr){ + lines.push_back(Line{box_min, box_max, {index}}); + }else{ + best->members.push_back(index); + best->min_y = std::min(best->min_y, box_min); + best->max_y = std::max(best->max_y, box_max); + } + } + + std::sort(lines.begin(), lines.end(), [](const Line& a, const Line& b){ + return a.min_y + a.max_y < b.min_y + b.max_y; + }); + + std::vector> ret; + for (Line& line : lines){ + std::sort(line.members.begin(), line.members.end(), [&](size_t a, size_t b){ + return boxes[a].extent().min_x < boxes[b].extent().min_x; + }); + ret.emplace_back(std::move(line.members)); + } + return ret; +} + + +std::vector merge_text_boxes_in_line( + const std::vector& line_boxes, + float max_gap_ratio +){ + if (line_boxes.empty()){ + return {}; + } + + Rectangle line_extent = line_boxes[0].extent(); + for (const PaddleOCRTextBox& box : line_boxes){ + line_extent.merge_with(box.extent()); + } + const float line_min_y = line_extent.min_y; + const float line_max_y = line_extent.max_y; + const float max_gap = max_gap_ratio * (line_max_y - line_min_y); + + // A segment being built: its horizontal extent and the sum of the box scores. + struct Segment{ + float min_x; + float max_x; + float score_sum; + size_t count; + }; + std::vector ret; + std::optional current; + auto flush = [&](){ + if (!current){ + return; + } + PaddleOCRTextBox merged; + merged.corners = { + PaddleOCRPoint{current->min_x, line_min_y}, + PaddleOCRPoint{current->max_x, line_min_y}, + PaddleOCRPoint{current->max_x, line_max_y}, + PaddleOCRPoint{current->min_x, line_max_y}, + }; + merged.score = current->score_sum / current->count; + ret.emplace_back(merged); + current.reset(); + }; + + for (const PaddleOCRTextBox& box : line_boxes){ + if (std::abs(box_angle_degrees(box)) >= AXIS_ALIGNED_MAX_ANGLE){ + flush(); + ret.emplace_back(box); + continue; + } + const Rectangle extent = box.extent(); + if (current && extent.min_x - current->max_x <= max_gap){ + current->max_x = std::max(current->max_x, extent.max_x); + current->score_sum += box.score; + current->count++; + continue; + } + flush(); + current = Segment{extent.min_x, extent.max_x, box.score, 1}; + } + flush(); + return ret; +} + + +cv::Mat crop_text_box(const cv::Mat& image_bgr, const PaddleOCRTextBox& box){ + const cv::Point2f tl = to_cv_point(box.corners[0]); + const cv::Point2f tr = to_cv_point(box.corners[1]); + const cv::Point2f br = to_cv_point(box.corners[2]); + const cv::Point2f bl = to_cv_point(box.corners[3]); + + // Game UI text is axis-aligned, but `minAreaRect()` on pixel text blobs often returns + // a box rotated by a degree or so. Warping by that tiny angle resamples the image + // and blurs sharp pixel fonts (e.g. FRLG), which noticeably hurts recognition. + // So if the box is nearly axis-aligned, crop its bounding rectangle directly. + if (std::abs(box_angle_degrees(box)) < AXIS_ALIGNED_MAX_ANGLE){ + ImagePixelBox rect = box.pixel_box(image_bgr.cols, image_bgr.rows); + if (rect.width() < 1 || rect.height() < 1){ + return {}; + } + return image_bgr(to_cv_rect(rect)).clone(); + } + + int crop_w = (int)std::round(std::max(cv::norm(tl - tr), cv::norm(bl - br))); + int crop_h = (int)std::round(std::max(cv::norm(tl - bl), cv::norm(tr - br))); + if (crop_w < 1 || crop_h < 1){ + return {}; + } + + const cv::Point2f src[4] = {tl, tr, br, bl}; + const cv::Point2f dst[4] = { + {0, 0}, + {(float)crop_w, 0}, + {(float)crop_w, (float)crop_h}, + {0, (float)crop_h}, + }; + cv::Mat transform = cv::getPerspectiveTransform(src, dst); + + cv::Mat cropped; + cv::warpPerspective( + image_bgr, cropped, transform, cv::Size(crop_w, crop_h), + cv::INTER_CUBIC, cv::BORDER_REPLICATE + ); + return cropped; +} + + + +} +} diff --git a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRDetector.h b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRDetector.h new file mode 100644 index 0000000000..a72e81e31c --- /dev/null +++ b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRDetector.h @@ -0,0 +1,158 @@ +/* ML Paddle OCR Detector + * + * From: https://github.com/PokemonAutomation/ + * + * Text detection half of the PaddleOCR pipeline. Given an arbitrary image, it finds + * where the lines of text are. Each detected region can then be cropped out and fed + * into the recognition model (see `PaddleOCRPipeline`). + * + * The detection model is PP-OCRv5 det, a DBNet ("Differentiable Binarization") model. + * Model source: https://huggingface.co/monkt/paddleocr-onnx/tree/main/detection/v5 + * The same detection model is shared by all languages. + */ + +#ifndef PokemonAutomation_ML_PaddleOCRDetector_H +#define PokemonAutomation_ML_PaddleOCRDetector_H + +#include +#include +#include +#include +#include "Common/Cpp/Rectangle.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" + +namespace cv{ + class Mat; +} + +namespace PokemonAutomation{ +namespace ML{ + + +// Tunable parameters of the DBNet detection pre/post-processing. +// Defaults follow PaddleOCR's PP-OCRv5 `DetResizeForTest` and `DBPostProcess` configs, +// except where noted. +struct PaddleOCRDetectorParams{ + // Preprocessing: if the shorter image side is smaller than this, upscale the image + // so that the shorter side matches it. Tiny images give the model too few pixels + // to find text in. + int min_side_len = 64; + // Preprocessing: if the longer image side is larger than this, downscale the image + // so that the longer side matches it. This bounds the inference cost for large + // images such as full 4K screenshots. + int max_side_len = 2048; + // Preprocessing: pad the image by this many pixels on each side (in the original + // image scale) using the estimated background color. DBNet tends to miss text that + // touches the image border, which is common for our pre-cropped OCR boxes. + // (Not part of PaddleOCR.) + int border_padding = 8; + + // Postprocessing: pixels whose text probability is above this are considered text. + float binary_threshold = 0.3f; + // Postprocessing: drop candidate boxes whose mean probability is below this. + float box_threshold = 0.6f; + // Postprocessing: how much to expand the shrunk text kernel back out. + // DBNet is trained to predict a shrunk version of each text region, so the found + // boxes must be "unclipped" (expanded) to cover the whole text. + float unclip_ratio = 1.5f; + // Postprocessing: drop boxes whose shorter side is smaller than this (in pixels of + // the model input). + float min_box_size = 3.0f; + // Postprocessing: maximum number of contours to consider. + size_t max_candidates = 1000; +}; + + +// A point in (sub-)pixel image coordinates: x from left to right, y from top to bottom. +struct PaddleOCRPoint{ + float x = 0; + float y = 0; +}; + +// One detected text region. +struct PaddleOCRTextBox{ + // Four corners of the (possibly slightly rotated) text rectangle in pixel + // coordinates of the original input image. Order: top-left, top-right, + // bottom-right, bottom-left. + std::array corners; + // Mean text probability inside the box, in [0, 1]. + float score = 0; + + // Axis-aligned extent of `corners`, in sub-pixel image coordinates. + Rectangle extent() const; + + // Smallest pixel box enclosing `corners`, clipped to the image size. + ImagePixelBox pixel_box(size_t image_width, size_t image_height) const; +}; + + +class PaddleOCRDetector{ +public: + // Load the detection model from `model_path`. + // Throws `MLModelSessionCreationError` if the ONNX session cannot be created. + PaddleOCRDetector(const std::string& model_path, PaddleOCRDetectorParams params = {}); + + // Detect text regions in a BGR image (CV_8UC3). Returns boxes in unspecified order. + // Returns an empty vector if the image is empty or contains no text. + // Thread-safe: ONNX Runtime sessions can be run concurrently. + // Throws InternalProgramError if ONNX Runtime fails. + std::vector detect(const cv::Mat& image_bgr) const; + + const PaddleOCRDetectorParams& params() const{ return m_params; } + +private: + // Run the DBNet postprocessing on the model's output probability map. + // `prob` is a CV_32F map of the model input size. `scale_x`, `scale_y` map model + // input pixel coordinates back to original image coordinates, after subtracting + // `offset` (the border padding, in original image pixels). + std::vector postprocess( + const cv::Mat& prob, + float scale_x, float scale_y, + float offset, + int image_width, int image_height + ) const; + +private: + PaddleOCRDetectorParams m_params; + mutable Ort::Session m_session; + std::string m_input_name; + std::string m_output_name; +}; + + +// Sort detected text boxes into reading order and group them into lines. +// Boxes whose vertical extents overlap by at least half of the shorter box height are +// put into the same line. Lines are ordered top to bottom, boxes within a line left to +// right. Returns indices into `boxes`: result[line][k]. +std::vector> group_text_boxes_into_lines(const std::vector& boxes); + +// Merge the boxes of one text line (`line_boxes`, sorted left to right) into +// horizontal segments. Neighboring boxes are merged when the horizontal gap between +// them is at most `max_gap_ratio` times the line height. Each merged segment is an +// axis-aligned box spanning the full height of the line. +// +// Why: DBNet often splits a line into separate words, especially with widely spaced +// pixel fonts like FRLG. Each word box then has its own height: "see" only covers the +// lowercase x-height while "BULBASAUR" is taller. After scaling every crop to the +// recognition model's fixed input height, short words get oversized glyphs and are +// misread (e.g. "see" -> "SPP"). Recognizing the whole line at a consistent height +// fixes this, and the recognition model outputs the spaces between words itself. +// Distant boxes (e.g. separate table columns) are kept apart. +// +// Rotated boxes (more than a few degrees) are never merged, and are returned as is. +std::vector merge_text_boxes_in_line( + const std::vector& line_boxes, + float max_gap_ratio = 1.0f +); + +// Crop the text region `box` from `image_bgr` and warp it into an axis-aligned +// rectangle, so slightly rotated text becomes horizontal. This is PaddleOCR's +// `get_rotate_crop_image()`, minus the auto 90-degree rotation of tall crops (game UI +// text is never vertical, while narrow single characters like "1" would be wrongly +// rotated). Returns an empty Mat if the box is degenerate. +cv::Mat crop_text_box(const cv::Mat& image_bgr, const PaddleOCRTextBox& box); + + +} +} +#endif diff --git a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp index 7388404abb..b7cb670f0a 100644 --- a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp +++ b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp @@ -56,11 +56,22 @@ std::pair PaddleOCRPipeline::get_paths(Language langua } PaddleOCRPipeline::PaddleOCRPipeline(Language language) - : PaddleOCRPipeline(language, get_paths(language).first, get_paths(language).second) + : PaddleOCRPipeline( + language, + // Shared by all languages. This is the downloadable resource "PaddleOCRDetection", + // see Packages/Resources/ResourceDownloadList.json. + DOWNLOADED_RESOURCE_PATH() + "PaddleOCRDetection/det.onnx", + get_paths(language).first, + get_paths(language).second + ) {} -PaddleOCRPipeline::PaddleOCRPipeline(Language language, std::string rec_path, std::string dict_path) - // , det_session(env, std::wstring(det_path.begin(), det_path.end()).c_str(), Ort::SessionOptions{}) +PaddleOCRPipeline::PaddleOCRPipeline( + Language language, + std::string det_path, + std::string rec_path, + std::string dict_path +) : m_rec_session( create_session( rec_path, @@ -68,29 +79,33 @@ PaddleOCRPipeline::PaddleOCRPipeline(Language language, std::string rec_path, st PerformanceOptions::instance().ONNX_OPTIONS.USE_GPU ) ) - // , memory_info(Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault)) + , m_language(language) , m_input_name(m_rec_session.GetInputNameAllocated(0, Ort::AllocatorWithDefaultOptions{}).get()) , m_output_name(m_rec_session.GetOutputNameAllocated(0, Ort::AllocatorWithDefaultOptions{}).get()) , m_logger(global_logger_raw(), "OCR") + , m_det_path(std::move(det_path)) { load_dictionary(Filesystem::Path(dict_path)); } -void PaddleOCRPipeline::run(const std::string& img_path){ -#if 0 - cv::Mat img = cv::imread(img_path); - if (img.empty()) return; - - // 1. Text Detection (simplified for brevity) - // In practice, use DBPostProcess to get boxes from detection output - std::vector boxes = {{10, 10, 100, 30}}; // Mock detected box - - for (auto& box : boxes){ - cv::Mat cropped = img(box); - std::string text = recognize(cropped); - std::cout << "Detected Text: " << text << std::endl; +PaddleOCRDetector& PaddleOCRPipeline::detector(){ + std::lock_guard lg(m_detector_lock); + if (!m_detector){ + // The detection model is a downloadable resource, so it may be missing. + // Programs that use multi-line OCR should list "PaddleOCRDetection" in their + // descriptor's required resources so it's downloaded before they start. + if (!Filesystem::exists(Filesystem::Path(m_det_path))){ + throw FileException( + &m_logger, PA_CURRENT_FUNCTION, + "PaddleOCR text detection model not found. " + "Please download the \"PaddleOCRDetection\" resource from the Settings panel.", + m_det_path + ); + } + m_logger.log("Loading PaddleOCR text detection model: " + m_det_path); + m_detector = std::make_unique(m_det_path); } -#endif + return *m_detector; } @@ -150,8 +165,6 @@ void PaddleOCRPipeline::load_dictionary(const Filesystem::Path& path){ std::string PaddleOCRPipeline::recognize(const ImageViewRGB32& image){ - - const bool debugging = STATIC_GLOBALS.PADDLE_OCR_DEBUG; const int debug_image_index = m_debug_image_index.fetch_add(1, std::memory_order_relaxed) + 1; if (image.width() == 0 || image.height() == 0) { @@ -171,6 +184,103 @@ std::string PaddleOCRPipeline::recognize(const ImageViewRGB32& image){ } + return recognize_line(cropped_image); +} + + +std::vector PaddleOCRPipeline::detect_and_recognize(const ImageViewRGB32& image){ + const int debug_image_index = m_debug_image_index.fetch_add(1, std::memory_order_relaxed) + 1; + + // Same as `recognize()`: drop alpha from our BGRA pixels into a BGR image. + if (image.width() == 0 || image.height() == 0){ + return {}; + } + cv::Mat image_bgr; + cv::cvtColor(to_OpenCV_ref(image), image_bgr, cv::COLOR_BGRA2BGR); + + // 1. Find all text regions. + std::vector boxes = detector().detect(image_bgr); + + // 2. Put them in reading order. + std::vector> lines = group_text_boxes_into_lines(boxes); + + // 3. Recognize each region. Words of the same line that are close together are + // merged and recognized as one piece. See `merge_text_boxes_in_line()`. + std::vector results; + for (size_t line_index = 0; line_index < lines.size(); line_index++){ + std::vector line_boxes; + for (size_t box_index : lines[line_index]){ + line_boxes.emplace_back(boxes[box_index]); + } + for (const PaddleOCRTextBox& box : merge_text_boxes_in_line(line_boxes)){ + cv::Mat crop = crop_text_box(image_bgr, box); + if (crop.empty()){ + continue; + } + // Same as `recognize()`: pad narrow crops (e.g. a lone "1") so the + // recognition model does not stretch them. + add_horizontal_padding(crop, debug_image_index); + + PaddleOCRTextResult result; + result.text = recognize_line(crop); + result.box = box; + result.line_index = line_index; + if (STATIC_GLOBALS.PADDLE_OCR_DEBUG){ + ImagePixelBox r = box.pixel_box(image_bgr.cols, image_bgr.rows); + m_logger.log( + "[OCR-DET-DEBUG] Line " + std::to_string(line_index) + + ", box (" + std::to_string(r.min_x) + ", " + std::to_string(r.min_y) + ", " + + std::to_string(r.width()) + ", " + std::to_string(r.height()) + + "), score " + std::to_string(box.score) + ": '" + result.text + "'" + ); + } + results.emplace_back(std::move(result)); + } + } + return results; +} + + +std::string PaddleOCRPipeline::recognize_multiline(const ImageViewRGB32& image){ + // Chinese and Japanese don't put spaces between words. + const char* word_separator = " "; + switch (m_language){ + case Language::ChineseSimplified: + case Language::ChineseTraditional: + case Language::Japanese: + word_separator = ""; + break; + default:; + } + + std::string text; + size_t current_line = 0; + bool line_empty = true; + for (const PaddleOCRTextResult& result : detect_and_recognize(image)){ + if (result.text.empty()){ + continue; + } + if (!text.empty()){ + if (result.line_index != current_line){ + text += "\n"; + line_empty = true; + } + if (!line_empty){ + text += word_separator; + } + } + text += result.text; + current_line = result.line_index; + line_empty = false; + } + return text; +} + + +std::string PaddleOCRPipeline::recognize_line(const cv::Mat& cropped_image){ + + const bool debugging = STATIC_GLOBALS.PADDLE_OCR_DEBUG; + // 2. Calculate dynamic width (maintain aspect ratio) // the model shape is {1, 3, 48, dynamic_width}. Note that the height is fixed at 48 pixels // the input image must be scaled to match the height of 48, for the neural network @@ -320,7 +430,10 @@ cv::Mat crop_to_text_region_with_padding(const cv::Mat& image, int debug_image_i int right_gap = image.cols - (bbox.x + bbox.width); // calculate the desired padding - int pad_x = std::max(4, bbox.width / 15); // ~5-10% + // Horizontal padding is also at least half the text height. With a too-tight crop, the + // recognition model tends to drop a thin character at the edge, e.g. "Lv. 1" read + // as "Lv." (PokemonHome/SummaryScreen/squirtle_Shiny.png). + int pad_x = std::max({4, bbox.width / 15, bbox.height / 2}); // ~5-10% int pad_y = std::max(2, bbox.height / 15); // ~5-10% diff --git a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.h b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.h index a0907453bd..8a68c401ed 100644 --- a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.h +++ b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.h @@ -9,29 +9,81 @@ #define PokemonAutomation_ML_PaddleOCRPipeline_H #include +#include #include #include #include #include +#include "Common/Cpp/Concurrency/Mutex.h" #include "Common/Cpp/Logging/TaggedLogger.h" #include "Common/Cpp/Filesystem/FilePath.h" #include "CommonFramework/Language.h" #include "CommonFramework/ImageTypes/ImageViewRGB32.h" #include "CommonFramework/ImageTools/ImageBoxes.h" +#include "ML_PaddleOCRDetector.h" namespace PokemonAutomation{ namespace ML{ +// One piece of text found by `PaddleOCRPipeline::detect_and_recognize()`. +struct PaddleOCRTextResult{ + // Recognized text of this box. May be empty if recognition found nothing. + std::string text; + // Where the text is, in pixel coordinates of the input image. + PaddleOCRTextBox box; + // Index of the text line this box belongs to, counting from the top (0-based). + // Several boxes can share one line, e.g. two words far apart on the same row. + size_t line_index = 0; +}; + + +// Full PaddleOCR pipeline: text detection (PP-OCRv5 det, DBNet) + text recognition +// (PP-OCRv5 rec, CTC). +// +// There are two ways to use it: +// - `recognize()`: single-line OCR. Assumes the image is already cropped around one +// line of text. It skips the detection model and instead tightens and pads the crop +// using a simple Otsu threshold (`crop_to_text_region_with_padding()`). This is fast +// and is what the majority of our game readers use. +// - `detect_and_recognize()` / `recognize_multiline()`: arbitrary images containing +// any number of lines. Runs the detection model to find each text region, then runs +// the recognition model on each region. +// +// The detection model file is the same for all languages. It is loaded lazily on first +// use, so single-line only users never pay for it. +// All public methods are thread-safe. class PaddleOCRPipeline{ public: PaddleOCRPipeline(Language language); - PaddleOCRPipeline(Language language, std::string rec_path, std::string dict_path); + PaddleOCRPipeline( + Language language, + std::string det_path, + std::string rec_path, + std::string dict_path + ); + + // Single-line OCR on an image pre-cropped to one line of text. + std::string recognize(const ImageViewRGB32& image); - void run(const std::string& img_path); + // Multi-line OCR: detect all text regions in `image` and recognize each of them. + // Results are in reading order: top to bottom, then left to right within a line. + // Throws FileException if the detection model has not been downloaded, or + // `MLModelSessionCreationError` if it cannot be loaded. + std::vector detect_and_recognize(const ImageViewRGB32& image); - std::string recognize(const ImageViewRGB32& image); + // Multi-line OCR returning plain text: each detected line is one line of the + // result, joined by '\n'. Boxes on the same line are joined by a space, except + // for Chinese and Japanese which do not use spaces between words. + // e.g. a two line dialog box gives "You hurry to the Pokemon Center, shielding your\neyes from the rain." + std::string recognize_multiline(const ImageViewRGB32& image); + + // Get the text detector, loading the detection model if it's not loaded yet. + // The detection model is the downloadable resource "PaddleOCRDetection". + // Throws FileException if it has not been downloaded. + PaddleOCRDetector& detector(); + // Return the (recognition model path, dictionary path) for a language. static std::pair get_paths(Language language); std::string decode_CTC(float* data, const std::vector& shape, const std::vector& dict); @@ -39,9 +91,13 @@ class PaddleOCRPipeline{ private: void load_dictionary(const Filesystem::Path& path); - // Ort::Session det_session; + // Run the recognition model on a BGR image (CV_8UC3) that is already cropped + // tightly around a single line of text. Resizes it to the model height, prepares + // the input tensor, runs the model and CTC-decodes the output. + std::string recognize_line(const cv::Mat& line_bgr); + Ort::Session m_rec_session; - // Ort::MemoryInfo memory_info; + Language m_language; std::string m_input_name; std::string m_output_name; std::vector m_dictionary; @@ -51,6 +107,9 @@ class PaddleOCRPipeline{ // (see PADDLE_OCR_DEBUG_IMAGE). Atomic because `recognize()` runs on many threads. std::atomic m_debug_image_index{0}; + std::string m_det_path; + Mutex m_detector_lock; + std::unique_ptr m_detector; }; // assumes the input image is BGR diff --git a/SerialPrograms/Source/VideoGames/PokemonHome/Inference/PokemonHome_SummaryReader.cpp b/SerialPrograms/Source/VideoGames/PokemonHome/Inference/PokemonHome_SummaryReader.cpp index 6d4eafa9f2..260c03808d 100644 --- a/SerialPrograms/Source/VideoGames/PokemonHome/Inference/PokemonHome_SummaryReader.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonHome/Inference/PokemonHome_SummaryReader.cpp @@ -375,7 +375,7 @@ void add_tests_SummaryReader(UnitTestDatabase& database){ database.add("PokemonHome/SummaryScreen/rowlet_ShinyAlpha.png", 722, 764041, 31); database.add("PokemonHome/SummaryScreen/scovillain_Regular.png", 952, 493124, 41); database.add("PokemonHome/SummaryScreen/slitherWing_Shiny.png", 988, 402737, 100); - //database.add("PokemonHome/SummaryScreen/squirtle_Shiny.png", 7, 700052, 1); + database.add("PokemonHome/SummaryScreen/squirtle_Shiny.png", 7, 700052, 1); database.add("PokemonHome/SummaryScreen/tapuLele_Shiny.png", 786, 181130, 62); database.add("PokemonHome/SummaryScreen/tatsugiri_Regular.png", 978, 493124, 52); database.add("PokemonHome/SummaryScreen/teddiursa_Regular.png", 216, 333685, 29); diff --git a/SerialPrograms/Source/VideoGames/PokemonSV/Programs/Farming/PokemonSV_BBQSoloFarmer.cpp b/SerialPrograms/Source/VideoGames/PokemonSV/Programs/Farming/PokemonSV_BBQSoloFarmer.cpp index fa0c6408dd..1c874e32a1 100644 --- a/SerialPrograms/Source/VideoGames/PokemonSV/Programs/Farming/PokemonSV_BBQSoloFarmer.cpp +++ b/SerialPrograms/Source/VideoGames/PokemonSV/Programs/Farming/PokemonSV_BBQSoloFarmer.cpp @@ -39,7 +39,7 @@ BBQSoloFarmer_Descriptor::BBQSoloFarmer_Descriptor() AllowCommandsWhenRunning::DISABLE_COMMANDS, PanelDeprecation::NOT_DEPRECATED, { - "Tesseract", + "PaddleOCRDetection", } ) {} diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index f5e1ea9435..9e61ce0a15 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -824,8 +824,8 @@ file(GLOB LIBRARY_SOURCES Source/CommonTools/OCR/OCR_LargeDictionaryMatcher.h Source/CommonTools/OCR/OCR_NumberReader.cpp Source/CommonTools/OCR/OCR_NumberReader.h - Source/CommonTools/OCR/OCR_RawPaddleOCR.cpp - Source/CommonTools/OCR/OCR_RawPaddleOCR.h + Source/CommonTools/OCR/OCR_PaddleOCR.cpp + Source/CommonTools/OCR/OCR_PaddleOCR.h Source/CommonTools/OCR/OCR_RawTesseractOCR.cpp Source/CommonTools/OCR/OCR_RawTesseractOCR.h Source/CommonTools/OCR/OCR_Routines.cpp @@ -1171,6 +1171,8 @@ file(GLOB LIBRARY_SOURCES Source/ML/DataLabeling/ML_SegmentAnythingModel.cpp Source/ML/DataLabeling/ML_SegmentAnythingModel.h Source/ML/DataLabeling/ML_SegmentAnythingModelConstants.h + Source/ML/Inference/ML_PaddleOCRDetector.cpp + Source/ML/Inference/ML_PaddleOCRDetector.h Source/ML/Inference/ML_PaddleOCRPipeline.cpp Source/ML/Inference/ML_PaddleOCRPipeline.h Source/ML/Inference/ML_YOLONavigation.cpp