add Paddle OCR det model

.github/workflows/cpp-ci-serial-programs-base.yml

@@ -133,6 +133,26 @@ jobs:
           repository: 'PokemonAutomation/CommandLineTests'
           path: 'CommandLineTests'
 
+      # The unit tests assume the downloadable resources they need are already downloaded
+      # (see Packages/Resources/ResourceDownloadList.json). Unzip them from the Packages
+      # checkout into where the program looks for them, DOWNLOADED_RESOURCE_PATH():
+      # the working directory on Windows/Linux, and ~/Library/Application Support/SerialPrograms
+      # on macOS.
+      - name: Install downloadable resources for tests
+        if: inputs.run-tests
+        shell: bash
+        run: |
+          if [[ "${{inputs.os}}" == mac* ]]; then
+            DOWNLOADED_RESOURCES="$HOME/Library/Application Support/SerialPrograms/DownloadedResources"
+          else
+            DOWNLOADED_RESOURCES="Arduino-Source/SerialPrograms/bin/DownloadedResources"
+          fi
+          for RESOURCE in PaddleOCRDetection; do
+            mkdir -p "$DOWNLOADED_RESOURCES/$RESOURCE"
+            (cd "$DOWNLOADED_RESOURCES/$RESOURCE" && cmake -E tar xf "${{github.workspace}}/Arduino-Source/Packages/DownloadedResources/$RESOURCE.zip")
+            ls -la "$DOWNLOADED_RESOURCES/$RESOURCE"
+          done
+
       - name: Run tests (Windows)
         if: startsWith(inputs.os, 'windows') && inputs.run-tests
         run: |

SerialPrograms/Source/CommonTools/OCR/OCR_RawPaddleOCR.cpp → SerialPrograms/Source/CommonTools/OCR/OCR_PaddleOCR.cpp

@@ -9,7 +9,7 @@
 #include "Common/Cpp/Concurrency/SpinLock.h"
 #include "CommonFramework/ImageTypes/ImageViewRGB32.h"
 #include "ML/Inference/ML_PaddleOCRPipeline.h"
-#include "OCR_RawPaddleOCR.h"
+#include "OCR_PaddleOCR.h"
 
 namespace PokemonAutomation{
 namespace OCR{
@@ -97,15 +97,26 @@ ML::PaddleOCRPipeline& ensure_paddle_ocr_instance(Language language){
 }
 
 
-std::string paddle_ocr_read(Language language, const ImageViewRGB32& image){
+std::string paddle_ocr_read(Language language, const ImageViewRGB32& image, PageSegMode psm){
 //    static size_t c = 0;
 //    image.save("ocr-" + std::to_string(c++) + ".png");
 
     ML::PaddleOCRPipeline& paddle_instance = ensure_paddle_ocr_instance(language);
 
     // Run inference with the paddle model. 
     // PaddleOCR with Onnx is threadsafe, so a single instance can be called by multiple threads.
-    std::string ret = paddle_instance.recognize(image);
+    std::string ret;
+    switch (psm){
+    case PageSegMode::AUTO:
+    case PageSegMode::SINGLE_BLOCK:
+    case PageSegMode::SINGLE_COLUMN:
+        // Multi-line: run text detection first, then recognize each line.
+        ret = paddle_instance.recognize_multiline(image);
+        break;
+    default:
+        // Single line: the image is already cropped to the text, skip detection.
+        ret = paddle_instance.recognize(image);
+    }
 
 //    global_logger_tagged().log(ret);
 

SerialPrograms/Source/CommonTools/OCR/OCR_RawPaddleOCR.h → SerialPrograms/Source/CommonTools/OCR/OCR_PaddleOCR.h

@@ -8,11 +8,12 @@
 // This file should not be inluded in other files.
 // Use OCR_Routines.h instead
 
-#ifndef PokemonAutomation_CommonTools_OCR_RawPaddleOCR_H
-#define PokemonAutomation_CommonTools_OCR_RawPaddleOCR_H
+#ifndef PokemonAutomation_CommonTools_OCR_PaddleOCR_H
+#define PokemonAutomation_CommonTools_OCR_PaddleOCR_H
 
 #include <string>
 #include "CommonFramework/Language.h"
+#include "OCR_RawTesseractOCR.h"
 
 namespace PokemonAutomation{
     class ImageViewRGB32;
@@ -36,9 +37,18 @@ ML::PaddleOCRPipeline& ensure_paddle_ocr_instance(Language language);
 //  It creates one PaddleOCR instance for each language. You can
 //  call `ensure_instances()` to pre-warm to pool with a given number of instances.
 //
+//  psm: Page segmentation mode, same meaning as in `tesseract_ocr_read()`.
+//       Defaults to SINGLE_LINE.
+//     - AUTO, SINGLE_BLOCK, SINGLE_COLUMN: multi-line OCR. Runs the PaddleOCR text
+//       detection model to find every line of text in the image, then recognizes each
+//       line. Lines are returned top to bottom, separated by '\n'. Use this when the
+//       image is not pre-cropped to a single line, e.g. a whole dialog box.
+//     - Any other mode: single-line OCR. Assumes the image is already cropped to one
+//       line of text and skips the detection model.
 std::string paddle_ocr_read(
     Language language,
-    const ImageViewRGB32& image
+    const ImageViewRGB32& image,
+    PageSegMode psm = PageSegMode::SINGLE_LINE
 );
 
 

SerialPrograms/Source/CommonTools/OCR/OCR_Routines.cpp

@@ -9,7 +9,7 @@
 #include "CommonFramework/GlobalSettingsPanel.h"
 #include "CommonFramework/Exceptions/OperationFailedException.h"
 #include "CommonTools/Images/ImageFilter.h"
-#include "OCR_RawPaddleOCR.h"
+#include "OCR_PaddleOCR.h"
 #include "OCR_RawTesseractOCR.h"
 #include "OCR_DictionaryMatcher.h"
 #include "OCR_Routines.h"
@@ -34,31 +34,15 @@ bool ocr_language_available(Language language){
 
 
 std::string ocr_read(Language language, const ImageViewRGB32& image, PageSegMode psm){
-    std::string ocr_text = "";
-    if (psm == PageSegMode::AUTO || psm == PageSegMode::SINGLE_BLOCK || psm == PageSegMode::SINGLE_COLUMN){
-        // if using multiline detection, force Tesseract
-        ocr_text = OCR::tesseract_ocr_read(language, image, psm);
+    if (GlobalSettings::instance().OCR_LIBRARY == OcrLibrary::PADDLE_OCR){
+        return OCR::paddle_ocr_read(language, image, psm);
     }else{
-        if (GlobalSettings::instance().OCR_LIBRARY == OcrLibrary::PADDLE_OCR){
-            ocr_text = OCR::paddle_ocr_read(language, image);
-        }else{
-            ocr_text = OCR::tesseract_ocr_read(language, image, psm);
-        }
+        return OCR::tesseract_ocr_read(language, image, psm);
     }
-    return ocr_text;
 }
 
 bool allow_parallel_ocr(PageSegMode psm){
-    if (psm == PageSegMode::AUTO || psm == PageSegMode::SINGLE_BLOCK || psm == PageSegMode::SINGLE_COLUMN){
-        // if using multiline detection, force Tesseract
-        return true;
-    }else{
-        if (GlobalSettings::instance().OCR_LIBRARY == OcrLibrary::PADDLE_OCR){
-            return false;
-        }else{
-            return true;
-        }
-    }
+    return GlobalSettings::instance().OCR_LIBRARY != OcrLibrary::PADDLE_OCR;
 }
 
 void ensure_ocr_instances(Language language, size_t instances){

SerialPrograms/Source/CommonTools/OCR/OCR_Routines.h

@@ -32,9 +32,15 @@ struct TextColorRange{
 
 bool ocr_language_available(Language language);
 
-// will use the OCR library as per the dropdown in GlobalSettings
-// if the preferred library's resource is missing (i.e. it has not been downloaded),
+// Main OCR function for Pokemon automation.
+// It uses the OCR library as per the dropdown in GlobalSettings
+// If the preferred library's resource is missing (i.e. it has not been downloaded),
 // an error will be thrown within OCR initialization infra.
+// psm: text layout mode to help improve OCR model accuracy.
+//   with PaddleOCR, SINGLE_LINE (and other single-line modes) assume the image is
+//   already cropped to one line of text. AUTO, SINGLE_BLOCK and SINGLE_COLUMN run the
+//   PaddleOCR text detection model to find each line first, and return the lines
+//   separated by '\n'.
 std::string ocr_read(Language language, const ImageViewRGB32& image, PageSegMode psm = PageSegMode::SINGLE_LINE);
 
 //

SerialPrograms/Source/CommonTools/OCR/OCR_Tests.cpp

@@ -28,6 +28,7 @@ void add_tests(UnitTestDatabase& database){
     add_tests_raw_OCR(database);
     add_tests_number_waterfill_OCR(database);
     Pokemon::add_tests_PokemonNameReader(database);
+    add_tests_multiline_OCR(database);
 }
 
 class Test_RawOCR : public UnitTest{
@@ -128,5 +129,97 @@ void add_tests_number_waterfill_OCR(UnitTestDatabase& database){
 
 
 
+// Crop `box` out of a screenshot, run multi-line OCR on it, and check that it finds
+// exactly the expected lines in order. Each line is compared after
+// `normalize_utf32()`, which ignores punctuation and whitespace.
+class Test_MultilineOCR : public UnitTest{
+public:
+    Test_MultilineOCR(
+        const std::string& image,
+        const ImagePixelBox& box,
+        Language language,
+        std::vector<std::string> expected_lines
+    )
+        : UnitTest("OCR::MultilineOCR - " + image)
+        , m_image(UNIT_TEST_RESOURCE_PATH() + image)
+        , m_box(box)
+        , m_language(language)
+        , m_expected_lines(std::move(expected_lines))
+    {}
+
+    virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override{
+        ImageRGB32 image(m_image);
+        ImageViewRGB32 cropped = extract_box_reference(image, m_box);
+
+        std::string result = ocr_read(m_language, cropped, PageSegMode::SINGLE_BLOCK);
+        logger.log("Multi-line OCR read: " + result);
+
+        std::vector<std::string> lines;
+        size_t start = 0;
+        while (true){
+            size_t end = result.find('\n', start);
+            lines.emplace_back(result.substr(start, end - start));
+            if (end == std::string::npos){
+                break;
+            }
+            start = end + 1;
+        }
+
+        if (lines.size() != m_expected_lines.size()){
+            logger.log(
+                "Expected " + std::to_string(m_expected_lines.size()) +
+                " lines, got " + std::to_string(lines.size())
+            );
+            return false;
+        }
+        for (size_t c = 0; c < lines.size(); c++){
+            if (normalize_utf32(lines[c]) != normalize_utf32(m_expected_lines[c])){
+                logger.log("Line " + std::to_string(c) + " mismatch. Expected: " + m_expected_lines[c]);
+                return false;
+            }
+        }
+        return true;
+    };
+
+private:
+    std::string m_image;
+    ImagePixelBox m_box;
+    Language m_language;
+    std::vector<std::string> m_expected_lines;
+};
+
+void add_tests_multiline_OCR(UnitTestDatabase& database){
+    //  Already cropped to a single line, but with lots of empty space around it.
+    database.add<Test_MultilineOCR>(
+        "OCR/sentence-1-1-tall.jpg", ImagePixelBox(0, 0, 564, 500), Language::English,
+        std::vector<std::string>{"You hurry to the Pokemon Center, shielding your"}
+    );
+    database.add<Test_MultilineOCR>(
+        "OCR/sentence-1-1-wide.jpg", ImagePixelBox(0, 0, 1000, 41), Language::English,
+        std::vector<std::string>{"You hurry to the Pokemon Center, shielding your"}
+    );
+
+    //  Two-line FRLG dialog boxes: blue text on white.
+    database.add<Test_MultilineOCR>(
+        "PokemonFRLG/AdvanceWhiteDialogDetector/English-Bulba_True.png",
+        ImagePixelBox(270, 790, 1650, 1000), Language::English,
+        std::vector<std::string>{"I see! BULBASAUR is your choice.", "It's very easy to raise."}
+    );
+    database.add<Test_MultilineOCR>(
+        "PokemonFRLG/AdvanceWhiteDialogDetector/Ger-KennyExile-Char_True.png",
+        ImagePixelBox(270, 790, 1650, 1000), Language::German,
+        std::vector<std::string>{"Ah! GLUMANDA soll es sein!", "Trainiere es geduldig!"}
+    );
+
+    //  SV dialog box: speaker name (white on dark blue) above the dialog text (dark on white).
+    database.add<Test_MultilineOCR>(
+        "PokemonSV/DialogBoxDetector/French_True.png",
+        ImagePixelBox(430, 740, 1470, 975), Language::French,
+        std::vector<std::string>{"Clavel", "Poussacha, le Pokémon Chat Plante."}
+    );
+}
+
+
+
 }
 }

SerialPrograms/Source/CommonTools/OCR/OCR_Tests.h

@@ -20,6 +20,10 @@ void add_tests_raw_OCR(UnitTestDatabase& database);
 
 void add_tests_number_waterfill_OCR(UnitTestDatabase& database);
 
+// Multi-line OCR tests. These exercise the PaddleOCR detection + recognition pipeline
+// through `ocr_read()` with a multi-line page segmentation mode.
+void add_tests_multiline_OCR(UnitTestDatabase& database);
+
 
 
 }