Agent mcp server

SerialPrograms/CMakeLists.txt

@@ -63,9 +63,13 @@ find_package(Threads REQUIRED)
 # Define a toggle option (ON by default)
 option(COMMAND_LINE_EXE "Build the optional executable" ON)
 
-# Build only the GUI-free core: CoreLib and SerialProgramsCommandLine. This needs
-# no Qt, OpenCV, ONNX Runtime, Tesseract, DPP or Discord SDK, so none of them are
-# searched for.
+# Python bindings and the MCP server for AI agents (see Source/PythonBindings/README.md).
+# OFF by default because it needs a Python interpreter with development headers.
+option(PA_PYTHON_BINDINGS "Build the _pa_core Python module (acts as a Switch controller)" OFF)
+
+# Build only the GUI-free core: CoreLib, SerialProgramsCommandLine and (with
+# PA_PYTHON_BINDINGS) the Python module. This needs no Qt, OpenCV, ONNX Runtime,
+# Tesseract, DPP or Discord SDK, so none of them are searched for.
 option(PA_CORE_ONLY "Build only the GUI-free core (basic microcontroller functions, no Qt, OpenCV, etc.)" OFF)
 
 if(PA_CORE_ONLY)
@@ -78,6 +82,9 @@ if(PA_CORE_ONLY)
     if(COMMAND_LINE_EXE)
         include(Source/CommandLine/CommandLineExecutable.cmake)
     endif()
+    if(PA_PYTHON_BINDINGS)
+        include(Source/PythonBindings/PythonBindings.cmake)
+    endif()
     return()
 endif()
 
@@ -108,6 +115,7 @@ if(WIN32 AND QT_DEPLOY_FILES)
             Qml
             Quick
             QuickWidgets
+            Network
         )
     else()
         # Find all subdirectories in the Qt base directory
@@ -120,6 +128,7 @@ if(WIN32 AND QT_DEPLOY_FILES)
             Qml
             Quick
             QuickWidgets
+            Network
         )
         file(GLOB QT_SUBDIRS LIST_DIRECTORIES true "${QT_BASE_DIR}/${QT_MAJOR}*")
 
@@ -157,6 +166,7 @@ else()
         Qml
         Quick
         QuickWidgets
+        Network
     )
 endif()
 
@@ -265,6 +275,7 @@ function(apply_common_target_properties target_name)
         Qt${QT_MAJOR}::Qml
         Qt${QT_MAJOR}::Quick
         Qt${QT_MAJOR}::QuickWidgets
+        Qt${QT_MAJOR}::Network
     )    
 
     # more include directories
@@ -273,6 +284,16 @@ function(apply_common_target_properties target_name)
 
 endfunction()
 
+# Compile the shared AI agent tool definitions into the app (see
+# Source/Integrations/AgentServer/AgentTools.json).
+include(cmake/EmbedTextFile.cmake)
+pa_embed_text_file(
+    SerialProgramsLib
+    ${CMAKE_CURRENT_SOURCE_DIR}/Source/Integrations/AgentServer/AgentTools.json
+    ${CMAKE_CURRENT_BINARY_DIR}/Generated/AgentServer_AgentToolsJson.cpp
+    "PokemonAutomation::AgentServer::agent_tools_json_text"
+)
+
 # Apply common properties to both targets
 apply_common_target_properties(SerialProgramsLib)
 apply_common_target_properties(SerialPrograms)
@@ -846,3 +867,7 @@ if(COMMAND_LINE_EXE)
     # Add command-line executable (GUI-free) from subdirectory
     include(Source/CommandLine/CommandLineExecutable.cmake)
 endif()
+
+if(PA_PYTHON_BINDINGS)
+    include(Source/PythonBindings/PythonBindings.cmake)
+endif()

SerialPrograms/Source/Controllers/Controller.cpp

@@ -8,6 +8,9 @@
 #include "Common/Cpp/ListenerSet.h"
 #include "Common/Cpp/RecursiveThrottler.h"
 #include "Common/Cpp/Containers/Pimpl.tpp"
+#include "Common/Cpp/Concurrency/Mutex.h"
+#include "Common/Cpp/Concurrency/ConditionVariable.h"
+#include "Common/Cpp/Concurrency/Thread.h"
 #include "Controller.h"
 
 namespace PokemonAutomation{
@@ -41,6 +44,45 @@ RecursiveThrottler& AbstractController::logging_throttler(){
 }
 
 
+bool AbstractController::cancel_all_commands_blocking(Milliseconds timeout){
+    if (!is_ready()){
+        return false;
+    }
+    cancel_all_commands();
+
+    //  Bound the confirmation: this timer cancels `scope` when the timeout passes,
+    //  which makes `issue_nop()` / `wait_for_all()` below throw
+    //  OperationCancelledException.
+    CancellableHolder<CancellableScope> scope;
+    Mutex lock;
+    ConditionVariable cv;
+    bool done = false;
+    Thread timer([&]{
+        std::unique_lock<Mutex> lg(lock);
+        if (!cv.wait_for(lg, timeout, [&]{ return done; })){
+            scope.cancel(nullptr);
+        }
+    });
+
+    bool confirmed = false;
+    try{
+        //  Queued after the cancel, so the device reports this no-op finished only
+        //  after it has dropped everything before it and held neutral for 10 ms.
+        issue_nop(&scope, Milliseconds(10));
+        wait_for_all(&scope);
+        confirmed = true;
+    }catch (OperationCancelledException&){}
+
+    {
+        std::lock_guard<Mutex> lg(lock);
+        done = true;
+    }
+    cv.notify_all();
+    timer.join();
+    return confirmed;
+}
+
+
 void AbstractController::throw_bad_cast(const char* desired_typename){
     throw UserSetupError(
         logger(),

SerialPrograms/Source/Controllers/Controller.h

@@ -112,6 +112,24 @@ class AbstractController{
     //  ever releasing it during the transition.
     virtual void replace_on_next_command() = 0;
 
+    //  Unlike the two functions above, this one blocks:
+    //  Cancel all commands (`cancel_all_commands()`), then wait at most `timeout` for
+    //  the device to confirm that it is in the neutral state.
+    //
+    //  `cancel_all_commands()` only starts the cancellation: it returns as soon as the
+    //  cancel request is handed to the connection, and the host marks its command
+    //  queue empty at that moment, so waiting for the queue proves nothing. Instead this
+    //  queues a 10 ms neutral no-op after the cancel and waits for the device to report
+    //  that the no-op finished. Commands are delivered in order, so that report means
+    //  the device has processed the cancel and is holding the neutral state.
+    //
+    //  Returns true if confirmed; false on timeout (e.g. the Switch is asleep and the
+    //  device can't execute commands) or if the controller is not ready.
+    //  Thread-safe: may be called while another thread is issuing commands or waiting on
+    //  this controller. The timeout covers waiting for the device; a command that
+    //  another thread is in the middle of issuing is allowed to finish first.
+    bool cancel_all_commands_blocking(Milliseconds timeout);
+
 
 public:
     //

SerialPrograms/Source/GameConsole/ConsoleHandle.cpp

@@ -83,6 +83,9 @@ ConsoleHandle::ConsoleHandle(ConsoleSystemSession& session)
 size_t ConsoleHandle::index() const{
     return m_data->m_index;
 }
+ConsoleSystemSession& ConsoleHandle::system_session(){
+    return m_data->m_session;
+}
 size_t ConsoleHandle::controllers() const{
     return m_data->m_session.controllers();
 }

SerialPrograms/Source/GameConsole/ConsoleHandle.h

@@ -32,6 +32,10 @@ class ConsoleHandle : public VideoStream{
 
     size_t index() const;
 
+    //  The console session this handle belongs to. Programs use it to listen for
+    //  session events, e.g. the user's keyboard input (see ConsoleSystemSession::Listener).
+    ConsoleSystemSession& system_session();
+
     operator Logger&(){ return logger(); }
     operator VideoFeed&(){ return video(); }
     operator VideoOverlay&(){ return overlay(); }

SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.cpp

@@ -236,25 +236,28 @@ void ConsoleSystemSession::on_focus_out(){
     m_listeners.run_method(&Listener::on_input_status_change, status);
 }
 void ConsoleSystemSession::run_controller_input(ControllerInputState& state){
-    std::lock_guard<Mutex> lg(m_lock);
-    if (!m_focused){
-        m_logger.log("Keyboard Command Suppressed: Not in focus.", COLOR_RED);
-        return;
-    }
-    if (!m_lock_controllers_reason.empty() && !allow_commands_while_locked()){
-        m_logger.log("Keyboard Command Suppressed: " + m_lock_controllers_reason, COLOR_RED);
-        return;
-    }
+    {
+        std::lock_guard<Mutex> lg(m_lock);
+        if (!m_focused){
+            m_logger.log("Keyboard Command Suppressed: Not in focus.", COLOR_RED);
+            return;
+        }
+        if (!m_lock_controllers_reason.empty() && !allow_commands_while_locked()){
+            m_logger.log("Keyboard Command Suppressed: " + m_lock_controllers_reason, COLOR_RED);
+            return;
+        }
 
-//    cout << "ConsoleSystemSession::run_controller_input()" << endl;
-    for (ControllerEntry& controller : m_controllers){
-        std::string error = controller.session.try_run<AbstractController>([&](AbstractController& controller){
-            controller.run_controller_input(state);
-        });
-        if (!error.empty()){
-            controller.session.logger().log("Keyboard Command Failed: " + error, COLOR_RED);
+//        cout << "ConsoleSystemSession::run_controller_input()" << endl;
+        for (ControllerEntry& controller : m_controllers){
+            std::string error = controller.session.try_run<AbstractController>([&](AbstractController& controller){
+                controller.run_controller_input(state);
+            });
+            if (!error.empty()){
+                controller.session.logger().log("Keyboard Command Failed: " + error, COLOR_RED);
+            }
         }
     }
+    m_listeners.run_method(&Listener::on_controller_input, state);
 }
 
 

SerialPrograms/Source/GameConsole/Framework/ConsoleSystemSession.h

@@ -52,6 +52,12 @@ class ConsoleSystemSession final
         virtual void on_input_status_change(const std::string& status){}
         virtual void on_lock_controllers(){}
         virtual void on_unlock_controllers(){}
+
+        //  Called after the user's keyboard (or other controller input) was sent to
+        //  this console's controllers, e.g. so a program can notice that the user is
+        //  steering by hand. Not called for input that was suppressed (console not in
+        //  focus, or controllers locked). Runs on the UI thread; keep it quick.
+        virtual void on_controller_input(const ControllerInputState& state){}
     };
 
     void add_listener(Listener& listener){

SerialPrograms/Source/Integrations/AgentServer/AgentInputTestCases.json

@@ -0,0 +1,61 @@
+{
+    "_comment": [
+        "Test cases for the input vocabulary of AgentTools.json, shared by the C++",
+        "(AgentServer_InputSteps.cpp) and Python (pokemon_automation/buttons.py)",
+        "implementations so both parse inputs identically.",
+        "Button bits are NintendoSwitch::Button values; d-pad positions are",
+        "NintendoSwitch::DpadPosition values (0 = up, clockwise, 8 = none).",
+        "Stick values are [x, y] with +y = up; diagonals are normalized to length 1."
+    ],
+    "buttons": [
+        {"input": "A",              "bitfield": 4,    "dpad": 8},
+        {"input": "a",              "bitfield": 4,    "dpad": 8},
+        {"input": "L+R",            "bitfield": 48,   "dpad": 8},
+        {"input": ["ZL", "a"],      "bitfield": 68,   "dpad": 8},
+        {"input": "+",              "bitfield": 512,  "dpad": 8},
+        {"input": "-",              "bitfield": 256,  "dpad": 8},
+        {"input": "start",          "bitfield": 512,  "dpad": 8},
+        {"input": "select",         "bitfield": 256,  "dpad": 8},
+        {"input": "L3",             "bitfield": 1024, "dpad": 8},
+        {"input": "rs",             "bitfield": 2048, "dpad": 8},
+        {"input": "HOME",           "bitfield": 4096, "dpad": 8},
+        {"input": "capture",        "bitfield": 8192, "dpad": 8},
+        {"input": ["A", "+"],       "bitfield": 516,  "dpad": 8},
+        {"input": "up",             "bitfield": 0,    "dpad": 0},
+        {"input": "up+right",       "bitfield": 0,    "dpad": 1},
+        {"input": "UP_RIGHT",       "bitfield": 0,    "dpad": 1},
+        {"input": "down-left",      "bitfield": 0,    "dpad": 5},
+        {"input": "UPLEFT",         "bitfield": 0,    "dpad": 7},
+        {"input": "dpad_left",      "bitfield": 0,    "dpad": 6},
+        {"input": "ZL+DOWN",        "bitfield": 64,   "dpad": 4},
+        {"input": " b , x ",        "bitfield": 10,   "dpad": 8},
+        {"input": "",               "bitfield": 0,    "dpad": 8},
+        {"input": "Q",              "error": "Unknown button"},
+        {"input": "up+down",        "error": "Contradictory"},
+        {"input": "left+right",     "error": "Contradictory"}
+    ],
+    "sticks": [
+        {"input": "up",             "x": 0.0,  "y": 1.0},
+        {"input": "DOWN",           "x": 0.0,  "y": -1.0},
+        {"input": "left",           "x": -1.0, "y": 0.0},
+        {"input": "neutral",        "x": 0.0,  "y": 0.0},
+        {"input": "center",         "x": 0.0,  "y": 0.0},
+        {"input": "down_right",     "x": 0.7071067811865475, "y": -0.7071067811865475},
+        {"input": "up-left",        "x": -0.7071067811865475, "y": 0.7071067811865475},
+        {"input": [0.5, -0.25],     "x": 0.5,  "y": -0.25},
+        {"input": [2, 0],           "error": "within [-1, 1]"},
+        {"input": [0.5],            "error": "two numbers"},
+        {"input": "sideways",       "error": "Unknown stick direction"}
+    ],
+    "steps": [
+        {"input": {"buttons": "A"},                                   "duration_ms": 160},
+        {"input": {"buttons": "A", "hold_ms": 100, "release_ms": 50, "repeat": 3}, "duration_ms": 450},
+        {"input": {"wait_ms": 1000},                                  "duration_ms": 1000},
+        {"input": {"left_stick": "up", "hold_ms": 500, "release_ms": 0}, "duration_ms": 500},
+        {"input": {"buttons": "B", "left_stick": [0.5, 1.0], "hold_ms": 1500, "wait_ms": 100}, "duration_ms": 1680},
+        {"input": {"button": "A"},                                    "error": "Unknown input step field"},
+        {"input": {"buttons": "A", "hold_ms": 0},                     "error": "hold_ms"},
+        {"input": {"buttons": "A", "repeat": 0},                      "error": "repeat"},
+        {"input": {"buttons": "A", "release_ms": -1},                 "error": "negative"}
+    ]
+}