diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5ecde0295..8849a4c54 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -227,6 +227,7 @@ It uses text matching. Runtime type checks and in-game validation remain necessa | `sync/state.cc` | Comparisons, removals, and per-target acknowledgements | | `sync/upload.cc` | Sessions, the record store, target workers, and HTTP delivery | | `sync/battlelogs.cc` | Battle-journal fetching | +| `sync/journal.cc` | Protobuf battle journals rebuilt in the JSON journal shape | | `sync/fleets.cc` | Client fleet records: slot polling, state changes, snapshots, and late re-reads, on the game thread | Payload construction and HTTP run outside the store lock. diff --git a/src/mod/game/proto/Digit.PrimeServer.Models.proto b/src/mod/game/proto/Digit.PrimeServer.Models.proto index 39392b81b..1f7fd4c27 100644 --- a/src/mod/game/proto/Digit.PrimeServer.Models.proto +++ b/src/mod/game/proto/Digit.PrimeServer.Models.proto @@ -5615,3 +5615,105 @@ message JournalHeaderProto { message BattleResultHeadersResponse { repeated JournalHeaderProto headers = 1; } + +// ===== Battle report (journal) messages for POST /journals/get. Recovered from the game client metadata and +// checked against live responses (2026-11-20). Wrapper: ServiceResponse.entityGroups[type == TYPE_BATTLEREPORT].group. +// Only new messages; existing messages are untouched. ===== +// EntityGroup type 221 (TYPE_BATTLEREPORT) payload; response to POST /journals/get. +message JournalResponse { + int64 id = 1; // id + int64 battleDuration = 2; // battle_duration + google.protobuf.Timestamp battleTime = 3; // battle_time (seconds only on the wire) + int32 battleType = 4; // battle_type + bool initiatorWins = 5; // initiator_wins + bool initiatorFiredFirst = 6; // initiator_fired_first + string initiatorId = 7; // initiator_id + string targetId = 8; // target_id + int64 systemId = 9; // system_id + PlanetSlotsConfig.Slot coords = 10; // coords {float x=1, float y=2} + repeated string battleLog = 11; // battle_log: one python-repr number per element ("-96", "1.2000000000000002", "5.0") + JournalFleetData initiatorFleetData = 12; // initiator_fleet_data + JournalFleetData targetFleetData = 13; // target_fleet_data + map resourcesDropped = 14; // resources_dropped + map resourcesTransferred = 15; // resources_transferred + JournalChestDrop chestDrop = 16; // chest_drop + JournalContextData battleContextData = 17; // battle_context_data +} + +message JournalFleetData { + int32 battleDataType = 1; // battle_data_type + int32 cargoMax = 2; // cargo_max + int64 defenseRating = 3; // defense_rating + int64 healthRating = 4; // health_rating + int64 attackRating = 5; // attack_rating (old JSON key: offense_rating) + int64 officerRating = 6; // officer_rating + int32 numDrydocks = 7; // num_drydocks + repeated int64 hullIds = 8; // hull_ids (packed) + repeated int64 shipIds = 9; // ship_ids (packed) + repeated int64 initialShipHps = 10; // initial_ship_hps (packed) + repeated int64 finalShipHps = 11; // final_ship_hps (packed) + repeated int64 maxShipHps = 12; // max_ship_hps (packed) + repeated int64 initialShipShps = 13; // initial_ship_shps (packed) + repeated int64 finalShipShps = 14; // final_ship_shps (packed) + repeated int64 maxShipShps = 15; // max_ship_shps (packed) + repeated float initialShipLevelPercentages = 16;// initial_ship_level_percentages (packed fixed32) + repeated float finalShipLevelPercentages = 17; // final_ship_level_percentages (packed fixed32) + repeated int32 initialShipLevels = 18; // initial_ship_levels (packed) + repeated int32 finalShipLevels = 19; // final_ship_levels (packed) + CargoProto initialCargo = 20; // initial_cargo + CargoProto finalCargo = 21; // final_cargo + repeated JournalFleetOfficerEntry fleetsOfficers = 22; // fleets_officers (old JSON: {fleetId: [officer|null ...]}) + map deployedFleets = 23; // deployed_fleets + DeployedFleet deployedFleet = 24; // deployed_fleet + repeated CargoProto initialCargoList = 25; // initial_cargo_list + repeated CargoProto finalCargoList = 26; // final_cargo_list + repeated int32 cargoMaxList = 27; // cargo_max_list (packed; -1 = 10-byte varint) + string armadaOwnerId = 28; // armada_owner_id + IdRefs refIds = 29; // ref_ids (NPC side only) +} + +message JournalFleetOfficerEntry { + int64 fleetId = 1; // fleet_id + repeated BridgeCrewOfficer officers = 2; // officers + int32 crewSize = 3; // crew_size (10 in all samples) +} + +message BridgeCrewOfficer { + int64 id = 1; // id + int32 level = 2; // level + int32 rank = 3; // rank + int64 slotIndex = 4; // slot_index (absent = slot 0) +} + +message CargoProto { + map resources = 1; // resources +} + +message JournalChestDrop { + repeated InventoryItem chestsGained = 1; // chests_gained (wire: type, count, commonParams.refId only) + repeated InventoryItem chestsDiscarded = 2; // chests_discarded (not in old JSON) + int32 capacity = 3; // capacity (not in old JSON) + int32 currentCount = 4; // current_count (not in old JSON) + repeated IdRefs chestIdRefs = 5; // chest_id_refs + repeated string userIdRefs = 6; // user_id_refs +} + +message JournalContextData { + JournalContextDataInner contextDataMapping = 1; // context_data_mapping (old JSON key: context_data) +} + +message JournalContextDataInner { + JournalMultitargetArmadaInfo multitargetArmadaInformation = 1; // multitarget_armada_information (old: multi_target_armadas_info) + int32 flags = 2; // flags + JournalGalacticAnomalyData galacticAnomalyData = 3; // galactic_anomaly_data (never seen in old JSON) +} + +message JournalMultitargetArmadaInfo { + int64 mtaId = 1; // mta_id (old: multi_armada_attack_id) + int32 mtaSize = 2; // mta_size (old: mta_group_size) +} + +message JournalGalacticAnomalyData { + int64 tournamentId = 1; // tournament_id + map systemBuffs = 2; // system_buffs +} diff --git a/src/mod/sync/battlelogs.cc b/src/mod/sync/battlelogs.cc index 1c2c29781..db3ccdba9 100644 --- a/src/mod/sync/battlelogs.cc +++ b/src/mod/sync/battlelogs.cc @@ -1,6 +1,7 @@ // Battle logs: the game only sends battle headers, so each new battle's journal is fetched from the game server // with the participants' player and alliance names resolved, then uploaded. +#include "sync/journal.h" #include "sync/sync.h" #include "version.h" @@ -181,8 +182,9 @@ namespace return response.text; } - // The players in a battle. Fleet battles list them in deployed_fleets (fleet data with ref_ids is an NPC's); - // cracked stations have no fleet, so a base raid's defender only shows up in the journal's own ids. + // The players in a battle. Fleet battles list them in deployed_fleets, an object keyed by fleet id (fleet data + // with ref_ids is an NPC's); cracked stations have no fleet, so a base raid's defender only shows up in the + // journal's own ids. std::unordered_set Participants(const json& journal) { std::unordered_set ids; @@ -193,11 +195,15 @@ namespace if (fleet == journal.end() || !fleet->is_object() || !fleet->value("ref_ids", json()).is_null()) continue; const auto deployed_fleets = fleet->find("deployed_fleets"); - if (deployed_fleets == fleet->end() || !deployed_fleets->is_array()) + if (deployed_fleets == fleet->end() || !deployed_fleets->is_structured()) continue; - for (const auto& deployed : *deployed_fleets) + for (const auto& deployed : *deployed_fleets) { + if (!deployed.is_object()) + continue; if (const auto uid = deployed.find("uid"); uid != deployed.end() && uid->is_string()) - ids.insert(uid->get()); + if (const auto& text = uid->get_ref(); !text.empty()) + ids.insert(text); + } } // NPC ids contain '_' (mar_61, npc__1); there is nothing to resolve for them. for (const char* key : {"initiator_id", "target_id"}) @@ -303,10 +309,23 @@ namespace const auto text = RequestGameData(session, "/journals/get", json{{"journal_id", battle.id}}.dump()); if (text.empty()) return false; - auto reply = json::parse(text, nullptr, false); - if (!reply.is_object() || !reply.contains("journal")) { - logging::Error("process", "battle log", "the journal of battle {} does not parse", battle.id); - return false; + json reply; + if (IsJsonBody(text)) { + reply = json::parse(text, nullptr, false); + if (!reply.is_object() || !reply.contains("journal")) { + logging::Error("process", "battle log", "the journal of battle {} does not parse", battle.id); + return false; + } + } else { + // Since 2026-11-19 the game server answers with protobuf; the journal is rebuilt in the JSON shape. + std::string error; + auto converted = JournalFromProtobuf(text, &error); + if (!converted) { + logging::Error("process", "battle log", "the protobuf journal of battle {} ({} bytes) does not convert: {}", + battle.id, text.size(), error); + return false; + } + reply = {{"journal", std::move(*converted)}}; } auto& journal = reply["journal"]; auto names = ResolveNames(session, journal); diff --git a/src/mod/sync/journal.cc b/src/mod/sync/journal.cc new file mode 100644 index 000000000..e32a27bdd --- /dev/null +++ b/src/mod/sync/journal.cc @@ -0,0 +1,550 @@ +// Protobuf battle journals rebuilt in the JSON journal shape. Keys follow the old JSON journals: values the wire +// does not carry are left out, never set to 0 or invented, and explicit nulls appear only where the JSON journals +// had them. + +#include "sync/journal.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace data_sync +{ +namespace +{ + using json = nlohmann::json; + namespace pb = Digit::PrimeServer::Models; + + constexpr int32_t kBattleReportGroup = pb::EntityGroup::TYPE_BATTLEREPORT; + // Larger bodies are not parsed; real journals are a few hundred kilobytes. + constexpr size_t kMaxJournalBytes = 64u << 20; + // Bridge crews have 10 slots; a slot index past this is dropped rather than allocated. + constexpr int64_t kMaxOfficerSlots = 64; + + // --- Values --- + + json Integer(int64_t value) + { return value; } + + // float32 in its shortest round-trip form: 1.1, not 1.100000023841858. Integral values stay doubles (48241632.0), + // as in the JSON journals. + json Float(float value) + { + if (!std::isfinite(value)) + return nullptr; + if (std::trunc(value) == value) + return static_cast(value); + char buffer[32]; + if (const auto [end, ec] = std::to_chars(std::begin(buffer), std::end(buffer), value); ec == std::errc{}) { + // Float from_chars needs macOS 26 in Apple's libc++; the JSON parser reads the shortest form back instead. + if (auto parsed = json::parse(std::begin(buffer), end, nullptr, false); parsed.is_number_float()) + return parsed; + } + return static_cast(value); + } + + // A float for a key that was an integer in the JSON journals (ratings, ship hit points). + json Rounded(float value) + { + if (!std::isfinite(value) || std::fabs(value) >= 9.2e18f) + return nullptr; + return static_cast(std::llround(value)); + } + + // "YYYY-MM-DDTHH:MM:SS" in UTC, with ".ffffff" only when the timestamp has nanoseconds. + std::string Time(const google::protobuf::Timestamp& time) + { + using namespace std::chrono; + // Far beyond any real date; keeps the calendar arithmetic in range. + constexpr int64_t kLimit = int64_t{1} << 40; + const auto seconds_ = std::clamp(time.seconds(), -kLimit, kLimit); + const auto instant = sys_seconds{seconds{seconds_}}; + const auto day = floor(instant); + const auto date = year_month_day{day}; + const auto clock = hh_mm_ss{instant - day}; + char text[64]; + std::snprintf(text, sizeof(text), "%04d-%02u-%02uT%02d:%02d:%02d", static_cast(date.year()), + static_cast(date.month()), static_cast(date.day()), + static_cast(clock.hours().count()), static_cast(clock.minutes().count()), + static_cast(clock.seconds().count())); + std::string result = text; + if (time.nanos() != 0) { + std::snprintf(text, sizeof(text), ".%06d", std::abs(time.nanos() / 1000)); + result += text; + } + return result; + } + + json TimeOrNull(bool present, const google::protobuf::Timestamp& time) + { return present ? json(Time(time)) : json(nullptr); } + + // Protobuf maps become objects with decimal-string keys; json sorts them as the JSON journals were sorted. + template json Object(const Map& map, Value&& value) + { + auto result = json::object(); + for (const auto& entry : map) + result[std::to_string(entry.first)] = value(entry.second); + return result; + } + + template json Array(const Repeated& items, Value&& value) + { + auto result = json::array(); + for (const auto& item : items) + result.push_back(value(item)); + return result; + } + + const auto kInteger = [](int64_t value) { return Integer(value); }; + const auto kFloat = [](float value) { return Float(value); }; + const auto kString = [](const std::string& value) { return json(value); }; + + json IdRefs(const pb::IdRefs& refs) + { + json result{ + {"art_file_reference", refs.artfilereference().empty() ? json(nullptr) : json(refs.artfilereference())}}; + if (refs.artid() != 0) + result["art_id"] = refs.artid(); + if (refs.locaid() != 0) + result["loca_id"] = refs.locaid(); + return result; + } + + json Cargo(const pb::CargoProto& cargo) + { return {{"resources", Object(cargo.resources(), kInteger)}}; } + + // A battle log entry is the server's Python repr of a number ("-96", "1.2000000000000002", "5.0"). It becomes the + // JSON number with the same text, so integers stay integers and "5.0" stays a float. Anything else stays text. + json LogEntry(const std::string& text) + { + size_t i = 0; + const auto digit = [&](size_t at) { return at < text.size() && text[at] >= '0' && text[at] <= '9'; }; + const auto run = [&] { + if (!digit(i)) + return false; + while (digit(i)) + ++i; + return true; + }; + bool integral = true; + if (i < text.size() && text[i] == '-') + ++i; + if (digit(i) && text[i] == '0') + ++i; + else if (!run()) + return text; + if (i < text.size() && text[i] == '.') { + integral = false; + ++i; + if (!run()) + return text; + } + if (i < text.size() && (text[i] == 'e' || text[i] == 'E')) { + integral = false; + ++i; + if (i < text.size() && (text[i] == '+' || text[i] == '-')) + ++i; + if (!run()) + return text; + } + if (i != text.size()) + return text; + + const auto* begin = text.data(); + const auto* end = text.data() + text.size(); + if (integral) { + if (int64_t value; std::from_chars(begin, end, value) == std::from_chars_result{end, std::errc{}}) + return value; + if (uint64_t value; std::from_chars(begin, end, value) == std::from_chars_result{end, std::errc{}}) + return value; + } + if (auto parsed = json::parse(begin, end, nullptr, false); + parsed.is_number() && (!parsed.is_number_float() || std::isfinite(parsed.get()))) + return parsed; + return text; + } + + // --- Chests --- + + json Item(const pb::InventoryItem& item, const pb::IdRefs* paired_refs) + { + const auto& common = item.commonparams(); + const auto type = static_cast(item.type()); + json params{{"ref_id", common.refid()}}; + if (type == pb::INVENTORYITEMTYPE_INVENTORYCHEST) { + const bool has_chest = item.has_chestparams(); + const auto& chest = item.chestparams(); + if (common.id() != 0) + params["id"] = common.id(); + if (has_chest && chest.has_idrefs()) + params["id_refs"] = IdRefs(chest.idrefs()); + else if (paired_refs) + params["id_refs"] = IdRefs(*paired_refs); + // Today's wire carries no chest contents; they appear here if the server sends them again. + if (has_chest && chest.chestitems_size() > 0) { + auto items = json::array(); + for (const auto& content : chest.chestitems()) { + items.push_back(Item(content, nullptr)); + const auto ref = std::to_string(content.commonparams().refid()); + params["chest_item_" + ref + "_amount"] = content.count(); + params["chest_item_" + ref + "_type"] = static_cast(content.type()); + } + params["items"] = std::move(items); + } + if (has_chest && chest.chestsource() != 0) + params["source"] = static_cast(chest.chestsource()); + } + return {{"count", item.count()}, + {"expiry_time", TimeOrNull(item.has_expirytime(), item.expirytime())}, + {"params", std::move(params)}, + {"type", type}}; + } + + // --- Fleets --- + + struct Side { + std::optional opponent_fleet_id; // Left out when the opponent has no fleet. + bool won = false; + }; + + json DeployedFleet(const pb::DeployedFleet& fleet, const Side& side) + { + const auto& attributes = fleet.attributes(); + const auto& stats = fleet.stats(); + json result; + + // Keys absent from the wire are left out, never sent as 0. ClientModifierType keys: -7 fleet grade, -8 offense, + // -9 defense, -10 health, -11 deflector, -12 sensor, -13 officer, -18 forbidden tech. + const auto attribute = [&](const char* name, int32_t key) { + if (const auto it = attributes.find(key); it != attributes.end()) + result[name] = Rounded(it->second); + }; + const auto stat = [&](const char* name, int32_t key) { + if (const auto it = stats.find(key); it != stats.end()) + result[name] = Float(it->second); + }; + // Keyed by the ships in ship_stats; a ship without the value has no entry. + const auto per_ship = [&](const google::protobuf::Map& source, int32_t key, + auto&& value) { + auto ships = json::object(); + for (const auto& entry : fleet.shipstats()) { + const auto ship = entry.first; + const auto found = source.find(ship); + if (found == source.end()) + continue; + const auto& values = found->second.shipattributes_(); + if (const auto it = values.find(key); it != values.end()) + ships[std::to_string(ship)] = value(it->second); + } + return ships; + }; + const auto floats = [](const auto& map) { return Object(map, kFloat); }; + + result["active_buffs"] = Array(fleet.activebuffs(), [](const pb::ActiveBuff& buff) { + return json{{"activation_time", TimeOrNull(buff.has_activationtime(), buff.activationtime())}, + {"activator_id", buff.activatorid()}, + {"buff_id", buff.buffid()}, + {"expiry_time", TimeOrNull(buff.has_expirytime(), buff.expirytime())}, + {"ranks", Array(buff.ranks(), kInteger)}}; + }); + result["attributes"] = floats(attributes); + if (fleet.battleopponentfleetid() != 0) + result["battle_opponent_fleet_id"] = fleet.battleopponentfleetid(); + else if (side.opponent_fleet_id) + result["battle_opponent_fleet_id"] = *side.opponent_fleet_id; + if (fleet.has_battlestarttime()) + result["battle_start_time"] = Time(fleet.battlestarttime()); + result["battle_won"] = fleet.battlewon() || side.won; + result["cause_of_destruction"] = static_cast(fleet.causeofdestruction()); + if (fleet.courseid() != 0) + result["course_id"] = fleet.courseid(); + attribute("defense_rating", -9); + attribute("deflector_rating", -11); + attribute("fleet_grade", -7); + result["fleet_id"] = fleet.fleetid(); + attribute("forbidden_tech_rating", -18); + attribute("health_rating", -10); + result["hull_ids"] = Array(fleet.hullids(), kInteger); + stat("impulse_speed", 62); + if (fleet.isactive()) + result["is_active"] = true; + if (fleet.has_lastupdatetime()) + result["last_update_time"] = Time(fleet.lastupdatetime()); + if (fleet.latestcoursevectorx() != 0.0f) + result["latest_course_vector_x"] = Float(fleet.latestcoursevectorx()); + if (fleet.latestcoursevectory() != 0.0f) + result["latest_course_vector_y"] = Float(fleet.latestcoursevectory()); + if (fleet.has_nodeaddress()) { + const auto& address = fleet.nodeaddress(); + result["node_address"] = {{"galaxy", address.galaxy()}, + {"instance", address.instance()}, + {"planet", address.planet() != 0 ? json(address.planet()) : json(nullptr)}, + {"system", address.system()}}; + } + if (fleet.nodeid() != 0) + result["node_id"] = fleet.nodeid(); + attribute("offense_rating", -8); + attribute("officer_rating", -13); + result["pursued_by_num"] = fleet.pursuedbynum(); + result["pursuit_target_id"] = fleet.pursuittargetid() != 0 ? json(fleet.pursuittargetid()) : json(nullptr); + attribute("sensor_rating", -12); + result["ship_attributes"] = + Object(fleet.shipattributes(), [&](const pb::ShipAttributes& ship) { return floats(ship.shipattributes_()); }); + result["ship_components"] = Object( + fleet.deployedships(), [](const pb::DeployedShip& ship) { return Array(ship.componentids(), kInteger); }); + result["ship_cosmetics"] = Object(fleet.shipcosmetics(), kInteger); + // Stat keys: 61 hull, 60 shield, 75 shield regeneration; attribute keys: -5 hull damage, -6 shield damage. + result["ship_dmg"] = per_ship(fleet.shipattributes(), -5, kFloat); + result["ship_hps"] = per_ship(fleet.shipstats(), 61, Rounded); + result["ship_ids"] = Array(fleet.shipids(), kInteger); + result["ship_level_percentages"] = floats(fleet.shiplevelpercentages()); + result["ship_levels"] = Object(fleet.shiplevels(), kInteger); + result["ship_shield_dmg"] = per_ship(fleet.shipattributes(), -6, kFloat); + result["ship_shield_hps"] = per_ship(fleet.shipstats(), 60, Rounded); + result["ship_shield_total_regeneration_durations"] = per_ship(fleet.shipstats(), 75, Rounded); + result["ship_stats"] = + Object(fleet.shipstats(), [&](const pb::ShipAttributes& ship) { return floats(ship.shipattributes_()); }); + result["ship_tiers"] = Object(fleet.shiptiers(), kInteger); + if (fleet.state() != 0) + result["state"] = static_cast(fleet.state()); + result["stats"] = floats(stats); + result["status_counters"] = Array(fleet.statuscounters(), [](const pb::StatusCounter& counter) { + return json{{"counter", counter.counter()}, {"status_id", counter.statusid()}}; + }); + result["status_effects"] = Object(fleet.statuseffects(), kInteger); + result["type"] = static_cast(fleet.fleettype()); + if (!fleet.uid().empty()) + result["uid"] = fleet.uid(); + // Stat keys: 64 warp range, 63 warp speed. + stat("warp_distance", 64); + stat("warp_speed", 63); + result["warp_data"] = nullptr; + result["warp_time"] = TimeOrNull(fleet.has_warptime(), fleet.warptime()); + // Not in the JSON journals; sent only when the wire has it. + if (fleet.armadaattackid() != 0) + result["armada_attack_id"] = fleet.armadaattackid(); + return result; + } + + // The side's own fleet: deployed_fleet, else the lowest-keyed deployed_fleets entry (map order is unspecified). + const pb::DeployedFleet* MainFleet(const pb::JournalFleetData& data) + { + if (data.has_deployedfleet()) + return &data.deployedfleet(); + const pb::DeployedFleet* main = nullptr; + int64_t key = 0; + for (const auto& entry : data.deployedfleets()) { + if (!main || entry.first < key) { + main = &entry.second; + key = entry.first; + } + } + return main; + } + + // battle_opponent_fleet_id for the other side: null against an armada, as the JSON journals had it. + std::optional OpponentFleetId(const pb::JournalFleetData& data) + { + const auto* main = MainFleet(data); + if (!main) + return std::nullopt; + if (data.deployedfleets_size() > 1) + return json(nullptr); + return json(main->fleetid()); + } + + // A crew as the JSON journals sent it: crew_size slots, null where nobody sits. + json Crew(const pb::JournalFleetOfficerEntry& entry) + { + const auto in_range = [](int64_t slot) { return slot >= 0 && slot < kMaxOfficerSlots; }; + auto size = std::clamp(entry.crewsize(), 0, kMaxOfficerSlots); + for (const auto& officer : entry.officers()) + if (in_range(officer.slotindex())) + size = std::max(size, officer.slotindex() + 1); + auto slots = json::array(); + for (int64_t i = 0; i < size; ++i) + slots.push_back(nullptr); + for (const auto& officer : entry.officers()) + if (in_range(officer.slotindex())) + slots[static_cast(officer.slotindex())] = { + {"id", officer.id()}, {"level", officer.level()}, {"rank", officer.rank()}}; + return slots; + } + + json FleetData(const pb::JournalFleetData& data, const Side& side) + { + auto crews = json::object(); + size_t crew_count = 0; + json first_crew; + for (const auto& entry : data.fleetsofficers()) { + auto crew = Crew(entry); + if (crew_count++ == 0) + first_crew = crew; + crews[std::to_string(entry.fleetid())] = std::move(crew); + } + const auto* main = MainFleet(data); + const auto zero_null = [](int64_t value) { return value != 0 ? json(value) : json(nullptr); }; + + json result; + result["armada_owner_id"] = data.armadaownerid().empty() ? json(nullptr) : json(data.armadaownerid()); + result["battle_data_type"] = data.battledatatype(); + // One crew, as an array; none (an NPC) as {}; several (an armada) as null. + result["bridge_officers"] = crew_count == 1 ? std::move(first_crew) : crew_count == 0 ? json::object() : json(); + result["cargo_max"] = zero_null(data.cargomax()); + result["cargo_max_list"] = Array(data.cargomaxlist(), kInteger); + result["defense_rating"] = zero_null(data.defenserating()); + if (main) + result["deployed_fleet"] = DeployedFleet(*main, side); + result["deployed_fleets"] = + Object(data.deployedfleets(), [&](const pb::DeployedFleet& fleet) { return DeployedFleet(fleet, side); }); + if (data.has_finalcargo()) + result["final_cargo"] = Cargo(data.finalcargo()); + result["final_cargo_list"] = Array(data.finalcargolist(), Cargo); + result["final_ship_hps"] = Array(data.finalshiphps(), kInteger); + result["final_ship_level_percentages"] = Array(data.finalshiplevelpercentages(), kFloat); + result["final_ship_levels"] = Array(data.finalshiplevels(), kInteger); + result["final_ship_shps"] = Array(data.finalshipshps(), kInteger); + // An armada side's fleet_id was an id the wire does not carry, so it is left out. + if (main && data.deployedfleets_size() <= 1) + result["fleet_id"] = main->fleetid(); + result["fleets_officers"] = std::move(crews); + result["health_rating"] = zero_null(data.healthrating()); + result["hull_ids"] = Array(data.hullids(), kInteger); + if (data.has_initialcargo()) + result["initial_cargo"] = Cargo(data.initialcargo()); + result["initial_cargo_list"] = Array(data.initialcargolist(), Cargo); + result["initial_ship_hps"] = Array(data.initialshiphps(), kInteger); + result["initial_ship_level_percentages"] = Array(data.initialshiplevelpercentages(), kFloat); + result["initial_ship_levels"] = Array(data.initialshiplevels(), kInteger); + result["initial_ship_shps"] = Array(data.initialshipshps(), kInteger); + result["max_ship_hps"] = Array(data.maxshiphps(), kInteger); + result["max_ship_shps"] = Array(data.maxshipshps(), kInteger); + result["num_drydocks"] = data.numdrydocks(); + result["offense_rating"] = zero_null(data.attackrating()); + result["officer_rating"] = data.officerrating(); + // Only NPC sides carry ref_ids; a player side has null. + result["ref_ids"] = data.has_refids() ? IdRefs(data.refids()) : json(nullptr); + result["ship_ids"] = Array(data.shipids(), kInteger); + result["xp_gained"] = nullptr; + return result; + } + + // --- Journal --- + + json Journal(const pb::JournalResponse& journal) + { + json result; + if (journal.has_battlecontextdata()) { + auto context = json::object(); + if (journal.battlecontextdata().has_contextdatamapping()) { + const auto& mapping = journal.battlecontextdata().contextdatamapping(); + if (mapping.flags() != 0) + context["flags"] = mapping.flags(); + if (mapping.has_multitargetarmadainformation()) { + const auto& armadas = mapping.multitargetarmadainformation(); + context["multi_target_armadas_info"] = {{"mta_group_size", armadas.mtasize()}, + {"multi_armada_attack_id", armadas.mtaid()}}; + } + if (mapping.has_galacticanomalydata()) { + const auto& anomaly = mapping.galacticanomalydata(); + context["galactic_anomaly_data"] = {{"system_buffs", Object(anomaly.systembuffs(), kInteger)}, + {"tournament_id", anomaly.tournamentid()}}; + } + } + result["battle_context_data"] = {{"context_data", std::move(context)}}; + } + result["battle_duration"] = journal.battleduration(); + result["battle_log"] = Array(journal.battlelog(), LogEntry); + if (journal.has_battletime()) + result["battle_time"] = Time(journal.battletime()); + result["battle_type"] = journal.battletype(); + if (journal.has_chestdrop()) { + const auto& drop = journal.chestdrop(); + // A chest's id_refs are its chest_id_refs entry when both lists have the same length. + const bool paired = drop.chestidrefs_size() == drop.chestsgained_size(); + auto gained = json::array(); + for (int i = 0; i < drop.chestsgained_size(); ++i) + gained.push_back(Item(drop.chestsgained(i), paired ? &drop.chestidrefs(i) : nullptr)); + result["chest_drop"] = {{"chest_id_refs", Array(drop.chestidrefs(), IdRefs)}, + {"chests_gained", std::move(gained)}, + {"user_id_refs", Array(drop.useridrefs(), kString)}}; + } + if (journal.has_coords()) + result["coords"] = {{"x", Float(journal.coords().x())}, {"y", Float(journal.coords().y())}}; + result["id"] = journal.id(); + result["initiator_fired_first"] = journal.initiatorfiredfirst(); + + const Side initiator{OpponentFleetId(journal.targetfleetdata()), journal.initiatorwins()}; + const Side target{OpponentFleetId(journal.initiatorfleetdata()), !journal.initiatorwins()}; + result["initiator_fleet_data"] = FleetData(journal.initiatorfleetdata(), initiator); + result["initiator_id"] = journal.initiatorid(); + result["initiator_wins"] = journal.initiatorwins(); + result["resources_dropped"] = Object(journal.resourcesdropped(), kInteger); + result["resources_transferred"] = Object(journal.resourcestransferred(), kInteger); + result["system_id"] = journal.systemid(); + result["target_fleet_data"] = FleetData(journal.targetfleetdata(), target); + result["target_id"] = journal.targetid(); + result["journal_src"] = "pb1"; + return result; + } + + std::optional Fail(std::string* error, const char* reason) + { + if (error) + *error = reason; + return std::nullopt; + } +} // namespace + +bool IsJsonBody(std::string_view body) +{ + const auto start = body.find_first_not_of(" \t\r\n"); + return start != std::string_view::npos && body[start] == '{'; +} + +std::optional JournalFromProtobuf(std::string_view body, std::string* error) +{ + try { + if (body.empty()) + return Fail(error, "the body is empty"); + if (body.size() > kMaxJournalBytes) + return Fail(error, "the body is too large"); + if (IsJsonBody(body)) + return Fail(error, "the body is JSON"); + if (body.size() >= 2 && static_cast(body[0]) == 0x1f && static_cast(body[1]) == 0x8b) + return Fail(error, "the body is still gzip-compressed"); + + pb::ServiceResponse response; + if (!response.ParseFromArray(body.data(), static_cast(body.size()))) + return Fail(error, "the ServiceResponse does not parse"); + const auto& groups = response.entitygroups(); + const auto group = std::ranges::find_if(groups, [](const auto& g) { return g.type() == kBattleReportGroup; }); + if (group == groups.end()) + return Fail(error, "there is no battle report entity group"); + + pb::JournalResponse journal; + if (group->group().size() > static_cast(std::numeric_limits::max()) + || !journal.ParseFromArray(group->group().data(), static_cast(group->group().size()))) + return Fail(error, "the JournalResponse does not parse"); + if (journal.id() == 0) + return Fail(error, "the JournalResponse has no id"); + + auto result = Journal(journal); + // Fails here rather than in the uploader if a wire string is not valid UTF-8. + (void)result.dump(); + return result; + } catch (const std::exception&) { + return Fail(error, "the journal could not be converted"); + } +} +} // namespace data_sync diff --git a/src/mod/sync/journal.h b/src/mod/sync/journal.h new file mode 100644 index 000000000..a573ad9c4 --- /dev/null +++ b/src/mod/sync/journal.h @@ -0,0 +1,22 @@ +#pragma once + +// Battle journals from POST /journals/get. Since 2026-11-19 the game server answers with protobuf +// (ServiceResponse, entity group TYPE_BATTLEREPORT, JournalResponse) instead of JSON. The converter rebuilds the +// journal object the JSON server sent as response["journal"]: same keys, nesting and JSON types, with keys the wire +// no longer carries left out rather than invented, and "journal_src":"pb1" added. It uses only the generated +// protobuf code, nlohmann::json and the standard library, so the sync tests link it without the game. + +#include + +#include +#include +#include + +namespace data_sync +{ +// True when the first non-whitespace byte is '{': a server that still answers with JSON. +bool IsJsonBody(std::string_view body); + +// The journal object for a protobuf /journals/get body, or nullopt with a short reason in `error`. Never throws. +std::optional JournalFromProtobuf(std::string_view body, std::string* error = nullptr); +} // namespace data_sync diff --git a/tests/sync_logic.cc b/tests/sync_logic.cc index 9ab716dd4..5f1e8641a 100644 --- a/tests/sync_logic.cc +++ b/tests/sync_logic.cc @@ -1,6 +1,7 @@ // Production encoders against independent wire fixtures, and delivery against controlled HTTP outcomes and time. // Check throws instead of asserting, so one run reports every failing case. +#include "sync/journal.h" #include "sync/sync.h" #include @@ -758,6 +759,157 @@ void NameCaches() Check(!names.Find("c", 3), "a zero TTL expires names at once"); settings->resolver_cache_ttl = previous_ttl; } +// Protobuf journals: a synthetic player-vs-NPC battle against the exact JSON journal shape, then bodies that +// must be refused. Values are invented; only their kinds matter. +std::string BattleReportBody(const game::JournalResponse& journal, int32_t type = game::EntityGroup::TYPE_BATTLEREPORT) +{ + game::ServiceResponse response; + auto* group = response.add_entitygroups(); + group->set_type(static_cast(type)); + group->set_group(journal.SerializeAsString()); + return response.SerializeAsString(); +} +game::JournalResponse SyntheticJournal() +{ + game::JournalResponse journal; + journal.set_id(2'860'000'000'000'000'001); + journal.set_battleduration(5); + journal.mutable_battletime()->set_seconds(1'763'600'000); + journal.set_battletype(5); + journal.set_initiatorwins(true); + journal.set_initiatorid("player1"); + journal.set_targetid("mar_30"); + journal.set_systemid(1'234'567'890'123); + journal.mutable_coords()->set_x(1.1f); + journal.mutable_coords()->set_y(-2.5f); + for (const char* entry : {"-96", "1234567890123456789", "5.0", "1.2000000000000002", "-97"}) + journal.add_battlelog(entry); + (*journal.mutable_resourcesdropped())[2] = 100; + journal.mutable_battlecontextdata(); + auto* chest = journal.mutable_chestdrop()->add_chestsgained(); + chest->set_type(game::INVENTORYITEMTYPE_INVENTORYCHEST); + chest->set_count(1); + chest->mutable_commonparams()->set_refid(77); + journal.mutable_chestdrop()->add_chestidrefs()->set_locaid(9); + + auto* player = journal.mutable_initiatorfleetdata(); + player->set_cargomax(1000); + player->set_officerrating(5); + player->add_hullids(11); + player->add_shipids(21); + player->add_initialshiplevels(30); + player->add_initialshiplevelpercentages(0.3f); + player->add_initialshiphps(5000); + player->mutable_initialcargo(); + auto* crew = player->add_fleetsofficers(); + crew->set_fleetid(31); + crew->set_crewsize(10); + auto* captain = crew->add_officers(); + captain->set_id(41); + captain->set_level(50); + captain->set_rank(3); + auto* officer = crew->add_officers(); + officer->set_id(42); + officer->set_level(40); + officer->set_rank(2); + officer->set_slotindex(3); + auto* fleet = player->mutable_deployedfleet(); + fleet->set_uid("player1"); + fleet->set_fleetid(31); + fleet->set_fleettype(game::DEPLOYEDFLEETTYPE_PLAYER); + fleet->set_causeofdestruction(game::DAMAGESOURCES_NOTSET); + fleet->add_shipids(21); + fleet->add_hullids(11); + (*fleet->mutable_attributes())[-9] = 1234.6f; + (*fleet->mutable_stats())[64] = 16.7f; + (*fleet->mutable_stats())[63] = 2.0f; + auto& ship = *(*fleet->mutable_shipstats())[21].mutable_shipattributes_(); + ship[61] = 5000.0f; + ship[60] = 2500.4f; + (*fleet->mutable_shiplevels())[21] = 30; + (*fleet->mutable_shiptiers())[21] = 4; + auto& components = (*fleet->mutable_deployedships())[21]; + components.add_componentids(101); + components.add_componentids(-1); + (*player->mutable_deployedfleets())[31] = *fleet; + + auto* npc = journal.mutable_targetfleetdata(); + npc->mutable_refids()->set_locaid(555); + npc->add_shipids(0); + auto* npc_fleet = npc->mutable_deployedfleet(); + npc_fleet->set_uid("mar_30"); + npc_fleet->set_fleetid(61); + npc_fleet->set_fleettype(game::DEPLOYEDFLEETTYPE_MARAUDER); + (*npc->mutable_deployedfleets())[61] = *npc_fleet; + return journal; +} +void ProtobufJournals() +{ + const auto body = BattleReportBody(SyntheticJournal()); + const auto converted = JournalFromProtobuf(body); + Check(!IsJsonBody(body) && converted.has_value(), "a protobuf battle report converts"); + // dump() compares the JSON number kinds too: 5.0 must stay a float and the ids integers. + const auto expected = json::parse(R"( + {"battle_context_data":{"context_data":{}},"battle_duration":5,"battle_log":[-96,1234567890123456789,5.0, + 1.2000000000000002,-97],"battle_time":"2025-11-20T00:53:20","battle_type":5, + "chest_drop":{"chest_id_refs":[{"art_file_reference":null,"loca_id":9}],"chests_gained":[{"count":1, + "expiry_time":null,"params":{"id_refs":{"art_file_reference":null,"loca_id":9},"ref_id":77},"type":2}], + "user_id_refs":[]},"coords":{"x":1.1,"y":-2.5},"id":2860000000000000001,"initiator_fired_first":false, + "initiator_fleet_data":{"armada_owner_id":null,"battle_data_type":0,"bridge_officers":[{"id":41,"level":50, + "rank":3},null,null,{"id":42,"level":40,"rank":2},null,null,null,null,null,null],"cargo_max":1000, + "cargo_max_list":[],"defense_rating":null,"deployed_fleet":{"active_buffs":[],"attributes":{"-9":1234.6}, + "battle_opponent_fleet_id":61,"battle_won":true,"cause_of_destruction":-1,"defense_rating":1235,"fleet_id":31, + "hull_ids":[11],"pursued_by_num":0,"pursuit_target_id":null,"ship_attributes":{},"ship_components":{"21":[101, + -1]},"ship_cosmetics":{},"ship_dmg":{},"ship_hps":{"21":5000},"ship_ids":[21],"ship_level_percentages":{}, + "ship_levels":{"21":30},"ship_shield_dmg":{},"ship_shield_hps":{"21":2500}, + "ship_shield_total_regeneration_durations":{},"ship_stats":{"21":{"60":2500.4,"61":5000.0}}, + "ship_tiers":{"21":4},"stats":{"63":2.0,"64":16.7},"status_counters":[],"status_effects":{},"type":1, + "uid":"player1","warp_data":null,"warp_distance":16.7,"warp_speed":2.0,"warp_time":null}, + "deployed_fleets":{"31":{"active_buffs":[],"attributes":{"-9":1234.6},"battle_opponent_fleet_id":61, + "battle_won":true,"cause_of_destruction":-1,"defense_rating":1235,"fleet_id":31,"hull_ids":[11], + "pursued_by_num":0,"pursuit_target_id":null,"ship_attributes":{},"ship_components":{"21":[101,-1]}, + "ship_cosmetics":{},"ship_dmg":{},"ship_hps":{"21":5000},"ship_ids":[21],"ship_level_percentages":{}, + "ship_levels":{"21":30},"ship_shield_dmg":{},"ship_shield_hps":{"21":2500}, + "ship_shield_total_regeneration_durations":{},"ship_stats":{"21":{"60":2500.4,"61":5000.0}}, + "ship_tiers":{"21":4},"stats":{"63":2.0,"64":16.7},"status_counters":[],"status_effects":{},"type":1, + "uid":"player1","warp_data":null,"warp_distance":16.7,"warp_speed":2.0,"warp_time":null}},"final_cargo_list":[], + "final_ship_hps":[],"final_ship_level_percentages":[],"final_ship_levels":[],"final_ship_shps":[],"fleet_id":31, + "fleets_officers":{"31":[{"id":41,"level":50,"rank":3},null,null,{"id":42,"level":40,"rank":2},null,null,null, + null,null,null]},"health_rating":null,"hull_ids":[11],"initial_cargo":{"resources":{}},"initial_cargo_list":[], + "initial_ship_hps":[5000],"initial_ship_level_percentages":[0.3],"initial_ship_levels":[30], + "initial_ship_shps":[],"max_ship_hps":[],"max_ship_shps":[],"num_drydocks":0,"offense_rating":null, + "officer_rating":5,"ref_ids":null,"ship_ids":[21],"xp_gained":null},"initiator_id":"player1", + "initiator_wins":true,"journal_src":"pb1","resources_dropped":{"2":100},"resources_transferred":{}, + "system_id":1234567890123,"target_fleet_data":{"armada_owner_id":null,"battle_data_type":0,"bridge_officers":{}, + "cargo_max":null,"cargo_max_list":[],"defense_rating":null,"deployed_fleet":{"active_buffs":[],"attributes":{}, + "battle_opponent_fleet_id":31,"battle_won":false,"cause_of_destruction":0,"fleet_id":61,"hull_ids":[], + "pursued_by_num":0,"pursuit_target_id":null,"ship_attributes":{},"ship_components":{},"ship_cosmetics":{}, + "ship_dmg":{},"ship_hps":{},"ship_ids":[],"ship_level_percentages":{},"ship_levels":{},"ship_shield_dmg":{}, + "ship_shield_hps":{},"ship_shield_total_regeneration_durations":{},"ship_stats":{},"ship_tiers":{},"stats":{}, + "status_counters":[],"status_effects":{},"type":2,"uid":"mar_30","warp_data":null,"warp_time":null}, + "deployed_fleets":{"61":{"active_buffs":[],"attributes":{},"battle_opponent_fleet_id":31,"battle_won":false, + "cause_of_destruction":0,"fleet_id":61,"hull_ids":[],"pursued_by_num":0,"pursuit_target_id":null, + "ship_attributes":{},"ship_components":{},"ship_cosmetics":{},"ship_dmg":{},"ship_hps":{},"ship_ids":[], + "ship_level_percentages":{},"ship_levels":{},"ship_shield_dmg":{},"ship_shield_hps":{}, + "ship_shield_total_regeneration_durations":{},"ship_stats":{},"ship_tiers":{},"stats":{},"status_counters":[], + "status_effects":{},"type":2,"uid":"mar_30","warp_data":null,"warp_time":null}},"final_cargo_list":[], + "final_ship_hps":[],"final_ship_level_percentages":[],"final_ship_levels":[],"final_ship_shps":[],"fleet_id":61, + "fleets_officers":{},"health_rating":null,"hull_ids":[],"initial_cargo_list":[],"initial_ship_hps":[], + "initial_ship_level_percentages":[],"initial_ship_levels":[],"initial_ship_shps":[],"max_ship_hps":[], + "max_ship_shps":[],"num_drydocks":0,"offense_rating":null,"officer_rating":0, + "ref_ids":{"art_file_reference":null,"loca_id":555},"ship_ids":[0],"xp_gained":null},"target_id":"mar_30"} + )"); + Check(converted && converted->dump() == expected.dump(), "a protobuf journal keeps the JSON journal shape"); + + Check(IsJsonBody(" \n{\"journal\":{}}"), "a JSON body keeps the JSON path"); + Check(!JournalFromProtobuf(R"({"journal":{}})"), "a JSON body is not read as protobuf"); + Check(!JournalFromProtobuf(""), "an empty body is refused"); + Check(!JournalFromProtobuf("\xff\xff\xff\xff garbage"), "garbage is refused"); + Check(!JournalFromProtobuf(body.substr(0, body.size() / 2)), "a truncated body is refused"); + Check(!JournalFromProtobuf(BattleReportBody(SyntheticJournal(), game::EntityGroup::TYPE_BATTLERESULTHEADERS)), + "a body without a battle report group is refused"); + Check(!JournalFromProtobuf(BattleReportBody(game::JournalResponse{})), "a journal without an id is refused"); +} } // namespace // Stand-ins for config.cc, sync.cc, battlelogs.cc and fleets.cc, which this test does not link: no threads, no game, @@ -821,6 +973,7 @@ int main() run("queues", Queues); run("name caches", NameCaches); run("logging", SyncLogging); + run("protobuf journals", ProtobufJournals); if (!failures) std::cout << "Sync wire and delivery tests passed\n"; return failures ? 1 : 0; diff --git a/zig/deps/protobuf_roots.txt b/zig/deps/protobuf_roots.txt index 70a62586e..44ac365ab 100644 --- a/zig/deps/protobuf_roots.txt +++ b/zig/deps/protobuf_roots.txt @@ -18,6 +18,7 @@ Digit.PrimeServer.Models.GlobalActiveBuffsResponse Digit.PrimeServer.Models.InventoryResponse Digit.PrimeServer.Models.Job Digit.PrimeServer.Models.JobResponse +Digit.PrimeServer.Models.JournalResponse # Declared by the mod in stfc.sync.proto; the game has no message for the MyDeployedFleets payload. Digit.PrimeServer.Models.MyDeployedFleetsResponse Digit.PrimeServer.Models.OfficerTraitsResponse @@ -30,6 +31,7 @@ Digit.PrimeServer.Models.PlayerShipsResponse Digit.PrimeServer.Models.ResearchTreesState Digit.PrimeServer.Models.ResourcesDeltaResponse Digit.PrimeServer.Models.ResourcesResponse +Digit.PrimeServer.Models.ServiceResponse Digit.PrimeServer.Models.StarbaseModulesResponse Digit.PrimeServer.Models.UserProfilesResponse diff --git a/zig/tests.zig b/zig/tests.zig index c8b30eac0..d2e6c7b75 100644 --- a/zig/tests.zig +++ b/zig/tests.zig @@ -136,7 +136,7 @@ pub fn add(build_ctx: *const Ctx, libs: Libs) void { inline for (.{ libs.proto, libs.proto_runtime, libs.spdlog, libs.toml, deps.nlohmannJson(ctx) }) |lib| lib.linkInto(module); module.addCSourceFiles(.{ - .files = &.{ "tests/sync_logic.cc", "src/mod/sync/records.cc", "src/mod/sync/state.cc" }, + .files = &.{ "tests/sync_logic.cc", "src/mod/sync/journal.cc", "src/mod/sync/records.cc", "src/mod/sync/state.cc" }, .flags = mod_flags, }); addFixture(ctx, build_tests, test_sync, "sync-logic-tests", module);