Set up basic deterministic lockstep
This commit is contained in:
1 parent
28c6e83eda
commit
23fef6f349
26 files changed
+1172
-239
No files matched your search
@@ -9,7 +9,7 @@ namespace FRG {
|
||||
|
||||
namespace {
|
||||
constexpr const char* kBattleZoneCampaignId = "battle_zone";
|
||||
constexpr const char* kBattleZoneTestBotId = "battle_zone_test_bot";
|
||||
constexpr const char* kBattleZonePlayer2Id = "player2";
|
||||
constexpr std::int64_t kBattleZoneTestBotIntervalMs = 5000;
|
||||
|
||||
void configurePlayerAnimations(
|
||||
@@ -44,11 +44,11 @@ void InternalGameServer::ensureBattleZoneTestBot()
|
||||
{
|
||||
if (state_.campaignId != kBattleZoneCampaignId) return;
|
||||
if (!battleZoneTestBotEnabled_) {
|
||||
state_.players.erase(kBattleZoneTestBotId);
|
||||
state_.players.erase(kBattleZonePlayer2Id);
|
||||
return;
|
||||
}
|
||||
|
||||
auto& bot = state_.players[kBattleZoneTestBotId];
|
||||
auto& bot = state_.players[kBattleZonePlayer2Id];
|
||||
if (!bot) bot = std::make_shared<CharacterState>();
|
||||
bot->configureAsPlayer();
|
||||
configurePlayerAnimations(*bot, resourceZipFile_);
|
||||
@@ -60,7 +60,7 @@ void InternalGameServer::updateBattleZoneTestBot(std::int64_t deltaMs)
|
||||
if (!battleZoneTestBotEnabled_ ||
|
||||
state_.campaignId != kBattleZoneCampaignId ||
|
||||
state_.currentLocationName.empty() ||
|
||||
state_.players.find(kBattleZoneTestBotId) == state_.players.end()) {
|
||||
state_.players.find(kBattleZonePlayer2Id) == state_.players.end()) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -72,7 +72,7 @@ void InternalGameServer::updateBattleZoneTestBot(std::int64_t deltaMs)
|
||||
static std::uniform_real_distribution<float> coordinate(-10.0f, 10.0f);
|
||||
submit(SetCharacterDestinationCommand{
|
||||
state_.currentLocationName,
|
||||
kBattleZoneTestBotId,
|
||||
kBattleZonePlayer2Id,
|
||||
Eigen::Vector3f(coordinate(randomEngine), 0.f, coordinate(randomEngine)),
|
||||
std::string()});
|
||||
}
|
||||
@@ -92,6 +92,7 @@ void InternalGameServer::startCampaign(
|
||||
bool createLocalPlayer, bool enableTestBot)
|
||||
{
|
||||
resourceZipFile_ = zipFile;
|
||||
networkPlayerSlots_ = campaign.networkPlayerSlots;
|
||||
battleZoneTestBotEnabled_ = enableTestBot;
|
||||
state_.campaignId = campaign.id;
|
||||
state_.currentLocationName.clear();
|
||||
@@ -125,6 +126,7 @@ void InternalGameServer::clearCampaign()
|
||||
state_.currentLocationName.clear();
|
||||
state_.players.clear();
|
||||
state_.locations.clear();
|
||||
networkPlayerSlots_.clear();
|
||||
battleZoneTestBotElapsedMs_ = 0;
|
||||
publishMutation();
|
||||
}
|
||||
@@ -487,6 +489,19 @@ void InternalGameServer::update(std::int64_t deltaMs)
|
||||
pendingSimulationLocationId_ = state_.currentLocationName;
|
||||
pendingSimulationEvents_ =
|
||||
stored->second.update(deltaMs, state_.players, interactiveObjectsLocked);
|
||||
// Navigation recovery is a simulation consequence, not presentation input.
|
||||
// Applying it here makes every lockstep replica produce the same follow-up.
|
||||
for (const CharacterSimulationEvent& event :
|
||||
pendingSimulationEvents_.characters) {
|
||||
if (!event.events.navigationRequested) continue;
|
||||
CharacterState* character = state_.findCharacter(
|
||||
state_.currentLocationName, event.characterId);
|
||||
if (character && stored->second.location) {
|
||||
character->setTarget(
|
||||
event.events.navigationTarget, {},
|
||||
*stored->second.location, event.characterId);
|
||||
}
|
||||
}
|
||||
publishMutation();
|
||||
}
|
||||
|
||||
@@ -499,6 +514,12 @@ void InternalGameServer::load(const nlohmann::json& in)
|
||||
{
|
||||
pendingEvents_.clear();
|
||||
state_.load(in);
|
||||
for (const std::string& playerId : networkPlayerSlots_) {
|
||||
auto player = state_.players.find(playerId);
|
||||
if (player == state_.players.end() || !player->second) continue;
|
||||
player->second->configureAsPlayer();
|
||||
configurePlayerAnimations(*player->second, resourceZipFile_);
|
||||
}
|
||||
battleZoneTestBotElapsedMs_ = 0;
|
||||
ensureBattleZoneTestBot();
|
||||
publishMutation();
|
||||
|
||||
@@ -23,6 +23,9 @@ public:
|
||||
void update(std::int64_t deltaMs) override;
|
||||
void save(nlohmann::json& out) const override;
|
||||
void load(const nlohmann::json& in) override;
|
||||
void setLocalPlayerId(std::string playerId) {
|
||||
localPlayerId_ = std::move(playerId);
|
||||
}
|
||||
void startCampaign(
|
||||
const CampaignConfig& campaign, const std::string& zipFile,
|
||||
bool createLocalPlayer, bool enableTestBot) override;
|
||||
@@ -32,6 +35,7 @@ public:
|
||||
private:
|
||||
friend class LocalGameServerHost;
|
||||
friend class RemoteGameServerHost;
|
||||
friend class WebsocketGameClient;
|
||||
|
||||
// Host-only lifecycle operations. They never expose authoritative state and
|
||||
// are intentionally absent from IGameServer.
|
||||
@@ -49,6 +53,7 @@ private:
|
||||
GameState state_;
|
||||
std::string localPlayerId_ = "player";
|
||||
std::string resourceZipFile_;
|
||||
std::vector<std::string> networkPlayerSlots_;
|
||||
std::int64_t battleZoneTestBotElapsedMs_ = 0;
|
||||
bool battleZoneTestBotEnabled_ = false;
|
||||
std::string pendingSimulationLocationId_;
|
||||
|
||||
@@ -1,24 +1,20 @@
|
||||
#include "NetworkProtocol.h"
|
||||
#include <cmath>
|
||||
#include <exception>
|
||||
#include <iomanip>
|
||||
#include <limits>
|
||||
#include <sstream>
|
||||
|
||||
|
||||
namespace FRG {
|
||||
namespace {
|
||||
|
||||
std::string makeStateMessage(const nlohmann::json& state)
|
||||
{
|
||||
return nlohmann::json{
|
||||
{"protocolVersion", kGameProtocolVersion},
|
||||
{"type", "state"},
|
||||
{"payload", state}
|
||||
}.dump();
|
||||
}
|
||||
|
||||
bool parseStateMessage(
|
||||
const std::string& message, nlohmann::json& state,
|
||||
std::string& error)
|
||||
bool parseEnvelope(
|
||||
const std::string& text, const char* expectedType,
|
||||
nlohmann::json& payload, std::string& error)
|
||||
{
|
||||
try {
|
||||
const nlohmann::json envelope = nlohmann::json::parse(message);
|
||||
const nlohmann::json envelope = nlohmann::json::parse(text);
|
||||
if (!envelope.is_object()) {
|
||||
error = "message must be a JSON object";
|
||||
return false;
|
||||
@@ -27,15 +23,15 @@ bool parseStateMessage(
|
||||
error = "unsupported protocol version";
|
||||
return false;
|
||||
}
|
||||
if (envelope.value("type", std::string()) != "state") {
|
||||
if (envelope.value("type", std::string()) != expectedType) {
|
||||
error = "unsupported message type";
|
||||
return false;
|
||||
}
|
||||
if (!envelope.contains("payload") || !envelope["payload"].is_object()) {
|
||||
error = "state payload must be an object";
|
||||
error = "message payload must be an object";
|
||||
return false;
|
||||
}
|
||||
state = envelope["payload"];
|
||||
payload = envelope["payload"];
|
||||
error.clear();
|
||||
return true;
|
||||
} catch (const std::exception& exception) {
|
||||
@@ -44,4 +40,229 @@ bool parseStateMessage(
|
||||
}
|
||||
}
|
||||
|
||||
std::string envelope(const char* type, nlohmann::json payload)
|
||||
{
|
||||
return nlohmann::json{
|
||||
{"protocolVersion", kGameProtocolVersion},
|
||||
{"type", type},
|
||||
{"payload", std::move(payload)}
|
||||
}.dump();
|
||||
}
|
||||
|
||||
bool readTick(
|
||||
const nlohmann::json& value, const char* name,
|
||||
std::uint64_t& out, std::string& error)
|
||||
{
|
||||
if (!value.contains(name) ||
|
||||
(!value[name].is_number_unsigned() && !value[name].is_number_integer())) {
|
||||
error = std::string(name) + " must be a non-negative integer";
|
||||
return false;
|
||||
}
|
||||
const auto signedValue = value[name].get<std::int64_t>();
|
||||
if (signedValue < 0) {
|
||||
error = std::string(name) + " must be non-negative";
|
||||
return false;
|
||||
}
|
||||
out = static_cast<std::uint64_t>(signedValue);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool readTarget(
|
||||
const nlohmann::json& value, Eigen::Vector3f& target, std::string& error)
|
||||
{
|
||||
if (!value.is_array() || value.size() != 3) {
|
||||
error = "target must contain three coordinates";
|
||||
return false;
|
||||
}
|
||||
for (std::size_t index = 0; index < 3; ++index) {
|
||||
if (!value[index].is_number()) {
|
||||
error = "target coordinates must be numbers";
|
||||
return false;
|
||||
}
|
||||
const float coordinate = value[index].get<float>();
|
||||
if (!std::isfinite(coordinate) || std::abs(coordinate) > 100000.f) {
|
||||
error = "target coordinate is not finite or is out of range";
|
||||
return false;
|
||||
}
|
||||
target[static_cast<Eigen::Index>(index)] =
|
||||
std::round(coordinate * 100.f) / 100.f;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
nlohmann::json targetJson(const Eigen::Vector3f& target)
|
||||
{
|
||||
return nlohmann::json::array({target.x(), target.y(), target.z()});
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
std::string makeInitialStateMessage(const InitialStateMessage& message)
|
||||
{
|
||||
return envelope("initial_state", {
|
||||
{"state", message.state},
|
||||
{"assignedPlayerId", message.assignedPlayerId},
|
||||
{"snapshotTick", message.snapshotTick},
|
||||
{"tickMs", message.tickMs},
|
||||
{"inputDelayTicks", message.inputDelayTicks},
|
||||
{"compatibilityFingerprint", message.compatibilityFingerprint}
|
||||
});
|
||||
}
|
||||
|
||||
bool parseInitialStateMessage(
|
||||
const std::string& text, InitialStateMessage& message, std::string& error)
|
||||
{
|
||||
nlohmann::json payload;
|
||||
if (!parseEnvelope(text, "initial_state", payload, error)) return false;
|
||||
if (!payload.contains("state") || !payload["state"].is_object() ||
|
||||
!payload.contains("assignedPlayerId") ||
|
||||
!payload["assignedPlayerId"].is_string() ||
|
||||
!payload.contains("compatibilityFingerprint") ||
|
||||
!payload["compatibilityFingerprint"].is_string()) {
|
||||
error = "invalid initial-state payload";
|
||||
return false;
|
||||
}
|
||||
if (!readTick(payload, "snapshotTick", message.snapshotTick, error))
|
||||
return false;
|
||||
message.tickMs = payload.value("tickMs", std::int64_t{0});
|
||||
message.inputDelayTicks =
|
||||
payload.value("inputDelayTicks", std::uint32_t{0});
|
||||
if (message.tickMs != kSimulationTickMs || message.inputDelayTicks == 0) {
|
||||
error = "unsupported lockstep timing";
|
||||
return false;
|
||||
}
|
||||
message.state = payload["state"];
|
||||
message.assignedPlayerId = payload["assignedPlayerId"].get<std::string>();
|
||||
message.compatibilityFingerprint =
|
||||
payload["compatibilityFingerprint"].get<std::string>();
|
||||
return !message.assignedPlayerId.empty();
|
||||
}
|
||||
|
||||
std::uint64_t LockstepTimeline::schedule(
|
||||
const std::string& playerId, const Eigen::Vector3f& target)
|
||||
{
|
||||
const std::uint64_t sequence = nextCommandSequence_++;
|
||||
pending_[currentTick_ + inputDelayTicks_].push_back(
|
||||
{sequence, playerId, target});
|
||||
return sequence;
|
||||
}
|
||||
|
||||
TickMessage LockstepTimeline::advance()
|
||||
{
|
||||
TickMessage result;
|
||||
result.tickId = ++currentTick_;
|
||||
auto commands = pending_.find(result.tickId);
|
||||
if (commands != pending_.end()) {
|
||||
result.commands = std::move(commands->second);
|
||||
pending_.erase(commands);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
std::string makeDestinationInputMessage(const DestinationInputMessage& message)
|
||||
{
|
||||
return envelope("destination_input", {
|
||||
{"clientSequence", message.clientSequence},
|
||||
{"target", targetJson(message.target)}
|
||||
});
|
||||
}
|
||||
|
||||
bool parseDestinationInputMessage(
|
||||
const std::string& text, DestinationInputMessage& message, std::string& error)
|
||||
{
|
||||
nlohmann::json payload;
|
||||
if (!parseEnvelope(text, "destination_input", payload, error)) return false;
|
||||
if (!readTick(payload, "clientSequence", message.clientSequence, error))
|
||||
return false;
|
||||
if (!payload.contains("target"))
|
||||
return error = "target is required", false;
|
||||
return readTarget(payload["target"], message.target, error);
|
||||
}
|
||||
|
||||
std::string makeTickMessage(const TickMessage& message)
|
||||
{
|
||||
nlohmann::json commands = nlohmann::json::array();
|
||||
for (const ScheduledDestinationCommand& command : message.commands) {
|
||||
commands.push_back({
|
||||
{"commandSequence", command.commandSequence},
|
||||
{"playerId", command.playerId},
|
||||
{"target", targetJson(command.target)}
|
||||
});
|
||||
}
|
||||
return envelope("tick", {
|
||||
{"tickId", message.tickId},
|
||||
{"commands", std::move(commands)},
|
||||
{"stateHash", message.stateHash}
|
||||
});
|
||||
}
|
||||
|
||||
bool parseTickMessage(
|
||||
const std::string& text, TickMessage& message, std::string& error)
|
||||
{
|
||||
nlohmann::json payload;
|
||||
if (!parseEnvelope(text, "tick", payload, error)) return false;
|
||||
if (!readTick(payload, "tickId", message.tickId, error)) return false;
|
||||
if (!payload.contains("commands") || !payload["commands"].is_array()) {
|
||||
error = "commands must be an array";
|
||||
return false;
|
||||
}
|
||||
message.commands.clear();
|
||||
std::uint64_t previousSequence = 0;
|
||||
for (const nlohmann::json& value : payload["commands"]) {
|
||||
ScheduledDestinationCommand command;
|
||||
if (!value.is_object() ||
|
||||
!readTick(value, "commandSequence", command.commandSequence, error) ||
|
||||
!value.contains("playerId") || !value["playerId"].is_string() ||
|
||||
!value.contains("target") ||
|
||||
!readTarget(value["target"], command.target, error)) {
|
||||
if (error.empty()) error = "invalid scheduled command";
|
||||
return false;
|
||||
}
|
||||
command.playerId = value["playerId"].get<std::string>();
|
||||
if (command.playerId.empty() || command.commandSequence == 0 ||
|
||||
command.commandSequence <= previousSequence) {
|
||||
error = "commands must have unique increasing sequences";
|
||||
return false;
|
||||
}
|
||||
previousSequence = command.commandSequence;
|
||||
message.commands.push_back(std::move(command));
|
||||
}
|
||||
message.stateHash = payload.value("stateHash", std::string());
|
||||
return true;
|
||||
}
|
||||
|
||||
std::string makeErrorMessage(const std::string& code, const std::string& detail)
|
||||
{
|
||||
return envelope("error", {{"code", code}, {"detail", detail}});
|
||||
}
|
||||
|
||||
bool parseErrorMessage(
|
||||
const std::string& text, std::string& code, std::string& detail)
|
||||
{
|
||||
nlohmann::json payload;
|
||||
std::string error;
|
||||
if (!parseEnvelope(text, "error", payload, error) ||
|
||||
!payload.contains("code") || !payload["code"].is_string() ||
|
||||
!payload.contains("detail") || !payload["detail"].is_string()) {
|
||||
return false;
|
||||
}
|
||||
code = payload["code"].get<std::string>();
|
||||
detail = payload["detail"].get<std::string>();
|
||||
return true;
|
||||
}
|
||||
|
||||
std::string canonicalStateHash(nlohmann::json state)
|
||||
{
|
||||
state.erase("savedAt");
|
||||
const std::string bytes = state.dump();
|
||||
std::uint64_t hash = 1469598103934665603ull;
|
||||
for (const unsigned char value : bytes) {
|
||||
hash ^= value;
|
||||
hash *= 1099511628211ull;
|
||||
}
|
||||
std::ostringstream output;
|
||||
output << std::hex << std::setfill('0') << std::setw(16) << hash;
|
||||
return output.str();
|
||||
}
|
||||
|
||||
} // namespace FRG
|
||||
@@ -1,15 +1,77 @@
|
||||
#pragma once
|
||||
|
||||
#include "../external/nlohmann/json.hpp"
|
||||
#include <Eigen/Dense>
|
||||
#include <cstdint>
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace FRG {
|
||||
|
||||
inline constexpr int kGameProtocolVersion = 1;
|
||||
inline constexpr int kGameProtocolVersion = 2;
|
||||
inline constexpr std::int64_t kSimulationTickMs = 100;
|
||||
inline constexpr std::uint32_t kDefaultInputDelayTicks = 2;
|
||||
|
||||
std::string makeStateMessage(const nlohmann::json& state);
|
||||
bool parseStateMessage(
|
||||
const std::string& message, nlohmann::json& state,
|
||||
std::string& error);
|
||||
struct InitialStateMessage {
|
||||
nlohmann::json state;
|
||||
std::string assignedPlayerId;
|
||||
std::uint64_t snapshotTick = 0;
|
||||
std::int64_t tickMs = kSimulationTickMs;
|
||||
std::uint32_t inputDelayTicks = kDefaultInputDelayTicks;
|
||||
std::string compatibilityFingerprint;
|
||||
};
|
||||
|
||||
struct DestinationInputMessage {
|
||||
std::uint64_t clientSequence = 0;
|
||||
Eigen::Vector3f target = Eigen::Vector3f::Zero();
|
||||
};
|
||||
|
||||
struct ScheduledDestinationCommand {
|
||||
std::uint64_t commandSequence = 0;
|
||||
std::string playerId;
|
||||
Eigen::Vector3f target = Eigen::Vector3f::Zero();
|
||||
};
|
||||
|
||||
struct TickMessage {
|
||||
std::uint64_t tickId = 0;
|
||||
std::vector<ScheduledDestinationCommand> commands;
|
||||
std::string stateHash;
|
||||
};
|
||||
|
||||
class LockstepTimeline {
|
||||
public:
|
||||
explicit LockstepTimeline(
|
||||
std::uint32_t inputDelayTicks = kDefaultInputDelayTicks)
|
||||
: inputDelayTicks_(inputDelayTicks) {}
|
||||
|
||||
std::uint64_t currentTick() const { return currentTick_; }
|
||||
std::uint64_t schedule(
|
||||
const std::string& playerId, const Eigen::Vector3f& target);
|
||||
TickMessage advance();
|
||||
|
||||
private:
|
||||
std::uint32_t inputDelayTicks_;
|
||||
std::uint64_t currentTick_ = 0;
|
||||
std::uint64_t nextCommandSequence_ = 1;
|
||||
std::map<std::uint64_t, std::vector<ScheduledDestinationCommand>> pending_;
|
||||
};
|
||||
|
||||
std::string makeInitialStateMessage(const InitialStateMessage& message);
|
||||
bool parseInitialStateMessage(
|
||||
const std::string& text, InitialStateMessage& message, std::string& error);
|
||||
std::string makeDestinationInputMessage(const DestinationInputMessage& message);
|
||||
bool parseDestinationInputMessage(
|
||||
const std::string& text, DestinationInputMessage& message, std::string& error);
|
||||
std::string makeTickMessage(const TickMessage& message);
|
||||
bool parseTickMessage(
|
||||
const std::string& text, TickMessage& message, std::string& error);
|
||||
std::string makeErrorMessage(const std::string& code, const std::string& detail);
|
||||
bool parseErrorMessage(
|
||||
const std::string& text, std::string& code, std::string& detail);
|
||||
|
||||
// Stable for the same executable/content build. Persistence-only wall-clock
|
||||
// fields are removed before hashing.
|
||||
std::string canonicalStateHash(nlohmann::json state);
|
||||
|
||||
} // namespace FRG
|
||||
@@ -20,6 +20,10 @@ void RemoteGameServerHost::initializeCampaign(
|
||||
CampaignRegistry::instance().find(campaignId);
|
||||
if (!campaign)
|
||||
throw std::runtime_error("Unknown remote campaign '" + campaignId + "'");
|
||||
client_.expectCompatibility(
|
||||
campaign->networkCompatibility.empty()
|
||||
? "lockstep-v2:" + campaignId
|
||||
: campaign->networkCompatibility);
|
||||
client_.replica_.startCampaign(*campaign, zipFile, true, false);
|
||||
}
|
||||
|
||||
@@ -38,6 +42,15 @@ void RemoteGameServerHost::resetCampaign(
|
||||
|
||||
bool RemoteGameServerHost::prepareLocation(const LocationConfig& config)
|
||||
{
|
||||
auto existing = client_.replica_.state_.locations.find(config.id);
|
||||
if (existing != client_.replica_.state_.locations.end() &&
|
||||
existing->second.definition) {
|
||||
// The location was prepared before initial_state. Loading the
|
||||
// authoritative snapshot overlays mutable state while retaining this
|
||||
// definition; preparing it again would reset the assigned player's
|
||||
// position to LocationConfig::playerPosition.
|
||||
return false;
|
||||
}
|
||||
return client_.replica_.prepareLocation(config);
|
||||
}
|
||||
|
||||
@@ -92,11 +105,12 @@ void RemoteGameServerHost::bindRuntimeLocation(
|
||||
void RemoteGameServerHost::update(std::int64_t deltaMs)
|
||||
{
|
||||
client_.update(deltaMs);
|
||||
const auto state = client_.snapshot();
|
||||
if (!state || state->currentLocationId.empty()) return;
|
||||
auto runtime = runtimeLocations_.find(state->currentLocationId);
|
||||
if (runtime == runtimeLocations_.end() || !runtime->second) return;
|
||||
runtime->second->update(deltaMs, LocationSimulationEvents{});
|
||||
for (auto& completed : client_.takeCompletedSimulationTicks()) {
|
||||
if (completed.locationId.empty()) continue;
|
||||
auto runtime = runtimeLocations_.find(completed.locationId);
|
||||
if (runtime == runtimeLocations_.end() || !runtime->second) continue;
|
||||
runtime->second->update(kSimulationTickMs, completed.events);
|
||||
}
|
||||
}
|
||||
|
||||
void RemoteGameServerHost::save(nlohmann::json&)
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
#include <boost/asio.hpp>
|
||||
#include <boost/beast.hpp>
|
||||
#include <atomic>
|
||||
#include <deque>
|
||||
#include <iostream>
|
||||
#include <mutex>
|
||||
#include <thread>
|
||||
@@ -28,6 +29,8 @@ public:
|
||||
, port_(std::move(port))
|
||||
, path_(std::move(path))
|
||||
{
|
||||
std::cout << "[network] Resolving ws://" << host_ << ':' << port_
|
||||
<< path_ << std::endl;
|
||||
resolve();
|
||||
thread_ = std::thread([this]() { io_.run(); });
|
||||
}
|
||||
@@ -49,6 +52,16 @@ public:
|
||||
bool connected() const { return connected_.load(); }
|
||||
bool connectionFailed() const { return connectionFailed_.load(); }
|
||||
|
||||
void send(std::string message)
|
||||
{
|
||||
asio::post(io_, [this, message = std::move(message)]() mutable {
|
||||
if (!connected_.load()) return;
|
||||
const bool writeInProgress = !outbound_.empty();
|
||||
outbound_.push_back(std::move(message));
|
||||
if (!writeInProgress) write();
|
||||
});
|
||||
}
|
||||
|
||||
private:
|
||||
void resolve()
|
||||
{
|
||||
@@ -56,11 +69,15 @@ private:
|
||||
host_, port_,
|
||||
[this](beast::error_code error, tcp::resolver::results_type results) {
|
||||
if (error) return fail("resolve", error);
|
||||
std::cout << "[network] WebSocket host resolved; connecting"
|
||||
<< std::endl;
|
||||
asio::async_connect(
|
||||
socket_.next_layer(), results,
|
||||
[this](beast::error_code connectError,
|
||||
const tcp::endpoint&) {
|
||||
if (connectError) return fail("connect", connectError);
|
||||
std::cout << "[network] TCP connected; starting WebSocket handshake"
|
||||
<< std::endl;
|
||||
socket_.set_option(
|
||||
websocket::stream_base::timeout::suggested(
|
||||
beast::role_type::client));
|
||||
@@ -72,7 +89,7 @@ private:
|
||||
connected_.store(true);
|
||||
std::cout
|
||||
<< "[network] Connected to ws://"
|
||||
<< host_ << ':' << port_ << path_ << '\n';
|
||||
<< host_ << ':' << port_ << path_ << std::endl;
|
||||
read();
|
||||
});
|
||||
});
|
||||
@@ -99,6 +116,17 @@ private:
|
||||
});
|
||||
}
|
||||
|
||||
void write()
|
||||
{
|
||||
socket_.async_write(
|
||||
asio::buffer(outbound_.front()),
|
||||
[this](beast::error_code error, std::size_t) {
|
||||
if (error) return fail("write", error);
|
||||
outbound_.pop_front();
|
||||
if (!outbound_.empty()) write();
|
||||
});
|
||||
}
|
||||
|
||||
void fail(const char* operation, beast::error_code error)
|
||||
{
|
||||
connected_.store(false);
|
||||
@@ -119,6 +147,7 @@ private:
|
||||
std::atomic_bool connectionFailed_{false};
|
||||
std::mutex messagesMutex_;
|
||||
std::vector<std::string> messages_;
|
||||
std::deque<std::string> outbound_;
|
||||
};
|
||||
|
||||
WebsocketGameClient::WebsocketGameClient(
|
||||
@@ -140,38 +169,122 @@ const std::string& WebsocketGameClient::localPlayerId() const
|
||||
return replica_.localPlayerId();
|
||||
}
|
||||
|
||||
void WebsocketGameClient::submit(const GameCommand&)
|
||||
void WebsocketGameClient::submit(const GameCommand& command)
|
||||
{
|
||||
// Commands intentionally remain local no-ops in the first iteration.
|
||||
if (!initialized_) return;
|
||||
if (const auto* destination =
|
||||
std::get_if<SetCharacterDestinationCommand>(&command)) {
|
||||
impl_->send(makeDestinationInputMessage(
|
||||
{nextClientSequence_++, destination->target}));
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<GameServerEvent> WebsocketGameClient::drainEvents()
|
||||
{
|
||||
return {};
|
||||
std::vector<GameServerEvent> result;
|
||||
result.swap(pendingEvents_);
|
||||
return result;
|
||||
}
|
||||
|
||||
void WebsocketGameClient::update(std::int64_t)
|
||||
{
|
||||
std::vector<std::string> messages = impl_->takeMessages();
|
||||
if (messages.empty()) return;
|
||||
|
||||
// State messages are complete snapshots; only the newest queued one matters.
|
||||
for (auto it = messages.rbegin(); it != messages.rend(); ++it) {
|
||||
nlohmann::json state;
|
||||
for (const std::string& text : messages) {
|
||||
std::string error;
|
||||
if (!parseStateMessage(*it, state, error)) {
|
||||
std::cerr << "[network] Ignoring WebSocket message: "
|
||||
<< error << '\n';
|
||||
InitialStateMessage initial;
|
||||
if (parseInitialStateMessage(text, initial, error)) {
|
||||
if (!expectedCompatibilityFingerprint_.empty() &&
|
||||
initial.compatibilityFingerprint !=
|
||||
expectedCompatibilityFingerprint_) {
|
||||
std::cerr << "[network] Incompatible server content: expected "
|
||||
<< expectedCompatibilityFingerprint_ << ", received "
|
||||
<< initial.compatibilityFingerprint << '\n';
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
replica_.load(initial.state);
|
||||
replica_.setLocalPlayerId(initial.assignedPlayerId);
|
||||
nextExpectedTick_ = initial.snapshotTick + 1;
|
||||
compatibilityFingerprint_ =
|
||||
std::move(initial.compatibilityFingerprint);
|
||||
pendingTicks_.clear();
|
||||
initialized_ = true;
|
||||
std::cout << "[network] Applied initial state at tick "
|
||||
<< initial.snapshotTick << " as player '"
|
||||
<< initial.assignedPlayerId << "' (delay "
|
||||
<< initial.inputDelayTicks << " ticks)" << std::endl;
|
||||
} catch (const std::exception& exception) {
|
||||
std::cerr << "[network] Invalid initial state: "
|
||||
<< exception.what() << '\n';
|
||||
}
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
replica_.load(state);
|
||||
} catch (const std::exception& exception) {
|
||||
std::cerr << "[network] Ignoring invalid game state: "
|
||||
<< exception.what() << '\n';
|
||||
TickMessage tick;
|
||||
if (parseTickMessage(text, tick, error)) {
|
||||
if (initialized_ && tick.tickId >= nextExpectedTick_) {
|
||||
if (!loggedFirstTick_) {
|
||||
std::cout << "[network] Received first lockstep tick "
|
||||
<< tick.tickId << std::endl;
|
||||
loggedFirstTick_ = true;
|
||||
}
|
||||
pendingTicks_.emplace(tick.tickId, std::move(tick));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
std::string errorCode;
|
||||
std::string errorDetail;
|
||||
if (parseErrorMessage(text, errorCode, errorDetail)) {
|
||||
std::cerr << "[network] Server rejected request (" << errorCode
|
||||
<< "): " << errorDetail << '\n';
|
||||
if (errorCode == "server_full" ||
|
||||
errorCode == "incompatible")
|
||||
rejected_ = true;
|
||||
continue;
|
||||
}
|
||||
std::cerr << "[network] Ignoring WebSocket message: "
|
||||
<< error << '\n';
|
||||
}
|
||||
|
||||
while (initialized_) {
|
||||
auto next = pendingTicks_.find(nextExpectedTick_);
|
||||
if (next == pendingTicks_.end()) break;
|
||||
TickMessage tick = std::move(next->second);
|
||||
pendingTicks_.erase(next);
|
||||
const std::string locationId =
|
||||
replica_.snapshot()->currentLocationId;
|
||||
for (const ScheduledDestinationCommand& command : tick.commands) {
|
||||
replica_.submit(SetCharacterDestinationCommand{
|
||||
locationId, command.playerId, command.target, {}});
|
||||
}
|
||||
replica_.update(kSimulationTickMs);
|
||||
completedSimulationTicks_.push_back({
|
||||
replica_.pendingSimulationLocationId_,
|
||||
replica_.pendingSimulationEvents_});
|
||||
std::vector<GameServerEvent> events = replica_.drainEvents();
|
||||
pendingEvents_.insert(
|
||||
pendingEvents_.end(),
|
||||
std::make_move_iterator(events.begin()),
|
||||
std::make_move_iterator(events.end()));
|
||||
if (!tick.stateHash.empty()) {
|
||||
nlohmann::json state;
|
||||
replica_.save(state);
|
||||
const std::string localHash = canonicalStateHash(std::move(state));
|
||||
if (localHash != tick.stateHash) {
|
||||
std::cerr << "[network] Lockstep divergence at tick "
|
||||
<< tick.tickId << ": server=" << tick.stateHash
|
||||
<< ", client=" << localHash << '\n';
|
||||
}
|
||||
}
|
||||
++nextExpectedTick_;
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<WebsocketGameClient::CompletedSimulationTick>
|
||||
WebsocketGameClient::takeCompletedSimulationTicks()
|
||||
{
|
||||
std::vector<CompletedSimulationTick> result;
|
||||
result.swap(completedSimulationTicks_);
|
||||
return result;
|
||||
}
|
||||
|
||||
void WebsocketGameClient::save(nlohmann::json&) const
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
#include "IGameServer.h"
|
||||
#include "InternalGameServer.h"
|
||||
#include "NetworkProtocol.h"
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
@@ -30,12 +32,32 @@ public:
|
||||
|
||||
bool connected() const;
|
||||
bool connectionFailed() const;
|
||||
bool initialized() const { return initialized_; }
|
||||
bool rejected() const { return rejected_; }
|
||||
|
||||
private:
|
||||
friend class RemoteGameServerHost;
|
||||
struct CompletedSimulationTick {
|
||||
std::string locationId;
|
||||
LocationSimulationEvents events;
|
||||
};
|
||||
std::vector<CompletedSimulationTick> takeCompletedSimulationTicks();
|
||||
void expectCompatibility(std::string fingerprint) {
|
||||
expectedCompatibilityFingerprint_ = std::move(fingerprint);
|
||||
}
|
||||
class Impl;
|
||||
InternalGameServer replica_;
|
||||
std::unique_ptr<Impl> impl_;
|
||||
std::map<std::uint64_t, TickMessage> pendingTicks_;
|
||||
std::vector<CompletedSimulationTick> completedSimulationTicks_;
|
||||
std::vector<GameServerEvent> pendingEvents_;
|
||||
std::uint64_t nextExpectedTick_ = 0;
|
||||
std::uint64_t nextClientSequence_ = 1;
|
||||
std::string compatibilityFingerprint_;
|
||||
std::string expectedCompatibilityFingerprint_;
|
||||
bool initialized_ = false;
|
||||
bool rejected_ = false;
|
||||
bool loggedFirstTick_ = false;
|
||||
};
|
||||
|
||||
} // namespace FRG
|
||||
@@ -3,10 +3,14 @@
|
||||
#include "NetworkProtocol.h"
|
||||
#include <boost/asio.hpp>
|
||||
#include <boost/beast.hpp>
|
||||
#include <algorithm>
|
||||
#include <deque>
|
||||
#include <iostream>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <thread>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
namespace FRG {
|
||||
@@ -19,13 +23,26 @@ using tcp = asio::ip::tcp;
|
||||
|
||||
class ServerSession : public std::enable_shared_from_this<ServerSession> {
|
||||
public:
|
||||
explicit ServerSession(tcp::socket socket)
|
||||
using InputHandler = std::function<void(
|
||||
const std::shared_ptr<ServerSession>&, const std::string&)>;
|
||||
using CloseHandler =
|
||||
std::function<void(const std::shared_ptr<ServerSession>&)>;
|
||||
|
||||
ServerSession(
|
||||
tcp::socket socket, std::string playerId,
|
||||
InputHandler inputHandler, CloseHandler closeHandler)
|
||||
: socket_(std::move(socket))
|
||||
, playerId_(std::move(playerId))
|
||||
, inputHandler_(std::move(inputHandler))
|
||||
, closeHandler_(std::move(closeHandler))
|
||||
{
|
||||
}
|
||||
|
||||
void start(std::string initialState)
|
||||
const std::string& playerId() const { return playerId_; }
|
||||
|
||||
void start(std::string initialState, bool closeAfterWrite = false)
|
||||
{
|
||||
closeAfterWrite_ = closeAfterWrite;
|
||||
socket_.set_option(
|
||||
websocket::stream_base::timeout::suggested(beast::role_type::server));
|
||||
socket_.async_accept(
|
||||
@@ -38,7 +55,7 @@ public:
|
||||
}
|
||||
self->socket_.text(true);
|
||||
self->send(std::move(initialState));
|
||||
self->read();
|
||||
if (!self->closeAfterWrite_) self->read();
|
||||
});
|
||||
}
|
||||
|
||||
@@ -59,10 +76,13 @@ private:
|
||||
if (error != websocket::error::closed)
|
||||
std::cerr << "[network] WebSocket read failed: "
|
||||
<< error.message() << '\n';
|
||||
self->notifyClosed();
|
||||
return;
|
||||
}
|
||||
// Commands are intentionally ignored in the first iteration.
|
||||
const std::string message =
|
||||
beast::buffers_to_string(self->inbound_.data());
|
||||
self->inbound_.consume(self->inbound_.size());
|
||||
self->inputHandler_(self, message);
|
||||
self->read();
|
||||
});
|
||||
}
|
||||
@@ -78,22 +98,50 @@ private:
|
||||
return;
|
||||
}
|
||||
self->outbound_.pop_front();
|
||||
if (!self->outbound_.empty()) self->write();
|
||||
if (!self->outbound_.empty()) {
|
||||
self->write();
|
||||
} else if (self->closeAfterWrite_) {
|
||||
self->socket_.async_close(
|
||||
websocket::close_code::policy_error,
|
||||
[self](beast::error_code) { self->notifyClosed(); });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void notifyClosed()
|
||||
{
|
||||
if (closed_) return;
|
||||
closed_ = true;
|
||||
if (closeHandler_) closeHandler_(shared_from_this());
|
||||
}
|
||||
|
||||
websocket::stream<tcp::socket> socket_;
|
||||
beast::flat_buffer inbound_;
|
||||
std::deque<std::string> outbound_;
|
||||
std::string playerId_;
|
||||
InputHandler inputHandler_;
|
||||
CloseHandler closeHandler_;
|
||||
bool closeAfterWrite_ = false;
|
||||
bool closed_ = false;
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
class WebsocketGameServer::Impl {
|
||||
public:
|
||||
Impl(const std::string& address, std::uint16_t port, std::string initialState)
|
||||
struct PendingInput {
|
||||
std::string playerId;
|
||||
DestinationInputMessage input;
|
||||
};
|
||||
|
||||
Impl(
|
||||
const std::string& address, std::uint16_t port,
|
||||
std::vector<std::string> playerSlots, nlohmann::json initialState,
|
||||
std::string compatibilityFingerprint)
|
||||
: acceptor_(io_)
|
||||
, playerSlots_(std::move(playerSlots))
|
||||
, latestState_(std::move(initialState))
|
||||
, compatibilityFingerprint_(std::move(compatibilityFingerprint))
|
||||
{
|
||||
const tcp::endpoint endpoint(asio::ip::make_address(address), port);
|
||||
acceptor_.open(endpoint.protocol());
|
||||
@@ -115,14 +163,33 @@ public:
|
||||
if (thread_.joinable()) thread_.join();
|
||||
}
|
||||
|
||||
void broadcast(std::string state)
|
||||
std::vector<PendingInput> takeInputs()
|
||||
{
|
||||
asio::post(io_, [this, state = std::move(state)]() mutable {
|
||||
latestState_ = std::move(state);
|
||||
std::lock_guard<std::mutex> lock(inputsMutex_);
|
||||
std::vector<PendingInput> result;
|
||||
result.swap(pendingInputs_);
|
||||
return result;
|
||||
}
|
||||
|
||||
std::vector<std::string> takeReleasedPlayers()
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(inputsMutex_);
|
||||
std::vector<std::string> result;
|
||||
result.swap(releasedPlayers_);
|
||||
return result;
|
||||
}
|
||||
|
||||
void publishTick(
|
||||
std::string tick, nlohmann::json latestState, std::uint64_t tickId)
|
||||
{
|
||||
asio::post(io_, [this, tick = std::move(tick),
|
||||
latestState = std::move(latestState), tickId]() mutable {
|
||||
latestState_ = std::move(latestState);
|
||||
latestTick_ = tickId;
|
||||
auto output = sessions_.begin();
|
||||
for (auto input = sessions_.begin(); input != sessions_.end(); ++input) {
|
||||
if (auto session = input->lock()) {
|
||||
session->send(latestState_);
|
||||
session->send(tick);
|
||||
*output++ = *input;
|
||||
}
|
||||
}
|
||||
@@ -138,10 +205,62 @@ private:
|
||||
acceptor_.async_accept(
|
||||
[this](beast::error_code error, tcp::socket socket) {
|
||||
if (!error) {
|
||||
std::string playerId;
|
||||
for (const std::string& candidate : playerSlots_) {
|
||||
if (assignedSlots_.count(candidate) == 0) {
|
||||
playerId = candidate;
|
||||
assignedSlots_[candidate] = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
auto session =
|
||||
std::make_shared<ServerSession>(std::move(socket));
|
||||
sessions_.push_back(session);
|
||||
session->start(latestState_);
|
||||
std::make_shared<ServerSession>(
|
||||
std::move(socket), playerId,
|
||||
[this](const std::shared_ptr<ServerSession>& source,
|
||||
const std::string& message) {
|
||||
DestinationInputMessage parsed;
|
||||
std::string parseError;
|
||||
if (!parseDestinationInputMessage(
|
||||
message, parsed, parseError)) {
|
||||
source->send(makeErrorMessage(
|
||||
"invalid_input", parseError));
|
||||
return;
|
||||
}
|
||||
std::lock_guard<std::mutex> lock(inputsMutex_);
|
||||
pendingInputs_.push_back(
|
||||
{source->playerId(), std::move(parsed)});
|
||||
},
|
||||
[this](const std::shared_ptr<ServerSession>& source) {
|
||||
if (!source->playerId().empty())
|
||||
std::cout << "[network] Player slot '"
|
||||
<< source->playerId()
|
||||
<< "' disconnected" << std::endl;
|
||||
assignedSlots_.erase(source->playerId());
|
||||
std::lock_guard<std::mutex> lock(inputsMutex_);
|
||||
releasedPlayers_.push_back(source->playerId());
|
||||
});
|
||||
if (playerId.empty()) {
|
||||
std::cout << "[network] Rejecting connection: no player slot"
|
||||
<< std::endl;
|
||||
session->start(
|
||||
makeErrorMessage(
|
||||
"server_full", "No multiplayer player slot is available"),
|
||||
true);
|
||||
} else {
|
||||
std::cout << "[network] Accepted connection as player slot '"
|
||||
<< playerId << "' at snapshot tick "
|
||||
<< latestTick_ << std::endl;
|
||||
sessions_.push_back(session);
|
||||
InitialStateMessage initial;
|
||||
initial.state = latestState_;
|
||||
initial.assignedPlayerId = playerId;
|
||||
initial.snapshotTick = latestTick_;
|
||||
initial.compatibilityFingerprint =
|
||||
compatibilityFingerprint_;
|
||||
session->start(makeInitialStateMessage(initial));
|
||||
std::cout << "[network] Initial state queued for player '"
|
||||
<< playerId << "'" << std::endl;
|
||||
}
|
||||
} else if (error != asio::error::operation_aborted) {
|
||||
std::cerr << "[network] TCP accept failed: "
|
||||
<< error.message() << '\n';
|
||||
@@ -154,12 +273,21 @@ private:
|
||||
tcp::acceptor acceptor_;
|
||||
std::thread thread_;
|
||||
std::vector<std::weak_ptr<ServerSession>> sessions_;
|
||||
std::string latestState_;
|
||||
std::vector<std::string> playerSlots_;
|
||||
std::unordered_map<std::string, bool> assignedSlots_;
|
||||
nlohmann::json latestState_;
|
||||
std::string compatibilityFingerprint_;
|
||||
std::uint64_t latestTick_ = 0;
|
||||
std::mutex inputsMutex_;
|
||||
std::vector<PendingInput> pendingInputs_;
|
||||
std::vector<std::string> releasedPlayers_;
|
||||
std::uint16_t port_ = 0;
|
||||
};
|
||||
|
||||
WebsocketGameServer::WebsocketGameServer(
|
||||
InternalGameServer& gameServer,
|
||||
std::vector<std::string> playerSlots,
|
||||
std::string compatibilityFingerprint,
|
||||
const std::string& address,
|
||||
std::uint16_t port)
|
||||
: gameServer_(gameServer)
|
||||
@@ -167,26 +295,34 @@ WebsocketGameServer::WebsocketGameServer(
|
||||
nlohmann::json state;
|
||||
gameServer_.save(state);
|
||||
impl_ = std::make_unique<Impl>(
|
||||
address, port, makeStateMessage(state));
|
||||
address, port, std::move(playerSlots), std::move(state),
|
||||
std::move(compatibilityFingerprint));
|
||||
}
|
||||
|
||||
WebsocketGameServer::~WebsocketGameServer() = default;
|
||||
|
||||
void WebsocketGameServer::update(std::int64_t deltaMs)
|
||||
void WebsocketGameServer::update()
|
||||
{
|
||||
gameServer_.update(deltaMs);
|
||||
broadcastElapsedMs_ += deltaMs;
|
||||
if (broadcastElapsedMs_ >= 5000) {
|
||||
broadcastElapsedMs_ %= 5000;
|
||||
broadcastState();
|
||||
for (const std::string& playerId : impl_->takeReleasedPlayers())
|
||||
lastClientSequences_.erase(playerId);
|
||||
for (const Impl::PendingInput& pending : impl_->takeInputs()) {
|
||||
std::uint64_t& previous = lastClientSequences_[pending.playerId];
|
||||
if (pending.input.clientSequence <= previous) continue;
|
||||
previous = pending.input.clientSequence;
|
||||
timeline_.schedule(pending.playerId, pending.input.target);
|
||||
}
|
||||
}
|
||||
|
||||
void WebsocketGameServer::broadcastState()
|
||||
{
|
||||
TickMessage tick = timeline_.advance();
|
||||
for (const ScheduledDestinationCommand& command : tick.commands) {
|
||||
gameServer_.submit(SetCharacterDestinationCommand{
|
||||
gameServer_.snapshot()->currentLocationId,
|
||||
command.playerId, command.target, {}});
|
||||
}
|
||||
gameServer_.update(kSimulationTickMs);
|
||||
nlohmann::json state;
|
||||
gameServer_.save(state);
|
||||
impl_->broadcast(makeStateMessage(state));
|
||||
tick.stateHash = canonicalStateHash(state);
|
||||
impl_->publishTick(makeTickMessage(tick), std::move(state), tick.tickId);
|
||||
}
|
||||
|
||||
std::uint16_t WebsocketGameServer::port() const
|
||||
|
||||
@@ -3,6 +3,9 @@
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
#include "NetworkProtocol.h"
|
||||
|
||||
namespace FRG {
|
||||
|
||||
@@ -12,6 +15,8 @@ class WebsocketGameServer {
|
||||
public:
|
||||
WebsocketGameServer(
|
||||
InternalGameServer& gameServer,
|
||||
std::vector<std::string> playerSlots,
|
||||
std::string compatibilityFingerprint,
|
||||
const std::string& address = "0.0.0.0",
|
||||
std::uint16_t port = 8080);
|
||||
~WebsocketGameServer();
|
||||
@@ -19,15 +24,15 @@ public:
|
||||
WebsocketGameServer(const WebsocketGameServer&) = delete;
|
||||
WebsocketGameServer& operator=(const WebsocketGameServer&) = delete;
|
||||
|
||||
void update(std::int64_t deltaMs);
|
||||
void broadcastState();
|
||||
void update();
|
||||
std::uint16_t port() const;
|
||||
|
||||
private:
|
||||
class Impl;
|
||||
InternalGameServer& gameServer_;
|
||||
std::unique_ptr<Impl> impl_;
|
||||
std::int64_t broadcastElapsedMs_ = 0;
|
||||
LockstepTimeline timeline_;
|
||||
std::unordered_map<std::string, std::uint64_t> lastClientSequences_;
|
||||
};
|
||||
|
||||
} // namespace FRG
|
||||
Reference in new issue
Block a user