From fa8eee53869eb12d51d7e763d140d9492af9679a Mon Sep 17 00:00:00 2001 From: Emil Axelsson Date: Mon, 19 Sep 2016 14:49:57 +0200 Subject: [PATCH] sync camera movement over parallel connection --- .../interaction/interactionhandler.h | 3 + .../openspace/interaction/interactionmode.h | 4 +- .../openspace/network/parallelconnection.h | 4 ++ src/engine/openspaceengine.cpp | 4 ++ src/interaction/interactionhandler.cpp | 19 ++++++ src/interaction/interactionmode.cpp | 64 ++++++++++++++++--- src/network/parallelconnection.cpp | 38 ++++++++++- 7 files changed, 126 insertions(+), 10 deletions(-) diff --git a/include/openspace/interaction/interactionhandler.h b/include/openspace/interaction/interactionhandler.h index 18170bcab2..01187de2b8 100644 --- a/include/openspace/interaction/interactionhandler.h +++ b/include/openspace/interaction/interactionhandler.h @@ -53,6 +53,9 @@ public: InteractionHandler(); ~InteractionHandler(); + void initialize(); + void deinitialize(); + // Mutators void setFocusNode(SceneGraphNode* node); void setCamera(Camera* camera); diff --git a/include/openspace/interaction/interactionmode.h b/include/openspace/interaction/interactionmode.h index 87f3d9f233..1d248c6f62 100644 --- a/include/openspace/interaction/interactionmode.h +++ b/include/openspace/interaction/interactionmode.h @@ -58,13 +58,14 @@ namespace interaction { // Mutators void addKeyframe(const network::datamessagestructures::CameraKeyframe &kf); void clearKeyframes(); + void clearOldKeyframes(); // Accessors const std::list >& getPressedKeys() const; const std::list& getPressedMouseButtons() const; glm::dvec2 getMousePosition() const; double getMouseScrollDelta() const; - std::vector& getKeyFrames() const; + const std::vector& keyframes() const; bool isKeyPressed(std::pair keyModPair) const; bool isMouseButtonPressed(MouseButton mouseButton) const; @@ -174,6 +175,7 @@ public: virtual void serialize(SyncBuffer* syncBuffer) {}; virtual void deserialize(SyncBuffer* syncBuffer) {}; private: + std::vector _keyframes; double _currentKeyframeTime; }; diff --git a/include/openspace/network/parallelconnection.h b/include/openspace/network/parallelconnection.h index 3731401869..bd02a381f5 100644 --- a/include/openspace/network/parallelconnection.h +++ b/include/openspace/network/parallelconnection.h @@ -244,6 +244,10 @@ namespace openspace { std::map _currentState; std::mutex _currentStateMutex; + std::mutex _latencyMutex; + std::deque _latencyDiffs; + double _initialTimeDiff; + std::shared_ptr> _connectionEvent; }; } // namespace network diff --git a/src/engine/openspaceengine.cpp b/src/engine/openspaceengine.cpp index 509b8248b5..f65243171f 100644 --- a/src/engine/openspaceengine.cpp +++ b/src/engine/openspaceengine.cpp @@ -164,6 +164,7 @@ OpenSpaceEngine::~OpenSpaceEngine() { #ifdef OPENSPACE_MODULE_ONSCREENGUI_ENABLED _gui->deinitializeGL(); #endif + _interactionHandler->deinitialize(); _renderEngine->deinitialize(); _globalPropertyNamespace = nullptr; @@ -425,6 +426,9 @@ bool OpenSpaceEngine::initialize() { _settingsEngine->initialize(); _settingsEngine->setModules(_moduleEngine->modules()); + // Initialize the InteractionHandler + _interactionHandler->initialize(); + // Initialize the Scene Scene* sceneGraph = new Scene; sceneGraph->initialize(); diff --git a/src/interaction/interactionhandler.cpp b/src/interaction/interactionhandler.cpp index 6fc4edc188..9330d51396 100644 --- a/src/interaction/interactionhandler.cpp +++ b/src/interaction/interactionhandler.cpp @@ -137,6 +137,25 @@ InteractionHandler::~InteractionHandler() { } +void InteractionHandler::initialize() { + OsEng.parallelConnection().connectionEvent()->subscribe("interactionHandler", "statusChanged", [this]() { + if (OsEng.parallelConnection().status() == network::Status::ClientWithHost) { + setInteractionMode("Keyframe"); + } else { + auto keyframeModeIter = _interactionModes.find("Keyframe"); + if (keyframeModeIter != _interactionModes.end()) { + if (_currentInteractionMode == keyframeModeIter->second) { + setInteractionMode("Orbital"); + } + } + } + }); +} + +void InteractionHandler::deinitialize() { + OsEng.parallelConnection().connectionEvent()->unsubscribe("interactionHandler"); +} + void InteractionHandler::setFocusNode(SceneGraphNode* node) { _currentInteractionMode->setFocusNode(node); } diff --git a/src/interaction/interactionmode.cpp b/src/interaction/interactionmode.cpp index a863efc708..c79c12be17 100644 --- a/src/interaction/interactionmode.cpp +++ b/src/interaction/interactionmode.cpp @@ -57,14 +57,35 @@ namespace interaction { } - void InputState::addKeyframe(const network::datamessagestructures::CameraKeyframe &kf) { - - //save a maximum of 10 samples (1 seconds of buffer) - if (_keyframes.size() >= 10) { - _keyframes.erase(_keyframes.begin()); - } - _keyframes.push_back(kf); + const std::vector& InputState::keyframes() const { + return _keyframes; + } + void InputState::addKeyframe(const network::datamessagestructures::CameraKeyframe &kf) { + clearOldKeyframes(); + + auto compareTimestamps = [](const network::datamessagestructures::CameraKeyframe a, + network::datamessagestructures::CameraKeyframe b) { + return a._timestamp < b._timestamp; + }; + + // Remove keyframes after the inserted keyframe. + _keyframes.erase(std::upper_bound(_keyframes.begin(), _keyframes.end(), kf, compareTimestamps), _keyframes.end()); + + _keyframes.push_back(kf); + } + + void InputState::clearOldKeyframes() { + double now = OsEng.runTime(); + auto isLater = [now](const network::datamessagestructures::CameraKeyframe kf) { + return kf._timestamp > now; + }; + + // Remote keyframes with earlier timestamps than the current time. + auto nextKeyframe = std::find_if(_keyframes.begin(), _keyframes.end(), isLater); + if (nextKeyframe != _keyframes.begin()) { + _keyframes.erase(_keyframes.begin(), nextKeyframe - 1); + } } void InputState::clearKeyframes() { @@ -171,11 +192,38 @@ KeyframeInteractionMode::~KeyframeInteractionMode() { } void KeyframeInteractionMode::updateMouseStatesFromInput(const InputState& inputState, double deltaTime) { - + _keyframes = inputState.keyframes(); } void KeyframeInteractionMode::updateCameraStateFromMouseStates(Camera& camera) { + if (_keyframes.size() == 0) { + return; + } + double now = OsEng.runTime(); + auto isLater = [now](const network::datamessagestructures::CameraKeyframe kf) { + return kf._timestamp > now; + }; + + auto nextKeyframe = std::find_if(_keyframes.begin(), _keyframes.end(), isLater); + if (nextKeyframe == _keyframes.end()) { + return; + } + + if (nextKeyframe == _keyframes.begin()) { + camera.setPositionVec3(_keyframes[0]._position); + camera.setRotation(_keyframes[0]._rotation); + return; + } + auto prevKeyframe = nextKeyframe - 1; + + double prevTime = prevKeyframe->_timestamp; + double nextTime = nextKeyframe->_timestamp; + + double t = (now - prevTime) / (nextTime - prevTime); + + camera.setPositionVec3(prevKeyframe->_position * (1 - t) + nextKeyframe->_position * t); + camera.setRotation(glm::slerp(prevKeyframe->_rotation, nextKeyframe->_rotation, t)); } // OrbitalInteractionMode diff --git a/src/network/parallelconnection.cpp b/src/network/parallelconnection.cpp index e2c262cdef..9b6195f345 100644 --- a/src/network/parallelconnection.cpp +++ b/src/network/parallelconnection.cpp @@ -55,6 +55,7 @@ //openspace includes #include #include +#include #include #include #include @@ -65,6 +66,8 @@ namespace { const uint32_t ProtocolVersion = 2; + const size_t MaxLatencyDiffs = 64; + const double BroadcastIntervalMilliseconds = 100; const std::string _loggerCat = "ParallelConnection"; } @@ -488,7 +491,40 @@ void ParallelConnection::dataMessageReceived(const std::vector& messageCon switch(static_cast(type)) { case network::datamessagestructures::Type::CameraData: { + + std::lock_guard latencyLock(_latencyMutex); + network::datamessagestructures::CameraKeyframe kf(buffer); + + double timeDiff = OsEng.runTime() - kf._timestamp; + if (_latencyDiffs.size() == 0) { + _initialTimeDiff = timeDiff; + } + double latencyDiff = timeDiff - _initialTimeDiff; + if (_latencyDiffs.size() >= MaxLatencyDiffs) { + _latencyDiffs.pop_front(); + } + _latencyDiffs.push_back(latencyDiff); + + double accumulatedLatencyDiffSquared = 0; + double accumulatedLatencyDiff = 0; + for (double diff : _latencyDiffs) { + accumulatedLatencyDiff += diff; + accumulatedLatencyDiffSquared += diff*diff; + } + double expectedLatencyDiffSquared = accumulatedLatencyDiffSquared / _latencyDiffs.size(); + double expectedLatencyDiff = accumulatedLatencyDiff / _latencyDiffs.size(); + + // V(X) = E(x^2) - E(x)^2 + double latencyVariance = expectedLatencyDiffSquared - expectedLatencyDiff*expectedLatencyDiff; + double latencyStandardDeviation = std::sqrt(latencyVariance); + + double frametime = OsEng.windowWrapper().averageDeltaTime(); + + double latencyCompensation = std::max(expectedLatencyDiff + 2*latencyStandardDeviation, latencyDiff); + + kf._timestamp += _initialTimeDiff + BroadcastIntervalMilliseconds / 1000 + latencyCompensation + 2*frametime; + OsEng.interactionHandler().addKeyframe(kf); break; } @@ -1102,7 +1138,7 @@ void ParallelConnection::broadcast(){ queueOutDataMessage(DataMessage(network::datamessagestructures::Type::CameraData, buffer)); //100 ms sleep - send keyframes 10 times per second - std::this_thread::sleep_for(std::chrono::milliseconds(100)); + std::this_thread::sleep_for(std::chrono::milliseconds(static_cast(BroadcastIntervalMilliseconds))); } }