sync camera movement over parallel connection

include/openspace/interaction/interactionhandler.h

@@ -53,6 +53,9 @@ public:
     InteractionHandler();
     ~InteractionHandler();
 
+    void initialize();
+    void deinitialize();
+
     // Mutators
     void setFocusNode(SceneGraphNode* node);
     void setCamera(Camera* camera);

include/openspace/interaction/interactionmode.h

@@ -58,13 +58,14 @@ namespace interaction {
         // Mutators
         void addKeyframe(const network::datamessagestructures::CameraKeyframe &kf);
         void clearKeyframes();
+        void clearOldKeyframes();
 
         // Accessors
         const std::list<std::pair<Key, KeyModifier> >& getPressedKeys() const;
         const std::list<MouseButton>& getPressedMouseButtons() const;
         glm::dvec2 getMousePosition() const;
         double getMouseScrollDelta() const;
-        std::vector<network::datamessagestructures::CameraKeyframe>& getKeyFrames() const;
+        const std::vector<network::datamessagestructures::CameraKeyframe>& keyframes() const;
 
         bool isKeyPressed(std::pair<Key, KeyModifier> keyModPair) const;
         bool isMouseButtonPressed(MouseButton mouseButton) const;
@@ -174,6 +175,7 @@ public:
     virtual void serialize(SyncBuffer* syncBuffer) {};
     virtual void deserialize(SyncBuffer* syncBuffer) {};
 private:
+    std::vector<network::datamessagestructures::CameraKeyframe> _keyframes;
     double _currentKeyframeTime;
 };
 

include/openspace/network/parallelconnection.h

@@ -244,6 +244,10 @@ namespace openspace {
             std::map<std::string, std::string> _currentState;
             std::mutex _currentStateMutex;
 
+            std::mutex _latencyMutex;
+            std::deque<double> _latencyDiffs;
+            double _initialTimeDiff;
+
             std::shared_ptr<ghoul::Event<>> _connectionEvent;
         };
     } // namespace network

src/engine/openspaceengine.cpp

@@ -164,6 +164,7 @@ OpenSpaceEngine::~OpenSpaceEngine() {
 #ifdef OPENSPACE_MODULE_ONSCREENGUI_ENABLED
     _gui->deinitializeGL();
 #endif
+    _interactionHandler->deinitialize();
     _renderEngine->deinitialize();
 
     _globalPropertyNamespace = nullptr;
@@ -425,6 +426,9 @@ bool OpenSpaceEngine::initialize() {
     _settingsEngine->initialize();
     _settingsEngine->setModules(_moduleEngine->modules());
 
+    // Initialize the InteractionHandler
+    _interactionHandler->initialize();
+
     // Initialize the Scene
     Scene* sceneGraph = new Scene;
     sceneGraph->initialize();

src/interaction/interactionhandler.cpp

@@ -137,6 +137,25 @@ InteractionHandler::~InteractionHandler() {
 
 }
 
+void InteractionHandler::initialize() {
+    OsEng.parallelConnection().connectionEvent()->subscribe("interactionHandler", "statusChanged", [this]() {
+        if (OsEng.parallelConnection().status() == network::Status::ClientWithHost) {
+            setInteractionMode("Keyframe");
+        } else {
+            auto keyframeModeIter = _interactionModes.find("Keyframe");
+            if (keyframeModeIter != _interactionModes.end()) {
+                if (_currentInteractionMode == keyframeModeIter->second) {
+                    setInteractionMode("Orbital");
+                }
+            }
+        }
+    });
+}
+
+void InteractionHandler::deinitialize() {
+    OsEng.parallelConnection().connectionEvent()->unsubscribe("interactionHandler");
+}
+
 void InteractionHandler::setFocusNode(SceneGraphNode* node) {
     _currentInteractionMode->setFocusNode(node);
 }

src/interaction/interactionmode.cpp

@@ -57,14 +57,35 @@ namespace interaction {
 
     }
 
-    void InputState::addKeyframe(const network::datamessagestructures::CameraKeyframe &kf) {
-        
-        //save a maximum of 10 samples (1 seconds of buffer)
-        if (_keyframes.size() >= 10) {
-            _keyframes.erase(_keyframes.begin());
-        }
-        _keyframes.push_back(kf);
+    const std::vector<network::datamessagestructures::CameraKeyframe>& InputState::keyframes() const {
+        return _keyframes;
+    }
 
+    void InputState::addKeyframe(const network::datamessagestructures::CameraKeyframe &kf) {
+        clearOldKeyframes();
+
+        auto compareTimestamps = [](const network::datamessagestructures::CameraKeyframe a,
+            network::datamessagestructures::CameraKeyframe b) {
+            return a._timestamp < b._timestamp;
+        };
+
+        // Remove keyframes after the inserted keyframe.
+        _keyframes.erase(std::upper_bound(_keyframes.begin(), _keyframes.end(), kf, compareTimestamps), _keyframes.end());
+
+        _keyframes.push_back(kf);
+    }
+
+    void InputState::clearOldKeyframes() {
+        double now = OsEng.runTime();
+        auto isLater = [now](const network::datamessagestructures::CameraKeyframe kf) {
+            return kf._timestamp > now;
+        };
+
+        // Remote keyframes with earlier timestamps than the current time.
+        auto nextKeyframe = std::find_if(_keyframes.begin(), _keyframes.end(), isLater);
+        if (nextKeyframe != _keyframes.begin()) {
+            _keyframes.erase(_keyframes.begin(), nextKeyframe - 1);
+        }
     }
 
     void InputState::clearKeyframes() {
@@ -171,11 +192,38 @@ KeyframeInteractionMode::~KeyframeInteractionMode() {
 }
 
 void KeyframeInteractionMode::updateMouseStatesFromInput(const InputState& inputState, double deltaTime) {
-
+    _keyframes = inputState.keyframes();
 }
 
 void KeyframeInteractionMode::updateCameraStateFromMouseStates(Camera& camera) {
+    if (_keyframes.size() == 0) {
+        return;
+    }
 
+    double now = OsEng.runTime();
+    auto isLater = [now](const network::datamessagestructures::CameraKeyframe kf) {
+        return kf._timestamp > now;
+    };
+
+    auto nextKeyframe = std::find_if(_keyframes.begin(), _keyframes.end(), isLater);
+    if (nextKeyframe == _keyframes.end()) {
+        return;
+    }
+
+    if (nextKeyframe == _keyframes.begin()) {
+        camera.setPositionVec3(_keyframes[0]._position);
+        camera.setRotation(_keyframes[0]._rotation);
+        return;
+    }
+    auto prevKeyframe = nextKeyframe - 1;
+
+    double prevTime = prevKeyframe->_timestamp;
+    double nextTime = nextKeyframe->_timestamp;
+
+    double t = (now - prevTime) / (nextTime - prevTime);
+
+    camera.setPositionVec3(prevKeyframe->_position * (1 - t) + nextKeyframe->_position * t);
+    camera.setRotation(glm::slerp(prevKeyframe->_rotation, nextKeyframe->_rotation, t));
 }
 
 // OrbitalInteractionMode

src/network/parallelconnection.cpp

@@ -55,6 +55,7 @@
 //openspace includes
 #include <openspace/network/parallelconnection.h>
 #include <openspace/engine/openspaceengine.h>
+#include <openspace/engine/wrapper/windowwrapper.h>
 #include <openspace/interaction/interactionhandler.h>
 #include <openspace/util/time.h>
 #include <openspace/openspace.h>
@@ -65,6 +66,8 @@
 
 namespace {
     const uint32_t ProtocolVersion = 2;
+    const size_t MaxLatencyDiffs = 64;
+    const double BroadcastIntervalMilliseconds = 100;
     const std::string _loggerCat = "ParallelConnection";
 }
 
@@ -488,7 +491,40 @@ void ParallelConnection::dataMessageReceived(const std::vector<char>& messageCon
  
     switch(static_cast<network::datamessagestructures::Type>(type)) {
         case network::datamessagestructures::Type::CameraData: {
+
+            std::lock_guard<std::mutex> latencyLock(_latencyMutex);
+
             network::datamessagestructures::CameraKeyframe kf(buffer);
+
+            double timeDiff = OsEng.runTime() - kf._timestamp;
+            if (_latencyDiffs.size() == 0) {
+                _initialTimeDiff = timeDiff;
+            }
+            double latencyDiff = timeDiff - _initialTimeDiff;
+            if (_latencyDiffs.size() >= MaxLatencyDiffs) {
+                _latencyDiffs.pop_front();
+            }
+            _latencyDiffs.push_back(latencyDiff);
+
+            double accumulatedLatencyDiffSquared = 0;
+            double accumulatedLatencyDiff = 0;
+            for (double diff : _latencyDiffs) {
+                accumulatedLatencyDiff += diff;
+                accumulatedLatencyDiffSquared += diff*diff;
+            }
+            double expectedLatencyDiffSquared = accumulatedLatencyDiffSquared / _latencyDiffs.size();
+            double expectedLatencyDiff = accumulatedLatencyDiff / _latencyDiffs.size();
+
+            // V(X) = E(x^2) - E(x)^2
+            double latencyVariance = expectedLatencyDiffSquared - expectedLatencyDiff*expectedLatencyDiff;
+            double latencyStandardDeviation = std::sqrt(latencyVariance);
+
+            double frametime = OsEng.windowWrapper().averageDeltaTime();
+
+            double latencyCompensation = std::max(expectedLatencyDiff + 2*latencyStandardDeviation, latencyDiff);
+
+            kf._timestamp += _initialTimeDiff + BroadcastIntervalMilliseconds / 1000 + latencyCompensation + 2*frametime;
+
             OsEng.interactionHandler().addKeyframe(kf);
             break;
         }
@@ -1102,7 +1138,7 @@ void ParallelConnection::broadcast(){
         queueOutDataMessage(DataMessage(network::datamessagestructures::Type::CameraData, buffer));
 
         //100 ms sleep - send keyframes 10 times per second
-        std::this_thread::sleep_for(std::chrono::milliseconds(100));
+        std::this_thread::sleep_for(std::chrono::milliseconds(static_cast<int>(BroadcastIntervalMilliseconds)));
     }
 }