Compare commits

...

9 Commits

Author SHA1 Message Date
David Markowitz
7d62ec543a add back header 2023-07-27 20:06:12 -07:00
David Markowitz
3bab2af1a4 Merge branch 'main' into cdclient-rework 2023-07-27 20:03:12 -07:00
Wincent Holm
633b87a5e7 Merge branch 'main' into cdclient-rework 2022-10-29 21:52:08 +02:00
wincent
3031f942dc Removed log statement 2022-10-29 21:46:09 +02:00
wincent
e3e9718db5 Reverted local hardcoded NET_VERSION value. 2022-08-11 16:40:17 +02:00
wincent
56f371216b Abstracted the CDClient tables
There is now an option to utilize shared memory for some CDClient tables by adding `CD_PROVIDER_MEMORY=1` to the CMakeVariables.txt file.

Allows masterconfig.ini to specify another run command for the world server, to allow for easier debugging through `valgrind`.
2022-08-11 16:36:03 +02:00
Wincent Holm
45ae46f6f1 Merge branch 'main' into cdclient-rework 2022-07-18 16:44:53 +02:00
wincent
db7f7c60b2 Optimizations to shared memory
Load the context at start in readonly mode instead of fetching for every entry.
2022-07-10 00:40:13 +02:00
wincent
34fa43ed93 Initial concept for CDClient rework.
Utilizes `boost::interprocess` to share a map between worlds.
The master server loads the table into memory, and the worlds hook into it when they want to get an entry.

This solves the issue of skill delay, but introduces the third-party library Boost.
There should be a conversation about the introduction of Boost — it is a large library and adds extra steps to the installation.
2022-07-09 22:59:36 +02:00
20 changed files with 634 additions and 42 deletions

View File

@@ -322,6 +322,15 @@ link_directories(${PROJECT_BINARY_DIR})
# Load all of our third party directories
add_subdirectory(thirdparty)
# Include Boost
find_package(Boost COMPONENTS interprocess)
if(Boost_FOUND)
include_directories(${Boost_INCLUDE_DIRS})
target_link_libraries(dGame ${Boost_LIBRARIES})
endif()
# Glob together all headers that need to be precompiled
file(
GLOB HEADERS_DDATABASE

View File

@@ -0,0 +1,22 @@
#pragma once
#include "GeneralUtils.h"
#include "Game.h"
#include "dLogger.h"
#include "dServer.h"
#include "CDTable.h"
template <
typename KeyType,
typename MappedType
>
class CDAbstractProvider
{
public:
virtual void LoadClient() = 0;
virtual void LoadHost() = 0;
virtual const MappedType& GetEntry(const KeyType& key, const MappedType& defaultValue) = 0;
};

View File

@@ -0,0 +1,124 @@
#pragma once
#include "GeneralUtils.h"
#include "Game.h"
#include "dLogger.h"
#include "dServer.h"
#include "CDTable.h"
#include "CDAbstractProvider.h"
#include <boost/interprocess/managed_shared_memory.hpp>
#include <boost/interprocess/containers/map.hpp>
#include <boost/interprocess/allocators/allocator.hpp>
#include <boost/interprocess/containers/string.hpp>
#include <functional>
#include <utility>
#include <chrono>
template <
typename KeyType,
typename MappedType
>
class CDAbstractSharedMemoryMap : public CDAbstractProvider<KeyType, MappedType>
{
typedef std::pair<const KeyType, MappedType> ValueType;
typedef boost::interprocess::allocator<ValueType, boost::interprocess::managed_shared_memory::segment_manager> ShmemAllocator;
typedef boost::interprocess::map<KeyType, MappedType, std::less<KeyType>, ShmemAllocator> Map;
public:
std::map<KeyType, MappedType> m_CacheMap;
std::string m_Name;
size_t m_Size;
bool m_Host;
Map* m_HostEntries;
boost::interprocess::managed_shared_memory m_ClientSegment;
ShmemAllocator* m_ClientAllocInst;
boost::interprocess::offset_ptr<Map> m_ClientEntires;
CDAbstractSharedMemoryMap(std::string name, size_t size)
{
m_Name = name;
m_Size = size;
m_Host = false;
m_HostEntries = nullptr;
m_ClientAllocInst = nullptr;
m_ClientEntires = nullptr;
LoadClient();
}
const MappedType& GetEntry(const KeyType& key, const MappedType& defaultValue) override {
const auto& cacheIt = m_CacheMap.find(key);
if (cacheIt != m_CacheMap.end()) {
return cacheIt->second;
}
const auto& it = m_ClientEntires->find(key);
if (it == m_ClientEntires->end()) {
return defaultValue;
}
return it->second;
}
const void SetEntry(const KeyType& key, const MappedType& value) {
if (m_Host) {
// If we are already hosting, we cannot add to the map, throw an error
throw std::runtime_error("Can not add to a map that is already being hosted");
}
m_CacheMap.emplace(key, value);
}
void LoadClient() override {
try {
m_ClientSegment = boost::interprocess::managed_shared_memory(boost::interprocess::open_read_only, m_Name.c_str());
m_ClientAllocInst = new ShmemAllocator(m_ClientSegment.get_segment_manager());
m_ClientEntires = m_ClientSegment.find<Map>(m_Name.c_str()).first;
if (m_ClientEntires == nullptr) {
throw std::runtime_error("Could not find shared memory segment " + m_Name);
}
} catch (std::exception &e) {
// Not open
}
}
void LoadHost() override {
try {
boost::interprocess::shared_memory_object::remove(m_Name.c_str());
boost::interprocess::managed_shared_memory segment(boost::interprocess::create_only, m_Name.c_str(), m_Size);
ShmemAllocator alloc(segment.get_segment_manager());
m_HostEntries = segment.construct<Map>(m_Name.c_str()) (std::less<KeyType>(), alloc);
// Copy cache
for (const auto& pair : m_CacheMap) {
m_HostEntries->insert(std::make_pair(pair.first, pair.second));
}
m_Host = true;
LoadClient();
} catch (std::exception &e) {
// Make sure the smemory is removed
boost::interprocess::shared_memory_object::remove(m_Name.c_str());
throw e;
}
}
};

View File

@@ -0,0 +1,150 @@
#pragma once
#include "CDAbstractProvider.h"
#include <boost/interprocess/managed_shared_memory.hpp>
#include <boost/interprocess/containers/map.hpp>
#include <boost/interprocess/allocators/allocator.hpp>
#include <boost/interprocess/containers/string.hpp>
#include <functional>
#include <utility>
#include <chrono>
#include <mutex>
template <
typename KeyType,
typename MappedType
>
class CDAbstractSharedMemoryProvider : public CDAbstractProvider<KeyType, MappedType>
{
typedef std::pair<const KeyType, MappedType> ValueType;
typedef boost::interprocess::allocator<ValueType, boost::interprocess::managed_shared_memory::segment_manager> ShmemAllocator;
typedef boost::interprocess::map<KeyType, MappedType, std::less<KeyType>, ShmemAllocator> Map;
public:
std::string m_Name;
std::string m_MapName;
std::function <ValueType(CppSQLite3Query&)> m_ParseEntry;
std::function <int32_t(int32_t)> m_CalculateSize;
bool m_Cache;
std::unordered_map<KeyType, MappedType> m_CacheMap;
bool m_Host;
Map* m_HostEntries;
boost::interprocess::managed_shared_memory m_ClientSegment;
ShmemAllocator* m_ClientAllocInst;
boost::interprocess::offset_ptr<Map> m_ClientEntires;
CDAbstractSharedMemoryProvider(std::string name, std::function <ValueType(CppSQLite3Query&)> parseEntry, std::function <int32_t(int32_t)> calculateSize, bool cache)
{
m_Name = name;
m_MapName = name + "Map";
m_ParseEntry = parseEntry;
m_CalculateSize = calculateSize;
m_Cache = cache;
m_Host = false;
m_HostEntries = nullptr;
LoadClient();
}
const MappedType& GetEntry(const KeyType& key, const MappedType& defaultValue) override {
if (m_Host) {
auto it = m_HostEntries->find(key);
if (it == m_HostEntries->end())
{
return defaultValue;
}
return it->second;
}
if (m_Cache) {
auto it = m_CacheMap.find(key);
if (it != m_CacheMap.end()) {
return it->second;
}
}
const auto& it = m_ClientEntires->find(key);
if (it == m_ClientEntires->end())
{
if (m_Cache) {
m_CacheMap.emplace(key, defaultValue);
}
return defaultValue;
}
if (m_Cache) {
m_CacheMap.emplace(key, it->second);
}
return it->second;
}
void LoadClient() override {
try {
m_ClientSegment = boost::interprocess::managed_shared_memory(boost::interprocess::open_read_only, m_MapName.c_str());
m_ClientAllocInst = new ShmemAllocator(m_ClientSegment.get_segment_manager());
m_ClientEntires = m_ClientSegment.find<Map>(m_Name.c_str()).first;
if (m_ClientEntires == nullptr) {
throw std::runtime_error("Could not find shared memory segment " + m_Name);
}
} catch (boost::interprocess::interprocess_exception& e) {
// Not open
}
}
void LoadHost() override {
try {
boost::interprocess::shared_memory_object::remove(m_MapName.c_str());
auto sizeQuery = CDClientDatabase::ExecuteQuery("SELECT COUNT(*) FROM " + m_Name);
if (sizeQuery.eof()) {
throw std::runtime_error("Could not get size of table " + m_Name);
return;
}
int32_t size = sizeQuery.getIntField(0);
size = m_CalculateSize(size);
boost::interprocess::managed_shared_memory segment(boost::interprocess::create_only, m_MapName.c_str(), size);
ShmemAllocator alloc_inst (segment.get_segment_manager());
m_HostEntries = segment.construct<Map>(m_Name.c_str()) (std::less<KeyType>(), alloc_inst);
auto tableData = CDClientDatabase::ExecuteQuery("SELECT * FROM " + m_Name);
while (!tableData.eof()) {
ValueType entry = m_ParseEntry(tableData);
m_HostEntries->insert(entry);
tableData.nextRow();
}
tableData.finalize();
m_Host = true;
LoadClient();
} catch (std::exception &e) {
// Make sure the smemory is removed
boost::interprocess::shared_memory_object::remove(m_MapName.c_str());
throw e;
}
}
};

View File

@@ -0,0 +1,74 @@
#pragma once
#include "GeneralUtils.h"
#include "Game.h"
#include "dLogger.h"
#include "dServer.h"
#include "CDTable.h"
#include "CDAbstractProvider.h"
template <
typename KeyType,
typename MappedType
>
class CDAbstractSqliteProvider : public CDAbstractProvider<KeyType, MappedType>
{
typedef std::pair<const KeyType, MappedType> ValueType;
public:
std::string m_Name;
std::function <ValueType(CppSQLite3Query&)> m_ParseEntry;
std::unordered_map<KeyType, MappedType> m_Entries;
CDAbstractSqliteProvider(std::string name, std::function <ValueType(CppSQLite3Query&)> parseEntry, bool cache)
{
m_Name = name;
m_ParseEntry = parseEntry;
LoadClient();
}
void LoadClient() override {
// First, get the size of the table
uint32_t size = 0;
auto tableSize = CDClientDatabase::ExecuteQuery("SELECT COUNT(*) FROM " + m_Name);
while (!tableSize.eof()) {
size = tableSize.getIntField(0, 0);
tableSize.nextRow();
}
tableSize.finalize();
// Reserve the size
m_Entries.reserve(size);
// Now get the data
auto tableData = CDClientDatabase::ExecuteQuery("SELECT * FROM " + m_Name);
while (!tableData.eof()) {
auto entry = m_ParseEntry(tableData);
m_Entries.insert(entry);
tableData.nextRow();
}
tableData.finalize();
}
void LoadHost() override {
return;
}
const MappedType& GetEntry(const KeyType& key, const MappedType& defaultValue) override {
auto it = m_Entries.find(key);
if (it != m_Entries.end()) {
return it->second;
}
return defaultValue;
}
};

View File

@@ -78,3 +78,11 @@ CDClientManager::CDClientManager() {
CDFeatureGatingTable::Instance();
CDRailActivatorComponentTable::Instance();
}
void CDClientManager::LoadHost() {
for (auto itr = this->tables.begin(); itr != this->tables.end(); ++itr) {
itr->second->LoadHost();
}
CDTable::InitalizeHost();
}

41
dDatabase/CDProvider.h Normal file
View File

@@ -0,0 +1,41 @@
#pragma once
#include "CDAbstractProvider.h"
/**
* Shared memory provider with CDAbstractSharedMemoryProvider<key, value>
*
* Depends on the boost::interprocess library — header only
*
* Requires that CD_PROVIDER_MEMORY is defined and CD_PROVIDER_SQLITE is not defined
*/
#if defined(CD_PROVIDER_MEMORY) && !defined(CD_PROVIDER_SQLITE)
#include "CDAbstractSharedMemoryProvider.h"
#define CD_PROVIDER(provider, key, value) CDAbstractSharedMemoryProvider<key, value>* provider; typedef key CD_KEY; typedef value CD_VALUE
#define NEW_CD_PROVIDER(provider, name, parser, size_calculator, cache) provider = new CDAbstractSharedMemoryProvider<CD_KEY, CD_VALUE>(name, parser, size_calculator, cache)
template<typename KeyType, typename MappedType>
using CDProvider = CDAbstractSharedMemoryProvider<KeyType, MappedType>;
#endif
/**
* SQLite provider with CDAbstractSqliteProvider<key, value>
*
* No extra dependencies
*
* Requires that CD_PROVIDER_SQLITE or CD_PROVIDER_MEMORY is not defined — the default option
*/
#if defined(CD_PROVIDER_SQLITE) || !defined(CD_PROVIDER_MEMORY)
#include "CDAbstractSqliteProvider.h"
#define CD_PROVIDER(provider, key, value) CDAbstractSqliteProvider<key, value>* provider; typedef key CD_KEY; typedef value CD_VALUE
#define NEW_CD_PROVIDER(provider, name, parser, size_calculator, cache) provider = new CDAbstractSqliteProvider<CD_KEY, CD_VALUE>(name, parser, cache)
template<typename KeyType, typename MappedType>
using CDProvider = CDAbstractSqliteProvider<KeyType, MappedType>;
#endif

View File

@@ -21,10 +21,7 @@ CDBehaviorParameterTable::CDBehaviorParameterTable(void) {
entry.value = tableData.getFloatField("value", -1.0f);
m_Entries.insert(std::make_pair(hash, entry));
tableData.nextRow();
}
tableData.finalize();
}
float CDBehaviorParameterTable::GetValue(const uint32_t behaviorID, const std::string& name, const float defaultValue) {

View File

@@ -9,7 +9,7 @@ struct CDBehaviorTemplate {
unsigned int behaviorID; //!< The Behavior ID
unsigned int templateID; //!< The Template ID (LOT)
unsigned int effectID; //!< The Effect ID attached
std::unordered_set<std::string>::iterator effectHandle; //!< The effect handle
size_t effectHandle; //!< The effect handle
};

View File

@@ -1,3 +1,4 @@
#include "CDComponentsRegistryTable.h"
#include "eReplicaComponentType.h"

View File

@@ -0,0 +1,68 @@
#include "CDTable.h"
#if defined(CD_PROVIDER_MEMORY)
#include "CDAbstractSharedMemoryMap.h"
typedef boost::interprocess::allocator<char, boost::interprocess::managed_shared_memory::segment_manager> CharAllocator;
typedef boost::interprocess::basic_string<char, std::char_traits<char>, CharAllocator> my_string;
CDAbstractSharedMemoryMap<size_t, boost::interprocess::string>* CDStringMap;
void CDTable::InitalizeHost()
{
CDStringMap->LoadHost();
}
void CDTable::Initalize()
{
CDStringMap = new CDAbstractSharedMemoryMap<size_t, boost::interprocess::string>("CDStringMap", 10 * 1000 * 1000);
}
std::string CDTable::GetString(size_t handle)
{
std::string str = std::string(CDStringMap->GetEntry(handle, "").c_str());
return str;
}
size_t CDTable::SetString(std::string value)
{
size_t hash = 0;
GeneralUtils::hash_combine(hash, value);
CDStringMap->SetEntry(hash, boost::interprocess::string(value.c_str()));
return hash;
}
#else
std::unordered_map<size_t, std::string> CDStringMap;
void CDTable::InitalizeHost()
{
}
void CDTable::Initalize()
{
}
std::string CDTable::GetString(size_t handle)
{
return CDStringMap[handle];
}
size_t CDTable::SetString(std::string value)
{
size_t hash = 0;
GeneralUtils::hash_combine(hash, value);
CDStringMap[hash] = value;
return hash;
}
#endif

View File

@@ -25,8 +25,50 @@
#pragma warning (disable : 4244) //Disable double to float conversion warnings
#pragma warning (disable : 4715) //Disable "not all control paths return a value"
#if defined(__unix) || defined(__APPLE__)
//For Linux:
typedef __int64_t __int64;
#endif
/*!
\file CDTable.hpp
\brief A virtual class for CDClient Tables
*/
//! The base class for all CD tables
template<class Table>
class CDTable : public Singleton<Table> {
public:
//! Returns the table's name
/*!
\return The table name
*/
virtual std::string GetName() const = 0;
//! Loads the table into shared memory
virtual void LoadHost() {};
//! Initalizes the table services
static void Initalize();
//! Initalizes the table services as host
static void InitalizeHost();
//! Get a string from a handle
/*!
\param handle The handle to get the string from
\return The string
*/
static std::string GetString(size_t handle);
//! Set a string
/*!
\param value The string to set
\return The handle to the string
*/
static size_t SetString(std::string value);
protected:
virtual ~CDTable() = default;
};

View File

@@ -1,4 +1,4 @@
set(DDATABASE_TABLES_SOURCES "CDActivitiesTable.cpp"
set(DDATABASE_TABLES_SOURCES "CDTable.cpp" "CDActivitiesTable.cpp"
"CDActivityRewardsTable.cpp"
"CDAnimationsTable.cpp"
"CDBehaviorParameterTable.cpp"

View File

@@ -3,8 +3,12 @@
#include "Game.h"
#include "dLogger.h"
void AndBehavior::Handle(BehaviorContext* context, RakNet::BitStream* bitStream, const BehaviorBranchContext branch) {
for (auto* behavior : this->m_behaviors) {
#include <sstream>
void AndBehavior::Handle(BehaviorContext* context, RakNet::BitStream* bitStream, const BehaviorBranchContext branch)
{
for (auto* behavior : this->m_behaviors)
{
behavior->Handle(context, bitStream, branch);
}
}
@@ -21,14 +25,21 @@ void AndBehavior::UnCast(BehaviorContext* context, const BehaviorBranchContext b
}
}
void AndBehavior::Load() {
const auto parameters = GetParameterNames();
void AndBehavior::Load()
{
std::string ss = "behavior ";
for (const auto& parameter : parameters) {
if (parameter.first.rfind("behavior", 0) == 0) {
auto* action = GetAction(parameter.second);
int i = 1;
this->m_behaviors.push_back(action);
while (true) {
std::string s = ss + std::to_string(i);
if (GetInt(s, 0) == 0) {
break;
}
m_behaviors.push_back(GetAction(s));
++i;
}
}

View File

@@ -439,7 +439,7 @@ Behavior::Behavior(const uint32_t behaviorId) {
this->m_effectId = templateInDatabase.effectID;
this->m_effectHandle = *templateInDatabase.effectHandle != "" ? new std::string(*templateInDatabase.effectHandle) : nullptr;
this->m_effectHandle = new std::string(CDTable::GetString(templateInDatabase.effectHandle));
}
@@ -471,14 +471,8 @@ Behavior* Behavior::GetAction(float value) const {
}
std::map<std::string, float> Behavior::GetParameterNames() const {
std::map<std::string, float> templatesInDatabase;
// Find behavior template by its behavior id.
if (!BehaviorParameterTable) BehaviorParameterTable = CDClientManager::Instance().GetTable<CDBehaviorParameterTable>();
if (BehaviorParameterTable) {
templatesInDatabase = BehaviorParameterTable->GetParametersByBehaviorID(this->m_behaviorId);
}
return templatesInDatabase;
return BehaviorParameterTable->GetParametersByBehaviorID(this->m_behaviorId);
}
void Behavior::Load() {

View File

@@ -26,14 +26,21 @@ void ChainBehavior::Calculate(BehaviorContext* context, RakNet::BitStream* bitSt
this->m_behaviors.at(0)->Calculate(context, bitStream, branch);
}
void ChainBehavior::Load() {
const auto parameters = GetParameterNames();
void ChainBehavior::Load()
{
std::string ss = "behavior ";
for (const auto& parameter : parameters) {
if (parameter.first.rfind("behavior", 0) == 0) {
auto* action = GetAction(parameter.second);
int i = 1;
this->m_behaviors.push_back(action);
while (true) {
std::string s = ss + std::to_string(i);
if (GetInt(s, 0) == 0) {
break;
}
m_behaviors.push_back(GetAction(s));
++i;
}
}

View File

@@ -14,13 +14,19 @@ void NpcCombatSkillBehavior::Calculate(BehaviorContext* context, RakNet::BitStre
void NpcCombatSkillBehavior::Load() {
this->m_npcSkillTime = GetFloat("npc skill time");
const auto parameters = GetParameterNames();
std::string ss = "behavior ";
for (const auto& parameter : parameters) {
if (parameter.first.rfind("behavior", 0) == 0) {
auto* action = GetAction(parameter.second);
int i = 1;
this->m_behaviors.push_back(action);
while (true) {
std::string s = ss + std::to_string(i);
if (GetInt(s, 0) == 0) {
break;
}
m_behaviors.push_back(GetAction(s));
++i;
}
}

View File

@@ -1761,6 +1761,41 @@ void SlashCommandHandler::HandleChatCommand(const std::u16string& command, Entit
return;
}
#ifdef __linux__
if (chatCommand == "statm" && entity->GetGMLevel() >= eGameMasterLevel::DEVELOPER)
{
// Print and format the output of /proc/self/statm
std::ifstream statm("/proc/self/statm");
if (!statm.is_open())
{
ChatPackets::SendSystemMessage(sysAddr, u"Failed to open /proc/self/statm");
return;
}
// Scan for the different fields
uint64_t size, resident, share, text, lib, data, dirty;
statm >> size >> resident >> share >> text >> lib >> data >> dirty;
// Get the page size
size_t pageSize = sysconf(_SC_PAGESIZE);
// Print the output
ChatPackets::SendSystemMessage(
sysAddr,
u"Size: " + GeneralUtils::to_u16string((float) ((double) size * pageSize / 1.024e6)) +
u"MB\nResident: " + GeneralUtils::to_u16string((float) ((double) resident * pageSize / 1.024e6)) +
u"MB\nShared: " + GeneralUtils::to_u16string((float) ((double) share * pageSize / 1.024e6)) +
u"MB\nText: " + GeneralUtils::to_u16string((float) ((double) text * pageSize / 1.024e6)) +
u"MB\nLibrary: " + GeneralUtils::to_u16string((float) ((double) lib * pageSize / 1.024e6)) +
u"MB\nData: " + GeneralUtils::to_u16string((float) ((double) data * pageSize / 1.024e6)) +
u"MB\nDirty: " + GeneralUtils::to_u16string((float) ((double) dirty * pageSize / 1.024e6)) +
u"MB"
);
}
#endif
if (chatCommand == "rollloot" && entity->GetGMLevel() >= eGameMasterLevel::OPERATOR && args.size() >= 3) {
if (chatCommand == "reloadconfig" && entity->GetGMLevel() >= eGameMasterLevel::DEVELOPER) {
Game::config->ReloadConfig();
VanityUtilities::SpawnVanity();

View File

@@ -69,6 +69,7 @@ Instance* InstanceManager::GetInstance(LWOMAPID mapID, bool isFriendTransfer, LW
}
#endif
cmd.append(" -zone ");
cmd.append(std::to_string(mapID));
cmd.append(" -port ");
cmd.append(std::to_string(port));

View File

@@ -331,6 +331,8 @@ int main(int argc, char** argv) {
ObjectIDManager::Instance()->Initialize(Game::logger);
Game::im = new InstanceManager(Game::logger, Game::server->GetIP());
CDClientManager::Instance()->LoadHost();
//Depending on the config, start up servers:
if (Game::config->GetValue("prestart_servers") != "" && Game::config->GetValue("prestart_servers") == "1") {
StartChatServer();