Merge topic 'server-refactor'

39c2feaf misc: Added utility method to allow working with stacks
f5d2988e server: Swapped to cm_thread impl
2636d86c utility: Added minimal std::thread drop-in
d46b4ba8 server: Updated server tests to try various communication channels
08dca583 Tests: reworked server tests to allow other operation modes

Acked-by: Kitware Robot <kwrobot@kitware.com>
Merge-request: !1230
This commit is contained in:
Brad King
2017-11-06 13:56:33 +00:00
committed by Kitware Robot
9 changed files with 336 additions and 142 deletions

View File

@@ -593,6 +593,7 @@ set(SRCS
cm_utf8.c
cm_codecvt.hxx
cm_codecvt.cxx
cm_thread.hxx
)
SET_PROPERTY(SOURCE cmProcessOutput.cxx APPEND PROPERTY COMPILE_DEFINITIONS

View File

@@ -438,6 +438,19 @@ void cmListFileBacktrace::PrintCallStack(std::ostream& out) const
}
}
size_t cmListFileBacktrace::Depth() const
{
size_t depth = 0;
if (this->Cur == nullptr) {
return 0;
}
for (Entry* i = this->Cur->Up; i; i = i->Up) {
depth++;
}
return depth;
}
std::ostream& operator<<(std::ostream& os, cmListFileContext const& lfc)
{
os << lfc.FilePath;

View File

@@ -6,6 +6,7 @@
#include "cmConfigure.h" // IWYU pragma: keep
#include <iosfwd>
#include <stddef.h>
#include <string>
#include <vector>
@@ -138,6 +139,9 @@ public:
// Print the call stack below the top of the backtrace.
void PrintCallStack(std::ostream& out) const;
// Get the number of 'frames' in this backtrace
size_t Depth() const;
private:
struct Entry;

View File

@@ -245,11 +245,10 @@ cmFileMonitor* cmServer::FileMonitor() const
void cmServer::WriteJsonObject(const Json::Value& jsonValue,
const DebugInfo* debug) const
{
uv_rwlock_rdlock(&ConnectionsMutex);
cm::shared_lock<cm::shared_mutex> lock(ConnectionsMutex);
for (auto& connection : this->Connections) {
WriteJsonObject(connection.get(), jsonValue, debug);
}
uv_rwlock_rdunlock(&ConnectionsMutex);
}
void cmServer::WriteJsonObject(cmConnection* connection,
@@ -456,14 +455,12 @@ bool cmServerBase::Serve(std::string* errorMessage)
OnServeStart();
{
uv_rwlock_rdlock(&ConnectionsMutex);
cm::shared_lock<cm::shared_mutex> lock(ConnectionsMutex);
for (auto& connection : Connections) {
if (!connection->OnServeStart(errorMessage)) {
uv_rwlock_rdunlock(&ConnectionsMutex);
return false;
}
}
uv_rwlock_rdunlock(&ConnectionsMutex);
}
if (uv_run(&Loop, UV_RUN_DEFAULT) != 0) {
@@ -501,12 +498,11 @@ void cmServerBase::StartShutDown()
}
{
uv_rwlock_wrlock(&ConnectionsMutex);
cm::unique_lock<cm::shared_mutex> lock(ConnectionsMutex);
for (auto& connection : Connections) {
connection->OnConnectionShuttingDown();
}
Connections.clear();
uv_rwlock_wrunlock(&ConnectionsMutex);
}
uv_walk(&Loop, on_walk_to_shutdown, nullptr);
@@ -525,9 +521,6 @@ cmServerBase::cmServerBase(cmConnection* connection)
(void)err;
assert(err == 0);
err = uv_rwlock_init(&ConnectionsMutex);
assert(err == 0);
AddNewConnection(connection);
}
@@ -540,14 +533,14 @@ cmServerBase::~cmServerBase()
}
uv_loop_close(&Loop);
uv_rwlock_destroy(&ConnectionsMutex);
}
void cmServerBase::AddNewConnection(cmConnection* ownedConnection)
{
uv_rwlock_wrlock(&ConnectionsMutex);
Connections.emplace_back(ownedConnection);
uv_rwlock_wrunlock(&ConnectionsMutex);
{
cm::unique_lock<cm::shared_mutex> lock(ConnectionsMutex);
Connections.emplace_back(ownedConnection);
}
ownedConnection->SetServer(this);
}
@@ -561,11 +554,13 @@ void cmServerBase::OnDisconnect(cmConnection* pConnection)
auto pred = [pConnection](const std::unique_ptr<cmConnection>& m) {
return m.get() == pConnection;
};
uv_rwlock_wrlock(&ConnectionsMutex);
Connections.erase(
std::remove_if(Connections.begin(), Connections.end(), pred),
Connections.end());
uv_rwlock_wrunlock(&ConnectionsMutex);
{
cm::unique_lock<cm::shared_mutex> lock(ConnectionsMutex);
Connections.erase(
std::remove_if(Connections.begin(), Connections.end(), pred),
Connections.end());
}
if (Connections.empty()) {
StartShutDown();
}

View File

@@ -5,6 +5,7 @@
#include "cmConfigure.h" // IWYU pragma: keep
#include "cm_jsoncpp_value.h"
#include "cm_thread.hxx"
#include "cm_uv.h"
#include <memory> // IWYU pragma: keep
@@ -61,7 +62,7 @@ public:
void OnDisconnect(cmConnection* pConnection);
protected:
mutable uv_rwlock_t ConnectionsMutex;
mutable cm::shared_mutex ConnectionsMutex;
std::vector<std::unique_ptr<cmConnection>> Connections;
bool ServeThreadRunning = false;

78
Source/cm_thread.hxx Normal file
View File

@@ -0,0 +1,78 @@
/* Distributed under the OSI-approved BSD 3-Clause License. See accompanying
file Copyright.txt or https://cmake.org/licensing for details. */
#ifndef CM_THREAD_HXX
#define CM_THREAD_HXX
#include "cmConfigure.h" // IWYU pragma: keep
#include "cm_uv.h"
namespace cm {
class mutex
{
uv_mutex_t _M_;
public:
mutex() { uv_mutex_init(&_M_); }
~mutex() { uv_mutex_destroy(&_M_); }
void lock() { uv_mutex_lock(&_M_); }
void unlock() { uv_mutex_unlock(&_M_); }
};
template <typename T>
class lock_guard
{
T& _mutex;
public:
lock_guard(T& m)
: _mutex(m)
{
_mutex.lock();
}
~lock_guard() { _mutex.unlock(); }
};
class shared_mutex
{
uv_rwlock_t _M_;
public:
shared_mutex() { uv_rwlock_init(&_M_); }
~shared_mutex() { uv_rwlock_destroy(&_M_); }
void lock() { uv_rwlock_wrlock(&_M_); }
void unlock() { uv_rwlock_wrunlock(&_M_); }
void lock_shared() { uv_rwlock_rdlock(&_M_); }
void unlock_shared() { uv_rwlock_rdunlock(&_M_); }
};
template <typename T>
class shared_lock
{
T& _mutex;
public:
shared_lock(T& m)
: _mutex(m)
{
_mutex.lock_shared();
}
~shared_lock() { _mutex.unlock_shared(); }
};
template <typename T>
class unique_lock : public lock_guard<T>
{
public:
unique_lock(T& m)
: lock_guard<T>(m)
{
}
};
}
#endif

View File

@@ -3,10 +3,10 @@ project(Server CXX)
find_package(PythonInterp REQUIRED)
macro(do_test bsname file)
macro(do_test bsname file type)
execute_process(COMMAND ${PYTHON_EXECUTABLE}
-B # no .pyc files
"${CMAKE_SOURCE_DIR}/server-test.py"
"${CMAKE_SOURCE_DIR}/${type}-test.py"
"${CMAKE_COMMAND}"
"${CMAKE_SOURCE_DIR}/${file}"
"${CMAKE_SOURCE_DIR}"
@@ -20,9 +20,9 @@ macro(do_test bsname file)
endif()
endmacro()
do_test("test_cache" "tc_cache.json")
do_test("test_handshake" "tc_handshake.json")
do_test("test_globalSettings" "tc_globalSettings.json")
do_test("test_buildsystem1" "tc_buildsystem1.json")
do_test("test_cache" "tc_cache.json" "server")
do_test("test_handshake" "tc_handshake.json" "server")
do_test("test_globalSettings" "tc_globalSettings.json" "server")
do_test("test_buildsystem1" "tc_buildsystem1.json" "server")
add_executable(Server empty.cpp)

View File

@@ -1,5 +1,5 @@
from __future__ import print_function
import sys, subprocess, json
import sys, subprocess, json, os, select, shutil, time, socket
termwidth = 150
@@ -38,11 +38,50 @@ def col_print(title, array):
for index in range(numRows):
print(indent + pad.join(item.ljust(maxitemwidth) for item in array[index::numRows]))
filterPacket = lambda x: x
STDIN = 0
PIPE = 1
communicationMethods = [STDIN]
if hasattr(socket, 'AF_UNIX'):
communicationMethods.append(PIPE)
def defaultExitWithError(proc):
data = ""
try:
while select.select([proc.outPipe], [], [], 3.)[0]:
data = data + proc.outPipe.read(1)
if len(data):
print("Rest of raw buffer from server:")
printServer(data)
except:
pass
proc.outPipe.close()
proc.inPipe.close()
proc.kill()
sys.exit(1)
exitWithError = lambda proc: defaultExitWithError(proc)
serverTag = "SERVER"
def printServer(*args):
print(serverTag + ">", *args)
print()
sys.stdout.flush()
def printClient(*args):
print("CLIENT>", *args)
print()
sys.stdout.flush()
def waitForRawMessage(cmakeCommand):
stdoutdata = ""
payload = ""
while not cmakeCommand.poll():
stdoutdataLine = cmakeCommand.stdout.readline()
stdoutdataLine = cmakeCommand.outPipe.readline()
if stdoutdataLine:
stdoutdata += stdoutdataLine.decode('utf-8')
else:
@@ -50,12 +89,16 @@ def waitForRawMessage(cmakeCommand):
begin = stdoutdata.find('[== "CMake Server" ==[\n')
end = stdoutdata.find(']== "CMake Server" ==]')
if (begin != -1 and end != -1):
if begin != -1 and end != -1:
begin += len('[== "CMake Server" ==[\n')
payload = stdoutdata[begin:end]
if print_communication:
print("\nSERVER>", json.loads(payload), "\n")
return json.loads(payload)
jsonPayload = json.loads(payload)
filteredPayload = filterPacket(jsonPayload)
if print_communication and filteredPayload:
printServer(filteredPayload)
if filteredPayload is not None or jsonPayload is None:
return jsonPayload
stdoutdata = stdoutdata[(end+len(']== "CMake Server" ==]')):]
def writeRawData(cmakeCommand, content):
writeRawData.counter += 1
@@ -71,27 +114,53 @@ def writeRawData(cmakeCommand, content):
payload = payload.replace('\n', '\r\n')
if print_communication:
print("\nCLIENT>", content, "(Use \\r\\n:", rn, ")\n")
cmakeCommand.stdin.write(payload.encode('utf-8'))
cmakeCommand.stdin.flush()
printClient(content, "(Use \\r\\n:", rn, ")")
cmakeCommand.write(payload.encode('utf-8'))
writeRawData.counter = 0
def writePayload(cmakeCommand, obj):
writeRawData(cmakeCommand, json.dumps(obj))
def initProc(cmakeCommand):
cmakeCommand = subprocess.Popen([cmakeCommand, "-E", "server", "--experimental", "--debug"],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
def getPipeName():
return "/tmp/server-test-socket"
def attachPipe(cmakeCommand, pipeName):
time.sleep(1)
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.connect(pipeName)
global serverTag
serverTag = "SERVER(PIPE)"
cmakeCommand.outPipe = sock.makefile()
cmakeCommand.inPipe = sock
cmakeCommand.write = cmakeCommand.inPipe.sendall
def writeAndFlush(pipe, val):
pipe.write(val)
pipe.flush()
def initServerProc(cmakeCommand, comm):
if comm == PIPE:
pipeName = getPipeName()
cmakeCommand = subprocess.Popen([cmakeCommand, "-E", "server", "--experimental", "--pipe=" + pipeName])
attachPipe(cmakeCommand, pipeName)
else:
cmakeCommand = subprocess.Popen([cmakeCommand, "-E", "server", "--experimental", "--debug"],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
cmakeCommand.outPipe = cmakeCommand.stdout
cmakeCommand.inPipe = cmakeCommand.stdin
cmakeCommand.write = lambda val: writeAndFlush(cmakeCommand.inPipe, val)
packet = waitForRawMessage(cmakeCommand)
if packet == None:
print("Not in server mode")
sys.exit(1)
sys.exit(2)
if packet['type'] != 'hello':
print("No hello message")
sys.exit(1)
sys.exit(3)
return cmakeCommand
@@ -115,7 +184,8 @@ def waitForMessage(cmakeCommand, expected):
packet = ordered(waitForRawMessage(cmakeCommand))
if packet != data:
sys.exit(-1)
print ("Received unexpected message; test failed")
exitWithError(cmakeCommand)
return packet
def waitForReply(cmakeCommand, originalType, cookie, skipProgress):
@@ -124,25 +194,27 @@ def waitForReply(cmakeCommand, originalType, cookie, skipProgress):
packet = waitForRawMessage(cmakeCommand)
t = packet['type']
if packet['cookie'] != cookie or packet['inReplyTo'] != originalType:
sys.exit(1)
print("cookie or inReplyTo mismatch")
sys.exit(4)
if t == 'message' or t == 'progress':
if skipProgress:
continue
if t == 'reply':
break
sys.exit(1)
print("Unrecognized message", packet)
sys.exit(5)
return packet
def waitForError(cmakeCommand, originalType, cookie, message):
packet = waitForRawMessage(cmakeCommand)
if packet['cookie'] != cookie or packet['type'] != 'error' or packet['inReplyTo'] != originalType or packet['errorMessage'] != message:
sys.exit(1)
sys.exit(6)
def waitForProgress(cmakeCommand, originalType, cookie, current, message):
packet = waitForRawMessage(cmakeCommand)
if packet['cookie'] != cookie or packet['type'] != 'progress' or packet['inReplyTo'] != originalType or packet['progressCurrent'] != current or packet['progressMessage'] != message:
sys.exit(1)
sys.exit(7)
def handshake(cmakeCommand, major, minor, source, build, generator, extraGenerator):
version = { 'major': major }
@@ -167,9 +239,9 @@ def validateGlobalSettings(cmakeCommand, cmakeCommandPath, data):
versionString = version['string']
vs = str(version['major']) + '.' + str(version['minor']) + '.' + str(version['patch'])
if (versionString != vs and not versionString.startswith(vs + '-')):
sys.exit(1)
sys.exit(8)
if (versionString != cmakeVersion):
sys.exit(1)
sys.exit(9)
# validate generators:
generatorObjects = capabilities['generators']
@@ -202,16 +274,16 @@ def validateGlobalSettings(cmakeCommand, cmakeCommandPath, data):
for gen in cmakeGenerators:
if (not gen in generators):
sys.exit(1)
sys.exit(10)
gen = packet['generator']
if (gen != '' and not (gen in generators)):
sys.exit(1)
sys.exit(11)
for i in data:
print("Validating", i)
if (packet[i] != data[i]):
sys.exit(1)
sys.exit(12)
def validateCache(cmakeCommand, data):
packet = waitForReply(cmakeCommand, 'cache', '', False)
@@ -236,3 +308,43 @@ def validateCache(cmakeCommand, data):
if (not hadHomeDir):
print('No CMAKE_HOME_DIRECTORY found in cache.')
sys.exit(1)
def handleBasicMessage(proc, obj, debug):
if 'sendRaw' in obj:
data = obj['sendRaw']
if debug: print("Sending raw:", data)
writeRawData(proc, data)
return True
elif 'send' in obj:
data = obj['send']
if debug: print("Sending:", json.dumps(data))
writePayload(proc, data)
return True
elif 'recv' in obj:
data = obj['recv']
if debug: print("Waiting for:", json.dumps(data))
waitForMessage(proc, data)
return True
elif 'message' in obj:
print("MESSAGE:", obj["message"])
sys.stdout.flush()
return True
return False
def shutdownProc(proc):
# Tell the server to exit.
proc.inPipe.close()
proc.outPipe.close()
# Wait for the server to exit.
# If this version of python supports it, terminate the server after a timeout.
try:
proc.wait(timeout=5)
except TypeError:
proc.wait()
except:
proc.terminate()
raise
print('cmake-server exited: %d' % proc.returncode)
sys.exit(proc.returncode)

View File

@@ -9,7 +9,7 @@ sourceDir = sys.argv[3]
buildDir = sys.argv[4] + "/" + os.path.splitext(os.path.basename(testFile))[0]
cmakeGenerator = sys.argv[5]
print("Test:", testFile,
print("Server Test:", testFile,
"\n-- SourceDir:", sourceDir,
"\n-- BuildDir:", buildDir,
"\n-- Generator:", cmakeGenerator)
@@ -17,99 +17,89 @@ print("Test:", testFile,
if os.path.exists(buildDir):
shutil.rmtree(buildDir)
proc = cmakelib.initProc(cmakeCommand)
cmakelib.filterBase = sourceDir
with open(testFile) as f:
testData = json.loads(f.read())
for obj in testData:
if 'sendRaw' in obj:
data = obj['sendRaw']
if debug: print("Sending raw:", data)
cmakelib.writeRawData(proc, data)
elif 'send' in obj:
data = obj['send']
if debug: print("Sending:", json.dumps(data))
cmakelib.writePayload(proc, data)
elif 'recv' in obj:
data = obj['recv']
if debug: print("Waiting for:", json.dumps(data))
cmakelib.waitForMessage(proc, data)
elif 'reply' in obj:
data = obj['reply']
if debug: print("Waiting for reply:", json.dumps(data))
originalType = ""
cookie = ""
skipProgress = False;
if 'cookie' in data: cookie = data['cookie']
if 'type' in data: originalType = data['type']
if 'skipProgress' in data: skipProgress = data['skipProgress']
cmakelib.waitForReply(proc, originalType, cookie, skipProgress)
elif 'error' in obj:
data = obj['error']
if debug: print("Waiting for error:", json.dumps(data))
originalType = ""
cookie = ""
message = ""
if 'cookie' in data: cookie = data['cookie']
if 'type' in data: originalType = data['type']
if 'message' in data: message = data['message']
cmakelib.waitForError(proc, originalType, cookie, message)
elif 'progress' in obj:
data = obj['progress']
if debug: print("Waiting for progress:", json.dumps(data))
originalType = ''
cookie = ""
current = 0
message = ""
if 'cookie' in data: cookie = data['cookie']
if 'type' in data: originalType = data['type']
if 'current' in data: current = data['current']
if 'message' in data: message = data['message']
cmakelib.waitForProgress(proc, originalType, cookie, current, message)
elif 'handshake' in obj:
data = obj['handshake']
if debug: print("Doing handshake:", json.dumps(data))
major = -1
minor = -1
generator = cmakeGenerator
extraGenerator = ''
sourceDirectory = sourceDir
buildDirectory = buildDir
if 'major' in data: major = data['major']
if 'minor' in data: minor = data['minor']
if 'buildDirectory' in data: buildDirectory = data['buildDirectory']
if 'sourceDirectory' in data: sourceDirectory = data['sourceDirectory']
if 'generator' in data: generator = data['generator']
if 'extraGenerator' in data: extraGenerator = data['extraGenerator']
if not os.path.isabs(buildDirectory):
buildDirectory = buildDir + "/" + buildDirectory
if sourceDirectory != '' and not os.path.isabs(sourceDirectory):
sourceDirectory = sourceDir + "/" + sourceDirectory
cmakelib.handshake(proc, major, minor, sourceDirectory, buildDirectory,
generator, extraGenerator)
elif 'validateGlobalSettings' in obj:
data = obj['validateGlobalSettings']
if not 'buildDirectory' in data: data['buildDirectory'] = buildDir
if not 'sourceDirectory' in data: data['sourceDirectory'] = sourceDir
if not 'generator' in data: data['generator'] = cmakeGenerator
if not 'extraGenerator' in data: data['extraGenerator'] = ''
cmakelib.validateGlobalSettings(proc, cmakeCommand, data)
elif 'validateCache' in obj:
data = obj['validateCache']
if not 'isEmpty' in data: data['isEmpty'] = false
cmakelib.validateCache(proc, data)
elif 'message' in obj:
print("MESSAGE:", obj["message"])
elif 'reconnect' in obj:
cmakelib.exitProc(proc)
proc = cmakelib.initProc(cmakeCommand)
else:
print("Unknown command:", json.dumps(obj))
sys.exit(2)
for communicationMethod in cmakelib.communicationMethods:
proc = cmakelib.initServerProc(cmakeCommand, communicationMethod)
if proc is None:
continue
for obj in testData:
if cmakelib.handleBasicMessage(proc, obj, debug):
pass
elif 'reply' in obj:
data = obj['reply']
if debug: print("Waiting for reply:", json.dumps(data))
originalType = ""
cookie = ""
skipProgress = False;
if 'cookie' in data: cookie = data['cookie']
if 'type' in data: originalType = data['type']
if 'skipProgress' in data: skipProgress = data['skipProgress']
cmakelib.waitForReply(proc, originalType, cookie, skipProgress)
elif 'error' in obj:
data = obj['error']
if debug: print("Waiting for error:", json.dumps(data))
originalType = ""
cookie = ""
message = ""
if 'cookie' in data: cookie = data['cookie']
if 'type' in data: originalType = data['type']
if 'message' in data: message = data['message']
cmakelib.waitForError(proc, originalType, cookie, message)
elif 'progress' in obj:
data = obj['progress']
if debug: print("Waiting for progress:", json.dumps(data))
originalType = ''
cookie = ""
current = 0
message = ""
if 'cookie' in data: cookie = data['cookie']
if 'type' in data: originalType = data['type']
if 'current' in data: current = data['current']
if 'message' in data: message = data['message']
cmakelib.waitForProgress(proc, originalType, cookie, current, message)
elif 'handshake' in obj:
data = obj['handshake']
if debug: print("Doing handshake:", json.dumps(data))
major = -1
minor = -1
generator = cmakeGenerator
extraGenerator = ''
sourceDirectory = sourceDir
buildDirectory = buildDir
if 'major' in data: major = data['major']
if 'minor' in data: minor = data['minor']
if 'buildDirectory' in data: buildDirectory = data['buildDirectory']
if 'sourceDirectory' in data: sourceDirectory = data['sourceDirectory']
if 'generator' in data: generator = data['generator']
if 'extraGenerator' in data: extraGenerator = data['extraGenerator']
if not os.path.isabs(buildDirectory):
buildDirectory = buildDir + "/" + buildDirectory
if sourceDirectory != '' and not os.path.isabs(sourceDirectory):
sourceDirectory = sourceDir + "/" + sourceDirectory
cmakelib.handshake(proc, major, minor, sourceDirectory, buildDirectory,
generator, extraGenerator)
elif 'validateGlobalSettings' in obj:
data = obj['validateGlobalSettings']
if not 'buildDirectory' in data: data['buildDirectory'] = buildDir
if not 'sourceDirectory' in data: data['sourceDirectory'] = sourceDir
if not 'generator' in data: data['generator'] = cmakeGenerator
if not 'extraGenerator' in data: data['extraGenerator'] = ''
cmakelib.validateGlobalSettings(proc, cmakeCommand, data)
elif 'validateCache' in obj:
data = obj['validateCache']
if not 'isEmpty' in data: data['isEmpty'] = false
cmakelib.validateCache(proc, data)
elif 'reconnect' in obj:
cmakelib.exitProc(proc)
proc = cmakelib.initServerProc(cmakeCommand, communicationMethod)
else:
print("Unknown command:", json.dumps(obj))
sys.exit(2)
cmakelib.shutdownProc(proc)
print("Completed")
cmakelib.exitProc(proc)
print('cmake-server exited: %d' % proc.returncode)
sys.exit(proc.returncode)