diff --git a/cpp/cdc/CDC.cpp b/cpp/cdc/CDC.cpp index 35f879f1..f7a33768 100644 --- a/cpp/cdc/CDC.cpp +++ b/cpp/cdc/CDC.cpp @@ -925,57 +925,6 @@ public: } }; -struct CDCStatsInserter : PeriodicLoop { -private: - CDCShared& _shared; - std::string _shuckleHost; - uint16_t _shucklePort; - XmonNCAlert _alert; - std::vector _stats; - -public: - CDCStatsInserter(Logger& logger, std::shared_ptr& xmon, const CDCOptions& options, CDCShared& shared): - PeriodicLoop(logger, xmon, "stats_inserter", {1_sec, 1_hours}), - _shared(shared), - _shuckleHost(options.shuckleHost), - _shucklePort(options.shucklePort), - _alert(XmonAppType::DAYTIME, 10_sec) - {} - - virtual ~CDCStatsInserter() = default; - - virtual bool periodicStep() override { - for (CDCMessageKind kind : allCDCMessageKind) { - { - std::ostringstream prefix; - prefix << "cdc." << kind; - _shared.timingsTotal[(int)kind].toStats(prefix.str(), _stats); - _shared.errors[(int)kind].toStats(prefix.str(), _stats); - } - } - const auto [err, errStr] = insertStats(_shuckleHost, _shucklePort, 10_sec, _stats); - if (err == EINTR) { return false; } - _stats.clear(); - if (err == 0) { - _env.clearAlert(_alert); - for (CDCMessageKind kind : allCDCMessageKind) { - _shared.timingsTotal[(int)kind].reset(); - _shared.errors[(int)kind].reset(); - } - return true; - } else { - _env.updateAlert(_alert, "Could not insert stats: %s", errStr); - return false; - } - } - - // TODO restore this when we can - // virtual void finish() override { - // LOG_INFO(_env, "inserting stats one last time"); - // periodicStep(); - // } -}; - static void logsDBstatsToMetrics(struct MetricsBuilder& metricsBuilder, const LogsDBStats& stats, ReplicaId replicaId, EggsTime now) { { metricsBuilder.measurement("eggsfs_cdc_logsdb"); @@ -1248,9 +1197,6 @@ void runCDC(const std::string& dbDir, CDCOptions& options) { threads.emplace_back(LoopThread::Spawn(std::make_unique(logger, xmon, options, shared))); threads.emplace_back(LoopThread::Spawn(std::make_unique(logger, xmon, options, shared, dbDir))); threads.emplace_back(LoopThread::Spawn(std::make_unique(logger, xmon, options, shared))); - if (options.shuckleStats) { - threads.emplace_back(LoopThread::Spawn(std::make_unique(logger, xmon, options, shared))); - } if (options.metrics) { threads.emplace_back(LoopThread::Spawn(std::make_unique(logger, xmon, shared, options.replicaId))); } diff --git a/cpp/cdc/CDC.hpp b/cpp/cdc/CDC.hpp index 9126ae19..9a81b024 100644 --- a/cpp/cdc/CDC.hpp +++ b/cpp/cdc/CDC.hpp @@ -20,7 +20,6 @@ struct CDCOptions { bool xmon = false; bool xmonProd = false; bool metrics = false; - bool shuckleStats = false; ReplicaId replicaId = 0; // LogsDB settings diff --git a/cpp/cdc/eggscdc.cpp b/cpp/cdc/eggscdc.cpp index 6f8f4df2..404c75fd 100644 --- a/cpp/cdc/eggscdc.cpp +++ b/cpp/cdc/eggscdc.cpp @@ -179,8 +179,6 @@ int main(int argc, char** argv) { } } else if (arg == "-metrics") { options.metrics = true; - } else if (arg == "-shuckle-stats") { - options.shuckleStats = true; } else if (arg == "-use-logsdb") { std::string logsDBMode = getNextArg(); if (logsDBMode == "LEADER") { diff --git a/cpp/core/ErrorCount.cpp b/cpp/core/ErrorCount.cpp index e6d47f69..82afb29e 100644 --- a/cpp/core/ErrorCount.cpp +++ b/cpp/core/ErrorCount.cpp @@ -1,28 +1,5 @@ #include "ErrorCount.hpp" -void ErrorCount::toStats(const std::string& prefix, std::vector& stats) { - static_assert(std::endian::native == std::endian::little); - auto now = eggsNow(); - { - std::vector> seen; - for (int i = 0; i < count.size(); i++) { - uint64_t c = count[i].load(); - if (c == 0) { continue; } - seen.push_back({ (EggsError)i, c }); - } - auto& errStat = stats.emplace_back(); - errStat.time = now; - errStat.name = BincodeBytes(prefix + ".errors"); - // error + count - errStat.value.els.resize(seen.size() * (2 + 8)); - for (int i = 0; i < seen.size(); i++) { - const auto& errCount = seen[i]; - memcpy(errStat.value.els.data() + i*(2 + 8), &errCount.first, 2); - memcpy(errStat.value.els.data() + i*(2 + 8) + 2, &errCount.second, 8); - } - } -} - void ErrorCount::reset() { for (int i = 0; i < count.size(); i++) { count[i].store(0); diff --git a/cpp/core/ErrorCount.hpp b/cpp/core/ErrorCount.hpp index 359960bd..610402ea 100644 --- a/cpp/core/ErrorCount.hpp +++ b/cpp/core/ErrorCount.hpp @@ -20,7 +20,6 @@ struct ErrorCount { count[(int)err]++; } - void toStats(const std::string& prefix, std::vector& stats); // void toMetrics(const MetricsBuilder& builder); // will add tags and then fields void reset(); }; diff --git a/cpp/core/Shuckle.cpp b/cpp/core/Shuckle.cpp index ca98a967..7fcda680 100644 --- a/cpp/core/Shuckle.cpp +++ b/cpp/core/Shuckle.cpp @@ -360,32 +360,3 @@ bool parseShuckleAddress(const std::string& fullShuckleAddress, std::string& shu shuckleHost = {fullShuckleAddress.begin(), fullShuckleAddress.begin()+colon}; return true; } - -std::pair insertStats( - const std::string& host, - uint16_t port, - Duration timeout, - const std::vector& stats -) { - const auto [sock, errStr] = shuckleSock(host, port, timeout); - if (sock.error()) { - return {sock.getErrno(), errStr}; - } - - ShuckleReqContainer reqContainer; - auto& req = reqContainer.setInsertStats(); - req.stats.els.insert(req.stats.els.end(), std::make_move_iterator(stats.begin()), std::make_move_iterator(stats.end())); - { - const auto [err, errStr] = writeShuckleRequest(sock.get(), reqContainer, timeout); - if (err) { return {err, errStr}; } - } - - ShuckleRespContainer respContainer; - { - const auto [err, errStr] = readShuckleResponse(sock.get(), respContainer, timeout); - if (err) { return {err, errStr}; } - } - - return {}; - -} diff --git a/cpp/core/Shuckle.hpp b/cpp/core/Shuckle.hpp index a5271247..140c36b4 100644 --- a/cpp/core/Shuckle.hpp +++ b/cpp/core/Shuckle.hpp @@ -63,12 +63,5 @@ std::pair fetchShards( std::array& shards ); -std::pair insertStats( - const std::string& shuckleHost, - uint16_t shucklePort, - Duration timeout, - const std::vector& stats -); - const std::string defaultShuckleAddress = "REDACTED"; bool parseShuckleAddress(const std::string& fullShuckleAddress, std::string& shuckleHost, uint16_t& shucklePort); diff --git a/cpp/core/Timings.cpp b/cpp/core/Timings.cpp index 3d3116ec..6888ed73 100644 --- a/cpp/core/Timings.cpp +++ b/cpp/core/Timings.cpp @@ -19,28 +19,6 @@ Timings::Timings(Duration firstUpperBound, double growth, int bins) : } } -void Timings::toStats(const std::string& prefix, std::vector& stats) { - static_assert(std::endian::native == std::endian::little); - auto now = eggsNow(); - auto elapsed = now - _startedAt; - { - auto& histStat = stats.emplace_back(); - histStat.time = now; - histStat.name = BincodeBytes(prefix + ".latency"); - // elapsed, then upperbound+count - histStat.value.els.resize(8 + 2*8*_bins.size()); - memcpy(histStat.value.els.data(), &elapsed, 8); - double upperBound = _firstUpperBound; - for (int i = 0; i < _bins.size(); i++) { - uint64_t x = upperBound; - memcpy(histStat.value.els.data() + 8 + i*2*8, &x, 8); - x = _bins[i].load(); - memcpy(histStat.value.els.data() + 8 + i*2*8 + 8, &x, 8); - upperBound *= _growth; - } - } -} - void Timings::reset() { _startedAt = eggsNow(); for (auto& bin : _bins) { @@ -93,4 +71,4 @@ void Timings::toMetrics(MetricsBuilder& builder, const std::string& name, const } builder.fieldU64("count", sum); } -#endif \ No newline at end of file +#endif diff --git a/cpp/core/Timings.hpp b/cpp/core/Timings.hpp index 19438686..9bc7048d 100644 --- a/cpp/core/Timings.hpp +++ b/cpp/core/Timings.hpp @@ -47,7 +47,6 @@ public: Duration mean() const; Duration percentile(double p) const; - void toStats(const std::string& prefix, std::vector& stats); // void toMetrics(MetricsBuilder& builder, const std::string& name, const std::vector>& tags); void reset(); }; diff --git a/cpp/shard/Shard.cpp b/cpp/shard/Shard.cpp index d5cc4832..9facc4d5 100644 --- a/cpp/shard/Shard.cpp +++ b/cpp/shard/Shard.cpp @@ -1014,57 +1014,6 @@ public: } }; -struct ShardStatsInserter : PeriodicLoop { -private: - ShardShared& _shared; - ShardReplicaId _shrid; - std::string _shuckleHost; - uint16_t _shucklePort; - XmonNCAlert _alert; - std::vector _stats; -public: - ShardStatsInserter(Logger& logger, std::shared_ptr& xmon, ShardReplicaId shrid, const ShardOptions& options, ShardShared& shared): - PeriodicLoop(logger, xmon, "stats", {1_mins, 1_hours}), - _shared(shared), - _shrid(shrid), - _shuckleHost(options.shuckleHost), - _shucklePort(options.shucklePort), - _alert(XmonAppType::DAYTIME) - {} - - virtual ~ShardStatsInserter() = default; - - virtual bool periodicStep() override { - for (ShardMessageKind kind : allShardMessageKind) { - std::ostringstream prefix; - prefix << "shard." << _shrid << "." << kind; - _shared.timings[(int)kind].toStats(prefix.str(), _stats); - _shared.errors[(int)kind].toStats(prefix.str(), _stats); - } - LOG_INFO(_env, "inserting stats"); - auto [err, errStr] = insertStats(_shuckleHost, _shucklePort, 10_sec, _stats); - if (err == EINTR) { return false; } - _stats.clear(); - if (err == 0) { - _env.clearAlert(_alert); - for (ShardMessageKind kind : allShardMessageKind) { - _shared.timings[(int)kind].reset(); - _shared.errors[(int)kind].reset(); - } - return true; - } else { - _env.updateAlert(_alert, "Could not insert stats: %s", errStr); - return false; - } - } - - // TODO restore this when we have the functionality to do so - // virtual void finish() override { - // LOG_INFO(_env, "insert stats one last time"); - // periodicStep(); - // } -}; - static void logsDBstatsToMetrics(struct MetricsBuilder& metricsBuilder, const LogsDBStats& stats, ShardReplicaId shrid, EggsTime now) { { metricsBuilder.measurement("eggsfs_shard_logsdb"); @@ -1355,9 +1304,6 @@ void runShard(ShardReplicaId shrid, const std::string& dbDir, ShardOptions& opti threads.emplace_back(LoopThread::Spawn(std::make_unique(logger, xmon, shrid, options, shared, dbDir))); threads.emplace_back(LoopThread::Spawn(std::make_unique(logger, xmon, shrid, options, shared))); threads.emplace_back(LoopThread::Spawn(std::make_unique(logger, xmon, shrid, options, shared))); - if (options.shuckleStats) { - threads.emplace_back(LoopThread::Spawn(std::make_unique(logger, xmon, shrid, options, shared))); - } if (options.metrics) { threads.emplace_back(LoopThread::Spawn(std::make_unique(logger, xmon, shrid, shared))); } diff --git a/cpp/shard/Shard.hpp b/cpp/shard/Shard.hpp index d819881f..9f7e8649 100644 --- a/cpp/shard/Shard.hpp +++ b/cpp/shard/Shard.hpp @@ -18,7 +18,6 @@ struct ShardOptions { bool xmon = false; bool xmonProd = false; bool metrics = false; - bool shuckleStats = false; Duration transientDeadlineInterval = DEFAULT_DEADLINE_INTERVAL; // LogsDB settings diff --git a/cpp/shard/eggsshard.cpp b/cpp/shard/eggsshard.cpp index 118f353e..45f298e3 100644 --- a/cpp/shard/eggsshard.cpp +++ b/cpp/shard/eggsshard.cpp @@ -42,8 +42,6 @@ static void usage(const char* binary) { fprintf(stderr, " Specify in which mode to use LogsDB, as LEADER|LEADER_NO_FOLLOWERS|FOLLOWER. Default is FOLLOWER.\n"); fprintf(stderr, " -force-last-released LogIdx\n"); fprintf(stderr, " Force forward last released. Used for manual leader election. Can not be combined with starting in any LEADER mode\n"); - fprintf(stderr, " -shuckle-stats\n"); - fprintf(stderr, " Insert shuckle histogram stats.\n"); } static double parseDouble(const std::string& arg) { @@ -213,8 +211,6 @@ int main(int argc, char** argv) { } } else if (arg == "-metrics") { options.metrics = true; - } else if (arg == "-shuckle-stats") { - options.shuckleStats = true; } else if (arg == "-transient-deadline-interval") { options.transientDeadlineInterval = parseDuration(getNextArg()); } else if (arg == "-use-logsdb") {