slim and full periphery. updated stats type

.vscode/tasks.json

@@ -120,6 +120,15 @@
 				"cwd": "${workspaceFolder}/tests"
 			},
 			"problemMatcher": []
+		},
+		{
+			"type": "shell",
+			"command": "docker compose build periphery",
+			"label": "docker compose build periphery",
+			"options": {
+				"cwd": "${workspaceFolder}/tests"
+			},
+			"problemMatcher": []
 		}
 	]
 }

Cargo.lock

@@ -1288,8 +1288,8 @@ version = "0.1.1"
 dependencies = [
  "async_timing_util",
  "bollard",
+ "bson",
  "diff-struct",
- "mungos",
  "serde",
  "serde_derive",
  "strum",

Dockerfile.core

@@ -1,11 +1,19 @@
 FROM rust:latest as builder
-WORKDIR /core
+WORKDIR /builder
 
-COPY . .
+COPY ./core ./core
 
-RUN cargo build -p core --release
+COPY ./lib/types ./lib/types
+COPY ./lib/helpers ./lib/helpers
+
+COPY ./lib/db_client ./lib/db_client
+COPY ./lib/periphery_client ./lib/periphery_client
+
+RUN cd core && cargo build --release
 
 FROM gcr.io/distroless/cc
-COPY --from=builder /core/target/release/core /
+COPY --from=builder /builder/core/target/release/core /
+
+EXPOSE 9000
 
 CMD ["./core"]

Dockerfile.periphery

@@ -1,15 +0,0 @@
-FROM rust:latest as builder
-WORKDIR /periphery
-
-COPY . .
-
-RUN cargo build -p periphery --release
-
-FROM debian:stable-slim
-# install git and docker
-COPY ./install_periphery_dependencies.sh ./
-RUN sh ./install_periphery_dependencies.sh
-
-COPY --from=builder /periphery/target/release/periphery /usr/local/bin/periphery
-
-CMD "periphery"

Dockerfile.periphery_full

@@ -0,0 +1,20 @@
+FROM rust:latest as builder
+WORKDIR /builder
+
+COPY ./periphery ./periphery
+
+COPY ./lib/types ./lib/types
+COPY ./lib/helpers ./lib/helpers
+
+RUN cd periphery && cargo build --release
+
+FROM debian:stable-slim
+
+COPY ./install_full_periphery_deps.sh ./
+RUN sh ./install_full_periphery_deps.sh
+
+COPY --from=builder /builder/periphery/target/release/periphery /usr/local/bin/periphery
+
+EXPOSE 9001
+
+CMD "periphery"

Dockerfile.periphery_slim

@@ -0,0 +1,20 @@
+FROM rust:latest as builder
+WORKDIR /builder
+
+COPY ./periphery ./periphery
+
+COPY ./lib/types ./lib/types
+COPY ./lib/helpers ./lib/helpers
+
+RUN cd periphery && cargo build --release
+
+FROM debian:stable-slim
+
+COPY ./install_slim_periphery_deps.sh ./
+RUN sh ./install_slim_periphery_deps.sh
+
+COPY --from=builder /builder/periphery/target/release/periphery /usr/local/bin/periphery
+
+EXPOSE 9001
+
+CMD "periphery"

cli/src/helpers.rs

@@ -57,7 +57,7 @@ pub fn gen_core_config(sub_matches: &ArgMatches) {
 
     write_to_toml(&path, config);
 
-    println!("\ncore config has been generated ✅");
+    println!("\n✅ core config has been generated ✅\n");
 }
 
 pub fn start_mongo(sub_matches: &ArgMatches) {
@@ -79,7 +79,7 @@ pub fn start_mongo(sub_matches: &ArgMatches) {
 
     // start mongo here
 
-    println!("\nmonitor mongo has been started up ✅")
+    println!("\n✅ monitor mongo has been started up ✅\n")
 }
 
 pub fn start_core(sub_matches: &ArgMatches) {
@@ -91,7 +91,7 @@ pub fn start_core(sub_matches: &ArgMatches) {
 
     // start core here
 
-    println!("\nmonitor core has been started up ✅");
+    println!("\n✅ monitor core has been started up ✅\n");
 }
 
 pub fn gen_periphery_config(sub_matches: &ArgMatches) {
@@ -117,7 +117,7 @@ pub fn gen_periphery_config(sub_matches: &ArgMatches) {
 
     write_to_toml(&path, config);
 
-    println!("\nperiphery config has been generated ✅");
+    println!("\n✅ periphery config has been generated ✅\n");
 }
 
 pub fn start_periphery(sub_matches: &ArgMatches) {
@@ -134,7 +134,7 @@ pub fn start_periphery(sub_matches: &ArgMatches) {
 
     // start periphery here
 
-    println!("\nmonitor periphery has been started up ✅");
+    println!("\n✅ monitor periphery has been started up ✅\n");
 }
 
 fn write_to_toml(path: &str, toml: impl Serialize) {
@@ -142,7 +142,7 @@ fn write_to_toml(path: &str, toml: impl Serialize) {
         path,
         toml::to_string(&toml).expect("failed to parse config into toml"),
     )
-    .expect("failed to write toml to file");
+    .expect("❌ failed to write toml to file ❌");
 }
 
 fn generate_secret(length: usize) -> String {

core/src/actions/build.rs

@@ -92,10 +92,10 @@ impl State {
             operator: user.id.clone(),
             logs: vec![
                 delete_repo_log,
-                Log::simple(format!(
-                    "deleted build {} on server {}",
-                    build.name, server.name
-                )),
+                Log::simple(
+                    "delete build",
+                    format!("deleted build {} on server {}", build.name, server.name),
+                ),
             ],
             success: true,
             ..Default::default()
@@ -129,7 +129,10 @@ impl State {
             target: UpdateTarget::Build(new_build.id.clone()),
             start_ts,
             status: UpdateStatus::InProgress,
-            logs: vec![Log::simple(serde_json::to_string_pretty(&diff).unwrap())],
+            logs: vec![Log::simple(
+                "build update",
+                serde_json::to_string_pretty(&diff).unwrap(),
+            )],
             operator: user.id.clone(),
             success: true,
             ..Default::default()

core/src/actions/deployment.rs

@@ -93,10 +93,13 @@ impl State {
             operator: user.id.clone(),
             logs: vec![
                 log,
-                Log::simple(format!(
-                    "deleted deployment {} on server {}",
-                    deployment.name, server.name
-                )),
+                Log::simple(
+                    "delete deployment",
+                    format!(
+                        "deleted deployment {} on server {}",
+                        deployment.name, server.name
+                    ),
+                ),
             ],
             success: true,
             ..Default::default()
@@ -133,7 +136,10 @@ impl State {
             target: UpdateTarget::Deployment(new_deployment.id.clone()),
             start_ts,
             status: UpdateStatus::InProgress,
-            logs: vec![Log::simple(serde_json::to_string_pretty(&diff).unwrap())],
+            logs: vec![Log::simple(
+                "deployment update",
+                serde_json::to_string_pretty(&diff).unwrap(),
+            )],
             operator: user.id.clone(),
             success: true,
             ..Default::default()
@@ -201,6 +207,7 @@ impl State {
 
         let deploy_log = self.periphery.deploy(&server, &deployment).await?;
 
+        update.success = deploy_log.success;
         update.logs.push(deploy_log);
         update.status = UpdateStatus::Complete;
         update.end_ts = Some(unix_timestamp_ms() as i64);

core/src/actions/server.rs

@@ -86,7 +86,10 @@ impl State {
             start_ts,
             end_ts: Some(unix_timestamp_ms() as i64),
             operator: user.id.clone(),
-            logs: vec![Log::simple(format!("deleted server {}", server.name))],
+            logs: vec![Log::simple(
+                "delete server",
+                format!("deleted server {}", server.name),
+            )],
             success: true,
             ..Default::default()
         };
@@ -120,7 +123,10 @@ impl State {
             start_ts,
             end_ts: Some(unix_timestamp_ms() as i64),
             status: UpdateStatus::Complete,
-            logs: vec![Log::simple(serde_json::to_string_pretty(&diff).unwrap())],
+            logs: vec![Log::simple(
+                "server update",
+                serde_json::to_string_pretty(&diff).unwrap(),
+            )],
             operator: user.id.clone(),
             success: true,
             ..Default::default()

install_full_periphery_deps.sh

@@ -0,0 +1,16 @@
+apt-get update
+apt-get install -y ca-certificates curl gnupg lsb-release make cmake g++ python3 node-gyp build-essential libssl-dev git
+git config --global pull.rebase false
+
+# install docker cli
+# mkdir -p /etc/apt/keyrings
+# curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+# echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+# chmod a+r /etc/apt/keyrings/docker.gpg
+# apt-get update
+# apt-get install -y docker-ce docker-ce-cli containerd.io
+curl -fsSL https://get.docker.com | sh
+
+# install nodejs and enable yarn
+curl -fsSL https://deb.nodesource.com/setup_19.x | bash - && apt-get install -y nodejs
+corepack enable

install_periphery_dependencies.sh

@@ -1,6 +0,0 @@
-apt-get update
-apt-get install ca-certificates curl gnupg lsb-release make cmake g++ python3 node-gyp build-essential libssl-dev git -y
-curl -fsSL https://get.docker.com | sh
-curl -fsSL https://deb.nodesource.com/setup_19.x | bash - && apt-get install -y nodejs
-git config pull.rebase false
-corepack enable

install_slim_periphery_deps.sh

@@ -0,0 +1,11 @@
+apt-get update
+apt-get install -y ca-certificates curl gnupg lsb-release git
+git config --global pull.rebase false
+
+# install docker cli
+# mkdir -p /etc/apt/keyrings
+# curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+# echo \ "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+# apt-get update
+# apt-get install docker-ce docker-ce-cli
+curl -fsSL https://get.docker.com | sh

lib/types/Cargo.toml

@@ -11,7 +11,7 @@ license = "MIT"
 [dependencies]
 serde = "1.0"
 serde_derive = "1.0"
-mungos = "0.2.24"
+bson = "2.4"
 strum = "0.24"
 strum_macros = "0.24"
 async_timing_util = "0.1.12"

lib/types/src/lib.rs

@@ -1,8 +1,8 @@
 use std::{collections::HashMap, path::PathBuf};
 
 use async_timing_util::{unix_timestamp_ms, Timelength};
+use bson::serde_helpers::hex_string_as_object_id;
 use diff::{Diff, HashMapDiff, OptionDiff, VecDiff};
-use mungos::mongodb::bson::serde_helpers::hex_string_as_object_id;
 use serde::{Deserialize, Serialize};
 use strum_macros::{Display, EnumString};
 
@@ -413,9 +413,10 @@ pub struct Log {
 }
 
 impl Log {
-    pub fn simple(msg: String) -> Log {
+    pub fn simple(stage: &str, msg: String) -> Log {
         let ts = unix_timestamp_ms() as i64;
         Log {
+            stage: stage.to_string(),
             stdout: msg,
             success: true,
             start_ts: ts,
@@ -561,34 +562,34 @@ pub struct UserCredentials {
 
 #[derive(Serialize, Deserialize, Debug)]
 pub struct SystemStats {
-    pub cpu: f32,       // in %
-    pub mem_used: f64,  // in MB
-    pub mem_total: f64, // in MB
+    pub cpu_perc: f32,     // in %
+    pub mem_used_gb: f64,  // in GB
+    pub mem_total_gb: f64, // in GB
     pub disk: DiskUsage,
     pub networks: Vec<SystemNetwork>,
 }
 
 #[derive(Serialize, Deserialize, Debug)]
 pub struct DiskUsage {
-    pub used: f64,  // in GB
-    pub total: f64, // in GB
-    pub read: f64,  // in kB
-    pub write: f64, // in kB
+    pub used_gb: f64,  // in GB
+    pub total_gb: f64, // in GB
+    pub read_kb: f64,  // in kB
+    pub write_kb: f64, // in kB
     pub disks: Vec<SingleDiskUsage>,
 }
 
 #[derive(Serialize, Deserialize, Debug)]
 pub struct SingleDiskUsage {
     pub mount: PathBuf,
-    pub used: f64,  // in GB
-    pub total: f64, // in GB
+    pub used_gb: f64,  // in GB
+    pub total_gb: f64, // in GB
 }
 
 #[derive(Serialize, Deserialize, Debug)]
 pub struct SystemNetwork {
     pub name: String,
-    pub recieved: f64,    // in kB
-    pub transmitted: f64, // in kB
+    pub recieved_kb: f64,    // in kB
+    pub transmitted_kb: f64, // in kB
 }
 
 #[derive(Serialize, Deserialize, Debug, Display, EnumString, PartialEq, Hash, Eq, Clone, Copy)]

periphery/src/api/git.rs

@@ -51,9 +51,10 @@ async fn delete_repo(
     repo_dir.push(&name);
     let destination = repo_dir.display().to_string();
     let deleted = std::fs::remove_dir_all(destination);
-    let log = match deleted {
-        Ok(_) => Log::simple(format!("deleted repo {name}")),
-        Err(_) => Log::simple(format!("no repo at {name} to delete")),
+    let msg = match deleted {
+        Ok(_) => format!("deleted repo {name}"),
+        Err(_) => format!("no repo at {name} to delete"),
     };
+    let log = Log::simple("delete repo", msg);
     Ok(Json(log))
 }

periphery/src/api/stats.rs

@@ -37,9 +37,9 @@ impl StatsClient {
         self.sys.refresh_cpu();
         self.sys.refresh_memory();
         SystemStats {
-            cpu: self.sys.global_cpu_info().cpu_usage(),
-            mem_used: self.sys.used_memory() as f64 / BYTES_PER_GB,
-            mem_total: self.sys.total_memory() as f64 / BYTES_PER_GB,
+            cpu_perc: self.sys.global_cpu_info().cpu_usage(),
+            mem_used_gb: self.sys.used_memory() as f64 / BYTES_PER_GB,
+            mem_total_gb: self.sys.total_memory() as f64 / BYTES_PER_GB,
             disk: self.get_disk_usage(),
             networks: self.get_networks(),
         }
@@ -52,44 +52,44 @@ impl StatsClient {
             .into_iter()
             .map(|(name, n)| SystemNetwork {
                 name: name.clone(),
-                recieved: n.received() as f64 / BYTES_PER_KB,
-                transmitted: n.transmitted() as f64 / BYTES_PER_KB,
+                recieved_kb: n.received() as f64 / BYTES_PER_KB,
+                transmitted_kb: n.transmitted() as f64 / BYTES_PER_KB,
             })
-            .filter(|n| n.recieved > 0.0 || n.transmitted > 0.0)
+            .filter(|n| n.recieved_kb > 0.0 || n.transmitted_kb > 0.0)
             .collect()
     }
 
     fn get_disk_usage(&mut self) -> DiskUsage {
         self.sys.refresh_disks();
-        let mut free = 0.0;
-        let mut total = 0.0;
+        let mut free_gb = 0.0;
+        let mut total_gb = 0.0;
         let mut disks = Vec::new();
         for disk in self.sys.disks() {
             let disk_total = disk.total_space() as f64 / BYTES_PER_GB;
             let disk_free = disk.available_space() as f64 / BYTES_PER_GB;
             disks.push(SingleDiskUsage {
                 mount: disk.mount_point().to_owned(),
-                used: disk_total - disk_free,
-                total: disk_total,
+                used_gb: disk_total - disk_free,
+                total_gb: disk_total,
             });
-            total += disk_total;
-            free += disk_free;
+            total_gb += disk_total;
+            free_gb += disk_free;
         }
-        let used = total - free;
+        let used_gb = total_gb - free_gb;
         self.sys
             .refresh_processes_specifics(ProcessRefreshKind::new().with_disk_usage());
-        let mut read = 0.0;
-        let mut write = 0.0;
+        let mut read_kb = 0.0;
+        let mut write_kb = 0.0;
         for (_, process) in self.sys.processes() {
             let disk_usage = process.disk_usage();
-            read += disk_usage.read_bytes as f64 / BYTES_PER_KB;
-            write += disk_usage.written_bytes as f64 / BYTES_PER_KB;
+            read_kb += disk_usage.read_bytes as f64 / BYTES_PER_KB;
+            write_kb += disk_usage.written_bytes as f64 / BYTES_PER_KB;
         }
         DiskUsage {
-            used,
-            total,
-            read,
-            write,
+            used_gb,
+            total_gb,
+            read_kb,
+            write_kb,
             disks,
         }
     }

tests/docker-compose.yml

@@ -16,11 +16,11 @@ services:
     depends_on:
       - mongo
 
-  periphery:
-    image: periphery
+  periphery-full:
+    image: periphery-full
     build:
       context: ..
-      dockerfile: Dockerfile.periphery
+      dockerfile: Dockerfile.periphery_full
     ports:
       - "9001:9001"
     networks:
@@ -29,7 +29,23 @@ services:
       - CONFIG_PATH=/config/config.toml
     volumes:
       - "/Users/max/monitor/periphery:/config"
-      - "/Users/max/monitor/repos:/repos"
+      - "/Users/max/monitor/repos-full:/repos"
+      - "/var/run/docker.sock:/var/run/docker.sock"
+
+  periphery-slim:
+    image: periphery-slim
+    build:
+      context: ..
+      dockerfile: Dockerfile.periphery_slim
+    ports:
+      - "9002:9001"
+    networks:
+      - monitor-network
+    environment:
+      - CONFIG_PATH=/config/config.toml
+    volumes:
+      - "/Users/max/monitor/periphery:/config"
+      - "/Users/max/monitor/repos-slim:/repos"
       - "/var/run/docker.sock:/var/run/docker.sock"
 
   mongo:

tests/src/main.rs

@@ -17,19 +17,19 @@ async fn main() -> anyhow::Result<()> {
 
     let start_ts = unix_timestamp_ms();
 
-    // let stats = get_server_stats(&monitor).await?;
-    // println!("{stats:#?}");
-
     let (server, deployment, build) = create_test_setup(&monitor, "test").await?;
 
     let server_stats = get_server_stats(&monitor).await?;
-    println!("{server_stats:#?}\n");
+    println!("server stats:\n{server_stats:#?}\n");
 
     let (update, container) = deploy_mongo(&monitor).await?;
-    println!("{update:#?}\n{container:#?}\n");
+    println!(
+        "mongo deploy update:\n{update:#?}\n\ncontainer: {:#?}\n",
+        container.container
+    );
 
-    let update = test_build(&monitor).await?;
-    println!("{update:#?}");
+    // let update = test_build(&monitor).await?;
+    // println!("build update:\n{update:#?}");
 
     let end_ts = unix_timestamp_ms();
     let finished_in = (end_ts - start_ts) as f64 / 1000.0;

tests/src/tests.rs

@@ -11,18 +11,36 @@ pub async fn create_test_setup(
     monitor: &MonitorClient,
     group_name: &str,
 ) -> anyhow::Result<(Server, Deployment, Build)> {
-    let server = monitor
-        .create_server(&format!("{group_name}_server"), "http://periphery:9001")
-        .await
-        .context("failed at create server")?;
-    let deployment = monitor
-        .create_deployment(&format!("{group_name}"), &server.id)
-        .await
-        .context("failed at create deployment")?;
-    let build = monitor
-        .create_build(&format!("{group_name}_build"), &server.id)
-        .await
-        .context("failed at create build")?;
+    let mut servers = monitor.list_servers(None).await?;
+    let server = if servers.is_empty() {
+        monitor
+            .create_server(
+                &format!("{group_name}_server"),
+                "http://periphery-full:9001",
+            )
+            .await
+            .context("failed at create server")?
+    } else {
+        servers.pop().unwrap()
+    };
+    let mut deployments = monitor.list_deployments(None).await?;
+    let deployment = if deployments.is_empty() {
+        monitor
+            .create_deployment(&format!("{group_name}_deployment"), &server.id)
+            .await
+            .context("failed at create deployment")?
+    } else {
+        deployments.pop().unwrap().deployment
+    };
+    let mut builds = monitor.list_builds(None).await?;
+    let build = if builds.is_empty() {
+        monitor
+            .create_build(&format!("{group_name}_build"), &server.id)
+            .await
+            .context("failed at create build")?
+    } else {
+        builds.pop().unwrap()
+    };
     Ok((server, deployment, build))
 }