From cec84fa8d3fff44e01448796b78d4f7bc8588683 Mon Sep 17 00:00:00 2001
From: theMackabu <theMackabu@gmail.com>
Date: Mon, 16 Oct 2023 22:59:27 -0700
Subject: [PATCH] client: commuication changes server: implement container
system
build: update gitignore
---
.gitignore | 1 +
Cargo.lock | 42 +++++
Maidfile.toml | 17 +-
crates/maid/client/src/cli/mod.rs | 10 +-
crates/maid/client/src/cli/tasks.rs | 5 +-
crates/maid/client/src/helpers/logger.rs | 2 +
crates/maid/client/src/server/cli.rs | 43 +++--
crates/maid/client/src/structs.rs | 19 +-
crates/maid/server/Cargo.toml | 21 ++-
crates/maid/server/src/docker/exec.rs | 0
crates/maid/server/src/docker/mod.rs | 2 +-
crates/maid/server/src/docker/run.rs | 229 +++++++++++++++++++++++
crates/maid/server/src/globals.rs | 5 +
crates/maid/server/src/helpers/file.rs | 65 +++++++
crates/maid/server/src/helpers/logger.rs | 25 +++
crates/maid/server/src/helpers/mod.rs | 15 +-
crates/maid/server/src/helpers/string.rs | 17 ++
crates/maid/server/src/main.rs | 29 +--
crates/maid/server/src/structs.rs | 90 +++++++++
crates/maid/server/src/table.rs | 71 +++++++
scripts/build.toml | 62 +++---
21 files changed, 691 insertions(+), 79 deletions(-)
delete mode 100644 crates/maid/server/src/docker/exec.rs
create mode 100644 crates/maid/server/src/docker/run.rs
create mode 100644 crates/maid/server/src/helpers/file.rs
create mode 100644 crates/maid/server/src/helpers/logger.rs
create mode 100644 crates/maid/server/src/helpers/string.rs
create mode 100644 crates/maid/server/src/table.rs
diff --git a/.gitignore b/.gitignore
index 3f67697..a3656a8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,6 +3,7 @@ target
TODO.md
bin
.idea
+build
testing
# prevent duplicate README
diff --git a/Cargo.lock b/Cargo.lock
index 58569c2..d12d6d1 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -641,6 +641,21 @@ dependencies = [
"libc",
]
+[[package]]
+name = "futures"
+version = "0.3.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "futures-executor",
+ "futures-io",
+ "futures-sink",
+ "futures-task",
+ "futures-util",
+]
+
[[package]]
name = "futures-channel"
version = "0.3.28"
@@ -657,6 +672,17 @@ version = "0.3.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c"
+[[package]]
+name = "futures-executor"
+version = "0.3.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0"
+dependencies = [
+ "futures-core",
+ "futures-task",
+ "futures-util",
+]
+
[[package]]
name = "futures-io"
version = "0.3.28"
@@ -692,6 +718,7 @@ version = "0.3.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533"
dependencies = [
+ "futures-channel",
"futures-core",
"futures-io",
"futures-macro",
@@ -1184,20 +1211,35 @@ dependencies = [
name = "maid_server"
version = "1.0.0"
dependencies = [
+ "anyhow",
"bollard",
+ "bytes",
"chrono",
"clap",
"clap-verbosity-flag",
+ "colored",
"env_logger",
"flate2",
+ "futures",
+ "futures-core",
"futures-util",
+ "global_placeholders",
+ "home",
+ "indicatif",
"libc",
"log",
"macros-rs",
"ntapi",
+ "serde",
+ "serde_derive",
"serde_json",
+ "tar",
+ "termcolor",
+ "text_placeholder",
"tokio",
+ "tokio-util",
"tungstenite",
+ "uuid",
"warp",
"winapi",
]
diff --git a/Maidfile.toml b/Maidfile.toml
index 484bdf8..ab79692 100644
--- a/Maidfile.toml
+++ b/Maidfile.toml
@@ -22,8 +22,7 @@ VERSION='1.0.0'
info = "Build binaries"
depends = ["clean"]
script = [
- # "maid clean -q",
- "cargo build --release",
+ "cargo zigbuild --release",
"mv target/release/maid bin/maid",
"mv target/release/maid_server bin/maid_server",
"mv target/release/exit_test bin/exit_test"
@@ -38,17 +37,15 @@ target = [
]
[tasks.build.remote]
-image = "rust"
+silent = true
+exclusive = false
+shell = "/bin/bash"
+image = "rust:1.73"
push = ["crates", "Cargo.toml", "Cargo.lock"]
-pull = [
- "bin/maid",
- "bin/exit_test",
- "bin/maid_server"
-]
+pull = "bin"
# basic task definition
[tasks]
api_server = { depends = ["build"], script = "./maid_server", path = "bin" }
clean = { info = "Clean binary files", script = ["rm -rf bin", "mkdir bin"] }
-install = { info = "Move binary file", script = ["sudo cp bin/maid /usr/local/bin", "echo Copied binary!"], depends = ["build"] }
-buildall = { info = "build all", script = ["rm -rf build", "mkdir build", "maid _build_macos", "maid _build_linux", "maid _build_windows"] }
\ No newline at end of file
+install = { info = "Move binary file", script = ["sudo cp bin/maid /usr/local/bin", "echo Copied binary!"], depends = ["build"] }
\ No newline at end of file
diff --git a/crates/maid/client/src/cli/mod.rs b/crates/maid/client/src/cli/mod.rs
index 19ceca5..432be7b 100644
--- a/crates/maid/client/src/cli/mod.rs
+++ b/crates/maid/client/src/cli/mod.rs
@@ -44,7 +44,11 @@ pub fn exec(task: &str, args: &Vec<String>, path: &String, silent: bool, is_dep:
log::info!("Starting maid {}", env!("CARGO_PKG_VERSION"));
if task.is_empty() {
- tasks::List::all(path, silent, log_level);
+ if is_remote {
+ tasks::List::remote(path, silent, log_level);
+ } else {
+ tasks::List::all(path, silent, log_level);
+ }
} else {
let values = helpers::maidfile::merge(path);
let project_root = parse::file::find_maidfile_root(path);
@@ -58,6 +62,10 @@ pub fn exec(task: &str, args: &Vec<String>, path: &String, silent: bool, is_dep:
crashln!("Maid could not find the remote task '{task}'. Does it exist?");
}
+ if is_remote && values.tasks.get(task).unwrap().remote.as_ref().unwrap().exclusive {
+ crashln!("Task '{task}' is remote only.");
+ }
+
if !is_remote {
match &values.tasks[task].depends {
Some(deps) => {
diff --git a/crates/maid/client/src/cli/tasks.rs b/crates/maid/client/src/cli/tasks.rs
index 5d33c81..e679aed 100644
--- a/crates/maid/client/src/cli/tasks.rs
+++ b/crates/maid/client/src/cli/tasks.rs
@@ -41,7 +41,10 @@ impl List {
true => true,
false => match task.hide {
Some(val) => val,
- None => false,
+ None => match task.remote.as_ref() {
+ Some(val) => val.exclusive,
+ None => false,
+ },
},
};
diff --git a/crates/maid/client/src/helpers/logger.rs b/crates/maid/client/src/helpers/logger.rs
index 992efb1..bb072c7 100644
--- a/crates/maid/client/src/helpers/logger.rs
+++ b/crates/maid/client/src/helpers/logger.rs
@@ -7,6 +7,8 @@ macro_rules! log {
("warning", ("WARN", "yellow")),
("success", ("SUCCESS", "green")),
("notice", ("NOTICE", "bright blue")),
+ ("docker", ("DOCKER", "bright yellow")),
+ ("build", ("BUILD", "bright green")),
("info", ("INFO", "cyan")),
("debug", ("DEBUG", "magenta")),
]
diff --git a/crates/maid/client/src/server/cli.rs b/crates/maid/client/src/server/cli.rs
index 76504d4..ef637b0 100644
--- a/crates/maid/client/src/server/cli.rs
+++ b/crates/maid/client/src/server/cli.rs
@@ -1,14 +1,12 @@
use crate::helpers;
use crate::server;
-use crate::structs::{Maidfile, Task, Websocket};
-use crate::table;
+use crate::structs::{ConnectionData, ConnectionInfo, Maidfile, Task, Websocket};
use colored::Colorize;
use macros_rs::{crashln, fmtstr, then};
use reqwest::blocking::Client;
-use text_placeholder::Template;
use tungstenite::protocol::frame::{coding::CloseCode::Normal, CloseFrame};
-use tungstenite::{client::IntoClientRequest, connect as connectWSS, Message};
+use tungstenite::{client::connect_with_config, client::IntoClientRequest, protocol::WebSocketConfig, Message};
fn health(client: Client, values: Maidfile) -> server::api::health::Route {
let address = server::parse::address(&values);
@@ -56,7 +54,6 @@ pub fn connect(path: &String) {
}
pub fn remote(task: Task) {
- let args = &task.args.clone();
let mut script: Vec<&str> = vec![];
if task.script.is_str() {
@@ -90,28 +87,31 @@ pub fn remote(task: Task) {
crate::log!("info", "connecting to {host}:{port}");
if body.status.healthy.data == "yes" {
- crate::log!("info", "connected successfully");
+ crate::log!("notice", "server reports healthy");
} else {
crate::log!("warning", "failed to connect");
}
+ let websocket_config = WebSocketConfig {
+ max_frame_size: Some(314572800),
+ ..Default::default()
+ };
+
let mut request = websocket.into_client_request().expect("Can't connect");
request.headers_mut().insert("Authorization", fmtstr!("Bearer {token}").parse().unwrap());
- let (mut socket, response) = connectWSS(request).expect("Can't connect");
+ let (mut socket, response) = connect_with_config(request, Some(websocket_config), 3).expect("Can't connect");
log::debug!("response code: {}", response.status());
- let connection_data = serde_json::json!({
- "info": {
- "name": &task.name,
- "args": &task.args,
- "remote": &task.remote,
- "script": script,
+ let connection_data = ConnectionData {
+ info: ConnectionInfo {
+ name: task.name.clone(),
+ args: task.args.clone(),
+ remote: task.remote.clone().unwrap(),
+ script: script.clone().iter().map(|&s| s.to_string()).collect(),
},
- "maidfile": Template::new_with_placeholder(
- &task.maidfile.clone().to_json(), "%{", "}"
- ).fill_with_hashmap(&table::create(task.maidfile.clone(), args, task.project)),
- });
+ maidfile: task.maidfile.clone(),
+ };
let file_name = match server::file::write_tar(&task.remote.unwrap().push) {
Ok(name) => name,
@@ -123,9 +123,6 @@ pub fn remote(task: Task) {
log::debug!("sending information");
socket.send(Message::Text(serde_json::to_string(&connection_data).unwrap())).unwrap();
- log::debug!("sending archive");
- socket.send(Message::Binary(std::fs::read(&file_name).unwrap())).unwrap();
-
loop {
match socket.read() {
Ok(Message::Text(text)) => {
@@ -135,6 +132,12 @@ pub fn remote(task: Task) {
crate::log!(level.as_str(), "{}", msg);
}
});
+
+ if data.get("binary").map_or(false, |d| d.as_bool().unwrap_or(false)) {
+ log::debug!("sending archive");
+ socket.send(Message::Binary(std::fs::read(&file_name).unwrap())).unwrap();
+ }
+
then!(data.get("done").map_or(false, |d| d.as_bool().unwrap_or(false)), break);
}
}
diff --git a/crates/maid/client/src/structs.rs b/crates/maid/client/src/structs.rs
index 6175911..3d8c778 100644
--- a/crates/maid/client/src/structs.rs
+++ b/crates/maid/client/src/structs.rs
@@ -70,8 +70,11 @@ pub struct CacheConfig {
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct Remote {
pub push: Vec<String>,
- pub pull: Vec<String>,
+ pub pull: String,
pub image: String,
+ pub shell: String,
+ pub silent: bool,
+ pub exclusive: bool,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
@@ -113,3 +116,17 @@ pub struct Websocket {
pub time: i64,
pub data: JsonValue,
}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct ConnectionInfo {
+ pub name: String,
+ pub remote: Remote,
+ pub args: Vec<String>,
+ pub script: Vec<String>,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct ConnectionData {
+ pub info: ConnectionInfo,
+ pub maidfile: Maidfile,
+}
diff --git a/crates/maid/server/Cargo.toml b/crates/maid/server/Cargo.toml
index 180b8b7..509d0d5 100644
--- a/crates/maid/server/Cargo.toml
+++ b/crates/maid/server/Cargo.toml
@@ -7,21 +7,36 @@ repository.workspace = true
description = "๐จ Build server for maid"
[dependencies]
-clap.workspace = true
-clap-verbosity-flag.workspace = true
-env_logger.workspace = true
+tar.workspace = true
log.workspace = true
+uuid.workspace = true
+clap.workspace = true
flate2.workspace = true
+colored.workspace = true
+env_logger.workspace = true
+clap-verbosity-flag.workspace = true
+global_placeholders.workspace = true
# make workspace
+home = "0.5.5"
warp = "0.3.6"
+bytes = "1.5.0"
ntapi = "0.4.1"
libc = "0.2.149"
winapi = "0.3.9"
+anyhow = "1.0.75"
+serde = "1.0.188"
chrono = "0.4.31"
+futures = "0.3.28"
bollard = "0.15.0"
+termcolor = "1.3.0"
macros-rs = "0.4.0"
+indicatif = "0.17.7"
+tokio-util = "0.7.9"
tungstenite = "0.20.1"
serde_json = "1.0.107"
+futures-core = "0.3.28"
futures-util = "0.3.28"
+serde_derive = "1.0.188"
+text_placeholder = "0.5.0"
tokio = { version = "1.33.0", features = ["full"] }
\ No newline at end of file
diff --git a/crates/maid/server/src/docker/exec.rs b/crates/maid/server/src/docker/exec.rs
deleted file mode 100644
index e69de29..0000000
diff --git a/crates/maid/server/src/docker/mod.rs b/crates/maid/server/src/docker/mod.rs
index 52f443c..25fb1fe 100644
--- a/crates/maid/server/src/docker/mod.rs
+++ b/crates/maid/server/src/docker/mod.rs
@@ -1,2 +1,2 @@
-pub mod exec;
+pub mod run;
pub mod container;
\ No newline at end of file
diff --git a/crates/maid/server/src/docker/run.rs b/crates/maid/server/src/docker/run.rs
new file mode 100644
index 0000000..d6c3fd7
--- /dev/null
+++ b/crates/maid/server/src/docker/run.rs
@@ -0,0 +1,229 @@
+use crate::table;
+use crate::structs::ConnectionData;
+
+use bytes::Bytes;
+use bollard::{Docker, errors::Error};
+use macros_rs::{string, fmtstr, str};
+use warp::ws::{Message, WebSocket};
+use bollard::image::CreateImageOptions;
+use futures_util::{SinkExt, StreamExt};
+use futures_core::Stream;
+use futures_util::stream::{SplitSink, SplitStream, TryStreamExt};
+use bollard::exec::{CreateExecOptions, StartExecResults};
+use bollard::container::{Config, RemoveContainerOptions, UploadToContainerOptions, DownloadFromContainerOptions};
+use std::default::Default;
+use warp::hyper::Body;
+use std::sync::Arc;
+use tokio::sync::Mutex;
+use std::path::PathBuf;
+use text_placeholder::Template;
+use flate2::{Compression, write::GzEncoder};
+use std::io::Write;
+
+pub async fn concat_byte_stream<S>(s: S) -> Result<Vec<u8>, Error>
+where
+ S: Stream<Item = Result<Bytes, Error>>,
+{
+ s.try_fold(Vec::new(), |mut acc, chunk| async move {
+ acc.extend_from_slice(&chunk[..]);
+ Ok(acc)
+ })
+ .await
+}
+
+// add error handling to all the unwraps
+pub async fn exec(tx: SplitSink<WebSocket, Message>, mut rx: SplitStream<WebSocket>, docker: Docker) -> Result<(), Box<dyn std::error::Error + 'static>> {
+ let mut parsed: Option<ConnectionData> = None;
+ let tx_ref = Arc::new(Mutex::new(tx));
+
+ while parsed.is_none() {
+ if let Some(result) = rx.next().await {
+ let msg = result.unwrap();
+ match serde_json::from_str::<ConnectionData>(msg.to_str().unwrap()) {
+ Ok(value) => {
+ parsed = Some(value);
+ }
+ Err(err) => {
+ eprintln!("Failed to deserialize JSON: {:?}", err);
+ }
+ }
+ }
+ }
+
+ let parsed = parsed.unwrap();
+ let name = &parsed.info.name;
+
+ println!("creating container for task [{name}]");
+ docker
+ .create_image(
+ Some(CreateImageOptions {
+ from_image: str!(parsed.info.remote.image.clone()),
+ ..Default::default()
+ }),
+ None,
+ None,
+ )
+ .for_each(|msg| {
+ let tx_ref = Arc::clone(&tx_ref);
+
+ async move {
+ let msg = msg.as_ref().expect("Failed to get CreateImageInfo");
+ let formatted = format!(
+ "{} {}",
+ msg.status.clone().unwrap_or_else(|| string!("Waiting")),
+ msg.progress.clone().unwrap_or_else(|| string!(""))
+ );
+
+ let mut tx_lock = tx_ref.lock().await;
+ tx_lock.send(Message::text(
+ serde_json::to_string(&serde_json::json!({
+ "level": "docker",
+ "time": chrono::Utc::now().timestamp_millis(),
+ "data": { "message": formatted },
+ }))
+ .unwrap(),
+ ))
+ .await;
+ }
+ })
+ .await;
+
+ let config = Config {
+ image: Some(parsed.info.remote.image),
+ tty: Some(true),
+ ..Default::default()
+ };
+
+ let id = docker.create_container::<&str, String>(None, config).await?.id;
+ println!("created container");
+
+ docker.start_container::<String>(&id, None).await?;
+ println!("started container");
+
+ let tx_ref = Arc::clone(&tx_ref);
+ let mut tx_lock = tx_ref.lock().await;
+
+ tx_lock.send(Message::text(
+ serde_json::to_string(&serde_json::json!({
+ "level": "success",
+ "time": chrono::Utc::now().timestamp_millis(),
+ "data": { "binary": true },
+ }))
+ .unwrap(),
+ ))
+ .await
+ .unwrap();
+
+ if let Some(result) = rx.next().await {
+ println!("received message: binary");
+
+ let msg = result.unwrap();
+ fn bytes_to_body(bytes: &[u8]) -> Body {
+ Body::from(bytes.to_vec())
+ }
+
+ // note: this `Result` may be an `Err` variant, which should be handled
+ // help: use `let _ = ...` to ignore the resulting value
+ docker.upload_to_container(&id, Some(UploadToContainerOptions{ path: "/opt", ..Default::default() }), bytes_to_body(&msg.as_bytes())).await;
+ println!("wrote tarfile to container");
+ }
+
+ let dependencies = match &parsed.maidfile.tasks[&parsed.info.name].depends {
+ Some(deps) => {
+ let mut dep_script: Vec<String> = vec![];
+ for item in deps.iter() {
+ dep_script.push(
+ parsed.maidfile.tasks[item]
+ .script
+ .as_array()
+ .map(|arr| {
+ arr.iter()
+ .map(|val| val.as_str().unwrap_or_default())
+ .collect::<Vec<_>>()
+ .join("\n")
+ })
+ .unwrap_or_default()
+ );
+ };
+ dep_script.join("\n")
+ }
+ None => { string!("") }
+ };
+
+ // move common things such as structs and helpers to seperate crate
+ let table = table::create(parsed.maidfile.clone(), &parsed.info.args, PathBuf::new().join("/opt"));
+ let script = Template::new_with_placeholder(str!(parsed.info.script.join("\n")), "%{", "}").fill_with_hashmap(&table);
+ let dependencies = Template::new_with_placeholder(str!(dependencies), "%{", "}").fill_with_hashmap(&table);
+
+ let exec = docker
+ .create_exec(
+ &id,
+ CreateExecOptions {
+ attach_stdout: Some(true),
+ attach_stderr: Some(true),
+ cmd: Some(vec![str!(parsed.info.remote.shell), "-c", fmtstr!("cd /opt && touch script.sh && echo '{dependencies}\n{script}' > script.sh && chmod +x script.sh && ./script.sh")]),
+ ..Default::default()
+ },
+ )
+ .await?
+ .id;
+
+ if let StartExecResults::Attached { mut output, .. } = docker.start_exec(&exec, None).await? {
+ tx_lock.send(Message::text(
+ serde_json::to_string(&serde_json::json!({
+ "level": "build",
+ "time": chrono::Utc::now().timestamp_millis(),
+ "data": { "message": "waiting for build to finish..." },
+ }))
+ .unwrap(),
+ ))
+ .await
+ .unwrap();
+
+ while let Some(Ok(msg)) = output.next().await {
+ if !parsed.info.remote.silent {
+ let parsed = format!("{msg}");
+ if parsed != "" {
+ tx_lock.send(Message::text(
+ serde_json::to_string(&serde_json::json!({
+ "level": "build",
+ "time": chrono::Utc::now().timestamp_millis(),
+ "data": { "message": parsed.trim() },
+ }))
+ .unwrap(),
+ ))
+ .await
+ .unwrap();
+ }
+ }
+ }
+ }
+
+ let res = docker.download_from_container(&id, Some(DownloadFromContainerOptions{ path: fmtstr!("/opt/{}", parsed.info.remote.pull.clone()) }));
+ let bytes = concat_byte_stream(res).await?;
+ let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
+
+ encoder.write_all(&bytes)?;
+ let compressed_data = encoder.finish()?;
+
+ tx_lock.send(Message::binary(compressed_data)).await.unwrap();
+ println!("sent message: binary, from [{}]", parsed.info.remote.pull);
+
+ tx_lock.send(Message::text(
+ serde_json::to_string(&serde_json::json!({
+ "level": "success",
+ "time": chrono::Utc::now().timestamp_millis(),
+ "data": { "done": true, "message": "" },
+ }))
+ .unwrap(),
+ ))
+ .await
+ .unwrap();
+ println!("sent message: [done]");
+
+
+ println!("deleted old container");
+ // delete container if socket closed
+ docker.remove_container(&id, Some(RemoveContainerOptions { force: true, ..Default::default() })).await?;
+ Ok(())
+}
diff --git a/crates/maid/server/src/globals.rs b/crates/maid/server/src/globals.rs
index e69de29..e39512f 100644
--- a/crates/maid/server/src/globals.rs
+++ b/crates/maid/server/src/globals.rs
@@ -0,0 +1,5 @@
+use global_placeholders::init;
+
+pub fn init() {
+ init!("maid.temp_dir", "/usr/tmp/maid");
+}
diff --git a/crates/maid/server/src/helpers/file.rs b/crates/maid/server/src/helpers/file.rs
new file mode 100644
index 0000000..498eb20
--- /dev/null
+++ b/crates/maid/server/src/helpers/file.rs
@@ -0,0 +1,65 @@
+use crate::helpers;
+
+use flate2::{read::GzDecoder, write::GzEncoder, Compression};
+use global_placeholders::global;
+use macros_rs::crashln;
+use std::{fs::write, fs::File, path::PathBuf};
+use tar::{Archive, Builder};
+use uuid::Uuid;
+
+fn append_to_tar(builder: &mut Builder<GzEncoder<File>>, path: &String) -> Result<(), std::io::Error> {
+ let pathbuf = PathBuf::from(path);
+
+ if pathbuf.is_file() {
+ builder.append_path(&pathbuf)?;
+ } else if pathbuf.is_dir() {
+ builder.append_dir_all(&pathbuf, &pathbuf)?;
+ }
+ Ok(())
+}
+
+pub fn remove_tar(file: &String) {
+ if let Err(_) = std::fs::remove_file(file) {
+ crashln!("Unable to remove temporary archive. does it exist?");
+ }
+}
+
+pub fn read_tar(archive: &Vec<u8>) -> Result<String, std::io::Error> {
+ if !helpers::Exists::folder(global!("maid.temp_dir")).unwrap() {
+ std::fs::create_dir_all(global!("maid.temp_dir")).unwrap();
+ log::info!("created maid temp dir");
+ }
+
+ let file_name = format!("{}/{}.tgz", global!("maid.temp_dir"), Uuid::new_v4());
+ write(&file_name, archive)?;
+
+ Ok(file_name)
+}
+
+pub fn unpack_tar(path: &String) -> std::io::Result<()> {
+ let archive = File::open(&path)?;
+ let tar = GzDecoder::new(archive);
+ let mut archive = Archive::new(tar);
+
+ archive.unpack(".")
+}
+
+pub fn write_tar(files: &Vec<String>) -> Result<String, std::io::Error> {
+ if !helpers::Exists::folder(global!("maid.temp_dir")).unwrap() {
+ std::fs::create_dir_all(global!("maid.temp_dir")).unwrap();
+ log::info!("created maid temp dir");
+ }
+
+ let file_name = format!("{}/{}.tgz", global!("maid.temp_dir"), Uuid::new_v4());
+ let archive = File::create(&file_name)?;
+ let enc = GzEncoder::new(archive, Compression::default());
+ let mut tar = Builder::new(enc);
+
+ log::info!("compressing to {}", &file_name);
+ for path in files {
+ append_to_tar(&mut tar, path)?;
+ log::info!("{} {:?}", helpers::string::add_icon(), path);
+ }
+
+ Ok(file_name)
+}
diff --git a/crates/maid/server/src/helpers/logger.rs b/crates/maid/server/src/helpers/logger.rs
new file mode 100644
index 0000000..992efb1
--- /dev/null
+++ b/crates/maid/server/src/helpers/logger.rs
@@ -0,0 +1,25 @@
+#[macro_export]
+macro_rules! log {
+ ($level:expr, $($arg:tt)*) => {
+ let level_colors: std::collections::HashMap<&str, (&str, &str)> = [
+ ("fatal", ("FATAL", "bright red")),
+ ("error", ("ERROR", "red")),
+ ("warning", ("WARN", "yellow")),
+ ("success", ("SUCCESS", "green")),
+ ("notice", ("NOTICE", "bright blue")),
+ ("info", ("INFO", "cyan")),
+ ("debug", ("DEBUG", "magenta")),
+ ]
+ .iter()
+ .cloned()
+ .collect();
+
+ match level_colors.get($level) {
+ Some((level_text, color_func)) => {
+ let level_text = level_text.color(color_func.to_string());
+ println!("{} {}", level_text, format_args!($($arg)*).to_string())
+ }
+ None => println!("Unknown log level: {}", $level),
+ }
+ };
+}
diff --git a/crates/maid/server/src/helpers/mod.rs b/crates/maid/server/src/helpers/mod.rs
index 265a13a..34eaa3e 100644
--- a/crates/maid/server/src/helpers/mod.rs
+++ b/crates/maid/server/src/helpers/mod.rs
@@ -1,2 +1,15 @@
+use anyhow::Error;
+use macros_rs::str;
+use std::path::Path;
+
+pub struct Exists;
+impl Exists {
+ pub fn folder(dir_name: String) -> Result<bool, Error> { Ok(Path::new(str!(dir_name)).is_dir()) }
+ pub fn file(file_name: String) -> Result<bool, Error> { Ok(Path::new(str!(file_name)).exists()) }
+}
+
pub mod os;
-pub mod format;
\ No newline at end of file
+pub mod file;
+pub mod format;
+pub mod string;
+pub mod logger;
\ No newline at end of file
diff --git a/crates/maid/server/src/helpers/string.rs b/crates/maid/server/src/helpers/string.rs
new file mode 100644
index 0000000..b02f4af
--- /dev/null
+++ b/crates/maid/server/src/helpers/string.rs
@@ -0,0 +1,17 @@
+use colored::{ColoredString, Colorize};
+use std::path::Path;
+
+pub fn seperator() -> ColoredString { ":".white() }
+pub fn arrow_icon() -> ColoredString { "ยป".white() }
+pub fn add_icon() -> ColoredString { "+".green() }
+pub fn cross_icon() -> ColoredString { "โ".red() }
+pub fn check_icon() -> ColoredString { "โ".green() }
+
+pub fn path_to_str(path: &Path) -> &'static str { Box::leak(String::from(path.to_string_lossy()).into_boxed_str()) }
+
+pub fn trim_start_end(value: &str) -> &str {
+ let mut chars = value.chars();
+ chars.next();
+ chars.next_back();
+ chars.as_str()
+}
diff --git a/crates/maid/server/src/main.rs b/crates/maid/server/src/main.rs
index 880331f..881ef78 100644
--- a/crates/maid/server/src/main.rs
+++ b/crates/maid/server/src/main.rs
@@ -1,5 +1,8 @@
mod helpers;
mod docker;
+mod globals;
+mod structs;
+mod table;
use futures_util::{SinkExt, StreamExt};
use macros_rs::{fmtstr, ternary};
@@ -11,6 +14,8 @@ use std::env;
#[tokio::main]
async fn main() {
+ globals::init();
+
let port = 3500;
let token = "test_token".to_string();
@@ -19,7 +24,8 @@ async fn main() {
let gateway = warp::path!("ws" / "gateway").and(warp::ws()).map(|ws: warp::ws::Ws| {
ws.on_upgrade(|websocket| async {
- let (mut tx, mut rx) = websocket.split();
+ let (mut tx, rx) = websocket.split();
+ let socket = Docker::connect_with_socket_defaults().unwrap();
tx.send(Message::text(
serde_json::to_string(&serde_json::json!({
@@ -32,22 +38,7 @@ async fn main() {
.await
.unwrap();
- tx.send(Message::binary(tokio::fs::read("../testing/test.tgz").await.unwrap())).await.unwrap();
- tx.send(Message::text(
- serde_json::to_string(&serde_json::json!({
- "level": "success",
- "time": chrono::Utc::now().timestamp_millis(),
- "data": { "done": true, "message": "" },
- }))
- .unwrap(),
- ))
- .await
- .unwrap();
-
- while let Some(result) = rx.next().await {
- let message = result.unwrap();
- println!("received message: {:?}", message);
- }
+ docker::run::exec(tx, rx, socket).await.unwrap();
})
});
@@ -61,9 +52,7 @@ async fn health_handler() -> Result<impl Reply, Infallible> {
let uptime = helpers::format::duration(helpers::os::uptime());
let version = format!("Docker v{} (build {})", &info.version.clone().unwrap(), &info.git_commit.clone().unwrap());
-
- println!("{:#?}", info.clone());
-
+
Ok(warp::reply::json(&serde_json::json!({
"version": {
"data": format!("v{}", env!("CARGO_PKG_VERSION")),
diff --git a/crates/maid/server/src/structs.rs b/crates/maid/server/src/structs.rs
index e69de29..aba6238 100644
--- a/crates/maid/server/src/structs.rs
+++ b/crates/maid/server/src/structs.rs
@@ -0,0 +1,90 @@
+use serde_derive::{Deserialize, Serialize};
+use serde_json::Value;
+use std::collections::BTreeMap;
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct ConnectionInfo {
+ pub name: String,
+ pub args: Vec<String>,
+ pub remote: Remote,
+ pub script: Vec<String>,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct ConnectionData {
+ pub info: ConnectionInfo,
+ pub maidfile: Maidfile,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct Maidfile {
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub import: Option<Vec<String>>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub env: Option<BTreeMap<String, Value>>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub project: Option<Project>,
+ pub tasks: BTreeMap<String, Tasks>,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct Project {
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub name: Option<String>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub version: Option<String>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub server: Option<Server>, // wip
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct Server {
+ pub address: Address, // wip
+ pub token: String, // wip
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct Address {
+ pub host: String,
+ pub port: i64,
+ pub ssl: bool,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct Tasks {
+ pub script: Value,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub hide: Option<bool>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub path: Option<String>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub info: Option<String>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub cache: Option<Cache>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub remote: Option<Remote>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub depends: Option<Vec<String>>,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct Cache {
+ pub path: String,
+ pub target: Vec<String>,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct CacheConfig {
+ pub target: Vec<String>,
+ pub hash: String,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct Remote {
+ pub push: Vec<String>,
+ pub pull: String,
+ pub image: String,
+ pub shell: String,
+ pub silent: bool,
+ pub exclusive: bool,
+}
diff --git a/crates/maid/server/src/table.rs b/crates/maid/server/src/table.rs
new file mode 100644
index 0000000..8001057
--- /dev/null
+++ b/crates/maid/server/src/table.rs
@@ -0,0 +1,71 @@
+use crate::helpers;
+use crate::structs::Maidfile;
+
+use colored::Colorize;
+use macros_rs::{errorln, str, ternary};
+use serde_json::{json, Value};
+use std::path::PathBuf;
+use std::{collections::BTreeMap, collections::HashMap, env};
+use text_placeholder::Template;
+
+pub fn create(values: Maidfile, args: &Vec<String>, project: PathBuf) -> HashMap<&str, &str> {
+ let mut table = HashMap::new();
+ let empty_env: BTreeMap<String, Value> = BTreeMap::new();
+
+ table.insert("os.platform", env::consts::OS);
+ table.insert("os.arch", env::consts::ARCH);
+
+ log::info!("{} os.platform: '{}'", helpers::string::add_icon(), env::consts::OS.yellow());
+ log::info!("{} os.arch: '{}'", helpers::string::add_icon(), env::consts::ARCH.yellow());
+
+ match env::current_dir() {
+ Ok(path) => {
+ table.insert("dir.current", helpers::string::path_to_str(&path));
+ log::info!("{} dir.current: '{}'", helpers::string::add_icon(), helpers::string::path_to_str(&path).yellow());
+ }
+ Err(err) => {
+ log::warn!("{err}");
+ errorln!("Current directory could not be added as script variable.");
+ }
+ }
+
+ match home::home_dir() {
+ Some(path) => {
+ table.insert("dir.home", helpers::string::path_to_str(&path));
+ log::info!("{} dir.home: '{}'", helpers::string::add_icon(), helpers::string::path_to_str(&path).yellow());
+ }
+ None => {
+ errorln!("Home directory could not be added as script variable.");
+ }
+ }
+
+ let project_root = helpers::string::path_to_str(&project);
+ table.insert("dir.project", project_root);
+ log::info!("{} dir.project: '{}'", helpers::string::add_icon(), project_root.yellow());
+
+ for (pos, arg) in args.iter().enumerate() {
+ log::info!("{} arg.{pos}: '{}'", helpers::string::add_icon(), arg.yellow());
+ table.insert(str!(format!("arg.{pos}")), arg);
+ }
+
+ let user_env = match &values.env {
+ Some(env) => env.iter(),
+ None => empty_env.iter(),
+ };
+
+ for (key, value) in user_env {
+ let value_formatted = ternary!(
+ value.to_string().starts_with("\""),
+ helpers::string::trim_start_end(str!(Template::new_with_placeholder(&value.to_string(), "%{", "}").fill_with_hashmap(&table))).replace("\"", "\\\""),
+ str!(Template::new_with_placeholder(&value.to_string(), "%{", "}").fill_with_hashmap(&table)).replace("\"", "\\\"")
+ );
+
+ env::set_var(key, value_formatted.clone());
+ log::info!("{} env.{key}: '{}'", helpers::string::add_icon(), value_formatted.yellow());
+ table.insert(str!(format!("env.{}", key.clone())), str!(value_formatted));
+ }
+
+ log::trace!("{}", json!({ "env": table }));
+
+ return table;
+}
diff --git a/scripts/build.toml b/scripts/build.toml
index 7defecf..1ee5bd0 100644
--- a/scripts/build.toml
+++ b/scripts/build.toml
@@ -1,24 +1,44 @@
-# hidden tasks (can also hide with `hide = true`)
-[tasks._build_macos]
+# task intended to run on remote only
+[tasks.build_all]
+info = "build all"
script = [
-"cargo build --release",
-"mv target/release/exact-maid build/maid",
-"zip build/maid_%{env.VERSION}_darwin_amd64.zip build/maid",
-"rm build/maid",
+ # install packages
+ "apt-get update",
+ "apt-get install zip mingw-w64 -y",
+ "mkdir build",
+
+ # build linux (x86_64)
+ "cargo zigbuild -r -p maid",
+ "mv target/release/maid build/maid",
+ "zip build/maid_%{env.VERSION}_linux_amd64.zip build/maid",
+ "rm build/maid",
+
+ # build windows (x86_64)
+ "cargo zigbuild -r -p maid --target x86_64-pc-windows-gnu",
+ "mv target/x86_64-pc-windows-gnu/release/maid.exe build/maid.exe",
+ "zip build/maid_%{env.VERSION}_windows_amd64.zip build/maid.exe",
+ "rm build/maid.exe",
+
+ # build macos (x86_64)
+ "cargo zigbuild -r -p maid --target x86_64-apple-darwin",
+ "mv target/x86_64-apple-darwin/release/maid build/maid",
+ "zip build/maid_%{env.VERSION}_darwin_amd64.zip build/maid",
+ "rm build/maid",
+
+ # build macos (aarch64)
+ "cargo zigbuild -r -p maid --target aarch64-apple-darwin",
+ "mv target/aarch64-apple-darwin/release/maid build/maid",
+ "zip build/maid_%{env.VERSION}_darwin_arm.zip build/maid",
+ "rm build/maid",
+
+ # post build
+ "ls -sh build",
]
-[tasks._build_linux]
-script = [
-"cargo build --release --target x86_64-unknown-linux-musl",
-"mv target/release/exact-maid build/maid",
-"zip build/maid_%{env.VERSION}_linux_amd64.zip build/maid",
-"rm build/maid",
-]
-
-[tasks._build_windows]
-script = [
-"cargo build --release --target x86_64-pc-windows-gnu",
-"mv target/release/exact-maid build/maid",
-"zip build/maid_%{env.VERSION}_windows_amd64.zip build/maid",
-"rm build/maid",
-]
+[tasks.build_all.remote]
+silent = false
+exclusive = true
+shell = "/bin/bash"
+image = "messense/cargo-zigbuild:latest"
+push = ["crates", "Cargo.toml", "Cargo.lock"]
+pull = "build"
--
GitLab