From 490f74109bb30451ccea63148e272e62399de127 Mon Sep 17 00:00:00 2001 From: theMackabu Date: Mon, 2 Sep 2024 18:54:49 -0700 Subject: [PATCH] implement caching system --- Cargo.lock | 116 ++++++++---- Cargo.toml | 11 +- src/config.rs | 2 +- src/globals.rs | 13 +- src/helpers.rs | 61 ++++--- src/main.rs | 389 +++++++++-------------------------------- src/routes.rs | 218 +++++++++++++++++++---- src/routes/grammar.peg | 6 +- src/routes/parse.rs | 43 +++-- 9 files changed, 437 insertions(+), 422 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 905b210..378902a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -588,21 +588,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bit-set" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" -dependencies = [ - "bit-vec", -] - -[[package]] -name = "bit-vec" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" - [[package]] name = "bitflags" version = "1.3.2" @@ -923,6 +908,20 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "dashmap" +version = "6.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "804c8821570c3f8b70230c2ba75ffa5c0f9a4189b9a432b6656c536712acae28" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "data-encoding" version = "2.6.0" @@ -1048,16 +1047,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "fancy-regex" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7493d4c459da9f84325ad297371a6b2b8a162800873a22e3b6b6512e61d18c05" -dependencies = [ - "bit-set", - "regex", -] - [[package]] name = "fastrand" version = "1.9.0" @@ -1119,6 +1108,21 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" +[[package]] +name = "futures" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.30" @@ -1126,6 +1130,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -1205,9 +1210,11 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ + "futures-channel", "futures-core", "futures-io", "futures-macro", + "futures-sink", "futures-task", "memchr", "pin-project-lite", @@ -1692,6 +1699,15 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + [[package]] name = "matches" version = "0.1.10" @@ -2165,7 +2181,7 @@ dependencies = [ "rand", "rand_chacha", "rand_xorshift", - "regex-syntax", + "regex-syntax 0.8.4", "unarray", ] @@ -2261,8 +2277,17 @@ checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" dependencies = [ "aho-corasick", "memchr", - "regex-automata", - "regex-syntax", + "regex-automata 0.4.7", + "regex-syntax 0.8.4", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", ] [[package]] @@ -2273,7 +2298,7 @@ checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.8.4", ] [[package]] @@ -2282,6 +2307,12 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + [[package]] name = "regex-syntax" version = "0.8.4" @@ -2516,6 +2547,15 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.23" @@ -2541,7 +2581,8 @@ dependencies = [ "chrono", "colored", "const-hex", - "fancy-regex", + "dashmap", + "futures", "global_placeholders", "home", "macros-rs", @@ -2567,6 +2608,7 @@ dependencies = [ "tracing", "tracing-bunyan-formatter", "tracing-subscriber", + "walkdir", ] [[package]] @@ -3237,10 +3279,14 @@ version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" dependencies = [ + "matchers", "nu-ansi-term", + "once_cell", + "regex", "sharded-slab", "smallvec", "thread_local", + "tracing", "tracing-core", "tracing-log 0.2.0", ] @@ -3418,6 +3464,16 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" diff --git a/Cargo.toml b/Cargo.toml index 4bd0cf5..273af6e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ home = "0.5.9" mime = "0.3.17" toml = "0.8.19" redis = "0.24.0" -tokio = "1.40.0" +tokio = { version = "1.40.0", features = ["fs"] } anyhow = "1.0.86" askama = "0.12.1" colored = "2.1.0" @@ -27,7 +27,7 @@ macros-rs = "1.2.5" termcolor = "1.4.1" smartstring = "1.0.1" serde_json = "1.0.127" -tracing-subscriber = "0.3.18" +tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } tracing-bunyan-formatter = "0.3.9" global_placeholders = "0.1.0" ron = "0.8.1" @@ -36,6 +36,9 @@ pest = "2.7.11" pest_derive = "2.7.11" md-5 = "0.10.6" const-hex = "1.12.0" +dashmap = "6.0.1" +walkdir = "2.5.0" +futures = "0.3.30" [dependencies.mongodb] default-features = false @@ -51,10 +54,6 @@ features = [ ] version = "0.5.1" -[dependencies.regex] -package = "fancy-regex" -version = "0.12.0" - [dependencies.reqwest] features = ["blocking"] version = "0.11.27" diff --git a/src/config.rs b/src/config.rs index 6db4520..b783f91 100644 --- a/src/config.rs +++ b/src/config.rs @@ -18,7 +18,7 @@ pub fn read() -> Config { database: None, workers: vec!["app.routes".into()], settings: Settings { - cache: string!(".script/cache"), + cache: string!(".script"), address: string!("127.0.0.1"), port: 3500, }, diff --git a/src/globals.rs b/src/globals.rs index b45ac1e..d0cecac 100644 --- a/src/globals.rs +++ b/src/globals.rs @@ -5,11 +5,18 @@ use std::fs::create_dir_all; pub fn init() { let config = config::read(); + let cache_dir = format!("{}/cache", config.settings.cache); - if !folder_exists!(&config.settings.cache) { - create_dir_all(&config.settings.cache).unwrap(); + if !folder_exists!(&cache_dir) { + create_dir_all(&cache_dir).unwrap(); tracing::info!("created cached dir"); } - init!("dirs.cache", format!("{}{{}}.route", config.settings.cache)); + init!("base.cache", config.settings.cache); + init!("base.handler", format!("{}/handler", config.settings.cache)); + + init!("dirs.cache", format!("{}/cache{{}}.route", config.settings.cache)); + init!("dirs.handler", format!("{}/handler{{}}.route", config.settings.cache)); + init!("dirs.cache.index", format!("{}/routes.toml", config.settings.cache)); + init!("dirs.cache.hash", format!("{}/hashes.toml", config.settings.cache)); } diff --git a/src/helpers.rs b/src/helpers.rs index 744b6c2..5056af1 100644 --- a/src/helpers.rs +++ b/src/helpers.rs @@ -2,41 +2,54 @@ pub mod file; use actix_web::http::StatusCode; use mongodb::{bson::doc, sync::Database}; -use regex::{Captures, Regex}; use rhai::{plugin::EvalAltResult, Engine, ParseError, AST}; +use std::collections::HashMap; pub mod prelude { pub use super::file::*; } -pub fn rm_first(s: &str) -> &str { - let mut chars = s.chars(); - chars.next(); - chars.as_str() -} - -pub fn convert_to_format(input: &str) -> String { - let re = Regex::new(r"\.(\w+)").unwrap(); - let input = super::replace_chars(input); - format!("_route_{}", re.replace_all(&input.replace("/", "_"), |captures: &Captures| format!("__d{}", rm_first(&captures[0])))) -} +pub fn replace_chars(input: &str) -> String { + let replacements = HashMap::from([ + ('#', "_fhas"), + (':', "_fcol"), + ('-', "_fdas"), + ('@', "_fats"), + ('!', "_fexl"), + ('&', "_famp"), + ('^', "_fcar"), + ('~', "_ftil"), + ]); -pub fn route_to_fn(input: &str) -> String { - let input = super::replace_chars(input); - let re = Regex::new(r#"\{([^{}\s]+)\}"#).unwrap(); - let re_dot = Regex::new(r"\.(\w+)").unwrap(); + let mut result = String::with_capacity(input.len()); - let result = re.replace_all(&input, |captures: ®ex::Captures| { - let content = captures.get(1).map_or("", |m| m.as_str()); - format!("_arg_{content}") - }); + for c in input.chars() { + if let Some(replacement) = replacements.get(&c) { + result.push_str(replacement); + } else { + result.push(c); + } + } - format!( - "_route_fmt_{}", - re_dot.replace_all(&result.replace("/", "_"), |captures: &Captures| format!("__d{}", rm_first(&captures[0]))) - ) + result } +// pub fn route_to_fn(input: &str) -> String { +// let input = super::replace_chars(input); +// let re = Regex::new(r#"\{([^{}\s]+)\}"#).unwrap(); +// let re_dot = Regex::new(r"\.(\w+)").unwrap(); +// +// let result = re.replace_all(&input, |captures: ®ex::Captures| { +// let content = captures.get(1).map_or("", |m| m.as_str()); +// format!("_arg_{content}") +// }); +// +// format!( +// "_route_fmt_{}", +// re_dot.replace_all(&result.replace("/", "_"), |captures: &Captures| format!("__d{}", rm_first(&captures[0]))) +// ) +// } + pub fn collection_exists(d: &Database, name: &String) -> Result> { let filter = doc! { "name": &name }; diff --git a/src/main.rs b/src/main.rs index 46692e9..6770bea 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,25 +8,23 @@ mod structs; use helpers::prelude::*; use modules::prelude::*; +use routes::prelude::*; use structs::{config::*, template::*}; use mime::Mime; -use regex::{Captures, Error, Regex}; use reqwest::blocking::Client as ReqwestClient; use smartstring::alias::String as SmString; use std::{collections::HashMap, fs}; use tracing_bunyan_formatter::{BunyanFormattingLayer, JsonStorageLayer}; -use tracing_subscriber::{filter::LevelFilter, prelude::*}; +use tracing_subscriber::{filter::EnvFilter, prelude::*}; use rhai::{packages::Package, plugin::*, Dynamic, Engine, Map, Scope}; use rhai_fs::FilesystemPackage; use rhai_url::UrlPackage; use macros_rs::{ - exp::ternary, fmt::{crashln, string}, - obj::lazy_lock, os::set_env_sync, }; @@ -36,15 +34,6 @@ use actix_web::{ App, HttpRequest, HttpResponse, HttpServer, Responder, }; -lazy_lock! { - static R_INDEX: Result = Regex::new(r"index\s*\{"); - static R_ERR: Result = Regex::new(r"(\b\d{3})\s*\{"); - static R_DOT: Result = Regex::new(r"\.(\w+)\((.*?)\)\s*\{"); - static R_WILD: Result = Regex::new(r"\*\s*\{|wildcard\s*\{"); - static R_FN: Result = Regex::new(r"([\w#:\-@!&^~]+)\((.*?)\)\s*\{"); - static R_SLASH: Result = Regex::new(r"(?m)\/(?=.*\((.*?)\)\s*\{[^{]*$)"); -} - pub fn response(data: String, content_type: String, status_code: i64) -> (String, ContentType, StatusCode) { let content_type = match content_type.as_str() { "xml" => ContentType::xml(), @@ -61,6 +50,22 @@ pub fn response(data: String, content_type: String, status_code: i64) -> (String (data, content_type, helpers::convert_status(status_code)) } +fn parse_bool(s: &str) -> bool { + match s.trim().to_lowercase().as_str() { + "true" | "1" | "yes" | "on" => true, + _ => false, + } +} + +fn parse_slash(s: &str) -> String { + let parts: Vec<&str> = s.splitn(3, '/').collect(); + if parts.len() > 1 { + format!("{}/{}", parts[0], parts[1]) + } else { + s.to_string() + } +} + pub fn proxy(url: String) -> (String, ContentType, StatusCode) { let client = ReqwestClient::new(); let response = match client.get(url).send() { @@ -78,87 +83,8 @@ pub fn proxy(url: String) -> (String, ContentType, StatusCode) { } } -fn parse_bool(s: &str) -> bool { - match s.trim().to_lowercase().as_str() { - "true" | "1" | "yes" | "on" => true, - _ => false, - } -} - -fn match_route(route_template: &str, placeholders: &[&str], url: &str) -> Option> { - let mut matched_placeholders = Vec::new(); - - let route_segments: Vec<&str> = route_template.split('/').collect(); - let url_segments: Vec<&str> = url.split('/').collect(); - - if route_segments.len() != url_segments.len() { - return None; - } - - for (route_segment, url_segment) in route_segments.iter().zip(url_segments.iter()) { - if let Some(placeholder_value) = match_segment(route_segment, url_segment, placeholders) { - if !placeholder_value.is_empty() { - matched_placeholders.push(placeholder_value); - } - } else { - return None; - } - } - - Some(matched_placeholders) -} - -pub fn replace_chars(input: &str) -> String { - let replacements = HashMap::from([ - ('#', "__fhas"), - (':', "__fcol"), - ('-', "__fdas"), - ('@', "__fats"), - ('!', "__fexl"), - ('&', "__famp"), - ('^', "__fcar"), - ('~', "__ftil"), - ]); - - let mut result = String::with_capacity(input.len()); - - for c in input.chars() { - if let Some(replacement) = replacements.get(&c) { - result.push_str(replacement); - } else { - result.push(c); - } - } - - result -} - -fn match_segment(route_segment: &str, url_segment: &str, placeholders: &[&str]) -> Option { - if route_segment.starts_with('{') && route_segment.ends_with('}') { - let placeholder = &route_segment[1..route_segment.len() - 1]; - if placeholders.contains(&placeholder) { - Some(url_segment.to_string()) - } else { - None - } - } else if route_segment == url_segment { - Some("".to_string()) - } else { - let route_parts: Vec<&str> = route_segment.split('.').collect(); - let url_parts: Vec<&str> = url_segment.split('.').collect(); - if route_parts.len() == url_parts.len() && route_parts.last() == url_parts.last() { - match_segment(route_parts[0], url_parts[0], placeholders) - } else { - None - } - } -} - async fn handler(req: HttpRequest, config: Data) -> impl Responder { - let url = match req.uri().to_string().strip_prefix("/") { - Some(url) => url.to_string(), - None => req.uri().to_string(), - }; + let url = req.uri().to_string(); macro_rules! send { ($response:expr) => {{ @@ -174,13 +100,20 @@ async fn handler(req: HttpRequest, config: Data) -> impl Responder { }}; } - if url.as_str() == "favicon.ico" { - // remove this - return HttpResponse::Ok().body(""); + macro_rules! error { + ($err:expr) => {{ + let body = Message { + error: "Function Not Found", + code: StatusCode::NOT_FOUND.as_u16(), + message: format!("Have you created the {url} route?"), + note: "You can add * {} or 404 {} routes as well", + }; + + tracing::error!(err = string!($err), "Error finding route"); + send!((body.render().unwrap(), ContentType::html(), StatusCode::NOT_FOUND)) + }}; } - let mut routes: HashMap> = HashMap::new(); - let filename = &config.workers.get(0).unwrap(); let fs_pkg = FilesystemPackage::new(); let url_pkg = UrlPackage::new(); @@ -192,11 +125,6 @@ async fn handler(req: HttpRequest, config: Data) -> impl Responder { let mut engine = Engine::new(); let mut scope = Scope::new(); - let path = match url.as_str() { - "" => "_route_index".to_string(), - _ => helpers::convert_to_format(&url.to_string()), - }; - fs_pkg.register_into_engine(&mut engine); url_pkg.register_into_engine(&mut engine); @@ -265,224 +193,73 @@ async fn handler(req: HttpRequest, config: Data) -> impl Responder { Err(err) => crashln!("Error reading script file: {}\n{}", filename.to_string_lossy(), err), }; - routes::parse::try_parse(&contents); - - let has_error_page = R_ERR.as_ref().unwrap().is_match(&contents).unwrap(); - let has_wildcard = R_WILD.as_ref().unwrap().is_match(&contents).unwrap(); - let has_index = R_INDEX.as_ref().unwrap().is_match(&contents).unwrap(); - - fn parse_cfg(cfg_str: &str) -> HashMap { - cfg_str - .split(',') - .filter_map(|pair| { - let mut parts = pair.split('='); - if let (Some(key), Some(value)) = (parts.next(), parts.next()) { - Some((key.trim().to_string(), value.trim().trim_matches('"').to_string())) - } else { - None + // move error handling here + routes::parse::try_parse(&contents).await; + + let route = match Route::get(&parse_slash(&url)).await { + Ok(route) => { + let mut matched_url = url.to_owned(); + + let cfg = match route.cfg { + Some(cfg) => cfg, + None => HashMap::new(), + }; + + for (item, val) in cfg { + // convert to enum Cfg::Wildcard, etc + match item.as_str() { + "wildcard" => { + if parse_slash(&url) == route.route && parse_bool(&val) { + matched_url = parse_slash(&url); + break; + } + } + _ => {} } - }) - .collect() - } - - let contents = { - let pattern = r#"\{([^{}\s]+)\}"#; - let pattern_rm_config = r#",?\s*cfg\([^)]*\)"#; - let pattern_combine = r#"(?m)^_route/(.*)\n(.*?)\((.*?)\)"#; - - let re = Regex::new(pattern).unwrap(); - let re_combine = Regex::new(pattern_combine).unwrap(); - let re_rm_config = Regex::new(pattern_rm_config).unwrap(); - - let result = re.replace_all(&contents, |captures: ®ex::Captures| { - let content = captures.get(1).map_or("", |m| m.as_str()); - format!("_arg_{content}") - }); - - let result = re_rm_config.replace_all(&result, ""); - let result = result.replace("#[route(\"", "_route").replace("\")]", ""); - - let new_result_fmt = re_combine - .replace_all(&result, |captures: ®ex::Captures| { - let path = captures.get(1).map_or("", |m| m.as_str()); - let args = captures.get(3).map_or("", |m| m.as_str()); - - if args != "" { - let r_path = Regex::new(r"(?m)_arg_(\w+)").unwrap(); - let key = r_path.replace_all(&path, |captures: ®ex::Captures| { - let key = captures.get(1).map_or("", |m| m.as_str()); - format!("{{{key}}}") - }); - - routes.insert(string!(key), args.split(",").map(|s| s.to_string().replace(" ", "")).collect()); - format!("fmt_{path}({args})") - } else { - routes.insert(string!(path), vec![]); - format!("{path}()") - } - }) - .into_owned(); - - std::mem::drop(result); - - new_result_fmt - }; - - // cache contents until file change - let contents = { - let result = R_SLASH.as_ref().unwrap().replace_all(&contents, "_").to_string(); - let result = R_INDEX.as_ref().unwrap().replace_all(&result, "index() {").to_string(); - let result = R_ERR.as_ref().unwrap().replace_all(&result, "error_$1() {").to_string(); - - let pattern_route = r#"(?m)^(?!_error)(?!_wildcard)(?!_index)(?!fmt_)(.*?)\((.*?)\)\s*\{"#; - let re_route = Regex::new(pattern_route).unwrap(); + } - for captures in re_route.captures_iter(&contents) { - let path = captures.unwrap().get(1).map_or("", |m| m.as_str()); - routes.insert(string!(path.replace("_", "/")), vec![]); + match Route::get(&matched_url).await { + Ok(route) => route, + Err(err) => error!(err), + } } - - let result = R_DOT - .as_ref() - .unwrap() - .replace_all(&result, |captures: &Captures| format!("__d{}", helpers::rm_first(&captures[0]))) - .to_string(); - - let result = R_FN - .as_ref() - .unwrap() - .replace_all(&result, |captures: &Captures| { - let fmt = replace_chars(&captures[0]); - format!("fn _route_{fmt}") - }) - .to_string(); - - ternary!(has_wildcard, R_WILD.as_ref().unwrap().replace_all(&result, "fn _wildcard() {").to_string(), result) + Err(err) => error!(err), }; - let contents = { - let slash = Regex::new(r"\$\((.*?)\)").unwrap(); - slash.replace_all(&contents, |caps: ®ex::Captures| format!("${{{}}}", &caps[1])).to_string() - }; - - let mut ast = match engine.compile(&contents) { + let mut ast = match engine.compile(route.construct_fn()) { Ok(ast) => ast, - Err(err) => helpers::error(&engine, &path, err), + // fix fn name error + Err(err) => helpers::error(&engine, &url, err), }; ast.set_source(filename.to_string_lossy().to_string()); - if url.as_str() == "" && has_index { - send!(engine.call_fn::<(String, ContentType, StatusCode)>(&mut scope, &ast, "_route_index", ()).unwrap()); + let fn_name = match route.fn_name.as_str() { + "/" => "/index", + name => name, }; - fn extract_context(contents: String, err: String) -> Vec<(String, String)> { - let re = Regex::new(r"line (\d+)").unwrap(); - - if let Some(captures) = re.captures(&err).unwrap() { - if let Some(num) = captures.get(1) { - if let Ok(line_number) = num.as_str().parse::() { - let lines: Vec<&str> = contents.lines().collect(); - let start_line = line_number.saturating_sub(3); - let end_line = (line_number + 4).min(lines.len()); - - return lines[start_line..end_line] - .iter() - .enumerate() - .map(|(i, line)| (format!("{:>4}", start_line + i + 1), line.to_string())) - .collect::>(); - } - } - } - - vec![] - } - - for (route, args) in routes { - let url = url.clone(); - let args: Vec<&str> = args.iter().map(AsRef::as_ref).collect(); - - if url.as_str() == route { - match engine.call_fn::<(String, ContentType, StatusCode)>(&mut scope, &ast, helpers::convert_to_format(&url.to_string()), ()) { - Ok(response) => send!(response), - Err(err) => { - let body = ServerError { - error: err.to_string().replace("\n", "
"), - context: extract_context(contents, err.to_string()), - }; - - return HttpResponse::build(StatusCode::INTERNAL_SERVER_ERROR).content_type(ContentType::html()).body(body.render().unwrap()); - } - } - } + let args = match route.args.to_owned() { + Some(args) => args, + None => vec![], + }; - match match_route(&route, &args, url.as_str()) { - Some(data) => match engine.call_fn::<(String, ContentType, StatusCode)>(&mut scope, &ast, helpers::route_to_fn(&route), data) { - Ok(response) => send!(response), - Err(err) => { - let body = ServerError { - error: err.to_string().replace("\n", "
"), - context: extract_context(contents, err.to_string()), - }; + match engine.call_fn::<(String, ContentType, StatusCode)>(&mut scope, &ast, fn_name, args) { + Ok(response) => send!(response), + Err(err) => { + let body = ServerError { + error: err.to_string().replace("\n", "
"), + context: vec![], + }; - return HttpResponse::build(StatusCode::INTERNAL_SERVER_ERROR).content_type(ContentType::html()).body(body.render().unwrap()); - } - }, - None => {} + return HttpResponse::build(StatusCode::INTERNAL_SERVER_ERROR).content_type(ContentType::html()).body(body.render().unwrap()); } - } - - // for data in route_data { - // let name = data.route; - // - // let cfg = match data.cfg { - // Some(cfg) => cfg, - // None => HashMap::new(), - // }; - // - // for (item, val) in cfg { - // match item.as_str() { - // "wildcard" => { - // if url.splitn(2, '/').next().unwrap_or(&url) == name && parse_bool(&val) { - // match engine.call_fn::<(String, ContentType, StatusCode)>(&mut scope, &ast, helpers::convert_to_format(&name), ()) { - // Ok(response) => send!(response), - // Err(err) => { - // let body = ServerError { - // error: err.to_string().replace("\n", "
"), - // context: extract_context(contents, err.to_string()), - // }; - // - // return HttpResponse::build(StatusCode::INTERNAL_SERVER_ERROR).content_type(ContentType::html()).body(body.render().unwrap()); - // } - // } - // } - // } - // _ => {} - // } - // } - // } - - if has_wildcard || has_error_page { - let (body, content_type, status_code) = engine - .call_fn::<(String, ContentType, StatusCode)>(&mut scope, &ast, ternary!(has_wildcard, "_wildcard", "_route_error_404"), ()) - .unwrap(); - - send!((body, content_type, ternary!(has_wildcard, status_code, StatusCode::NOT_FOUND))) - } else { - let body = Message { - error: "Function Not Found", - code: StatusCode::NOT_FOUND.as_u16(), - message: format!("Have you created the {url}() route?"), - note: "You can add * {} or 404 {} routes as well", - }; - - send!((body.render().unwrap(), ContentType::html(), StatusCode::NOT_FOUND)) - } + }; } #[actix_web::main] async fn main() -> std::io::Result<()> { - set_env_sync!(RUST_LOG = "info"); + set_env_sync!(RUST_LOG = "debug"); globals::init(); let config = config::read(); @@ -492,12 +269,8 @@ async fn main() -> std::io::Result<()> { .skip_fields(vec!["file", "line"].into_iter()) .expect("Unable to create logger"); - tracing_subscriber::registry() - .with(LevelFilter::from(tracing::Level::INFO)) - .with(JsonStorageLayer) - .with(formatting_layer) - .init(); - + tracing_subscriber::registry().with(EnvFilter::from_default_env()).with(JsonStorageLayer).with(formatting_layer).init(); tracing::info!(address = config.settings.address, port = config.settings.port, "server started"); + HttpServer::new(app).bind(config.get_address()).unwrap().run().await } diff --git a/src/routes.rs b/src/routes.rs index 9cd58ca..512a71b 100644 --- a/src/routes.rs +++ b/src/routes.rs @@ -1,31 +1,45 @@ pub mod parse; +use anyhow::{anyhow, Error}; use chrono::{DateTime, Duration, Utc}; +use dashmap::DashMap; use global_placeholders::global; -use macros_rs::fmt::string; +use macros_rs::{fmt::string, fs::file_exists, obj::lazy_lock}; use md5::{Digest, Md5}; use serde::{Deserialize, Serialize}; use smartstring::{LazyCompact, SmartString}; +use tokio::sync::Mutex; +use walkdir::WalkDir; + +use tokio::fs::{read, write}; use std::{ - collections::HashMap, - fs::{create_dir_all, read, write}, + collections::{HashMap, HashSet}, + fs::{create_dir_all, read_dir, remove_dir, remove_file}, + mem::take, path::{Path, PathBuf}, + sync::Arc, }; +pub type RtTime = DateTime; +pub type RtIndex = (String, Route); pub type RtData = SmartString; pub type RtArgs = Option>; pub type RtConfig = Option>; -pub type RtTime = DateTime; +pub type RtGlobalIndex = Arc>>; pub enum RtKind { - Index, + Normal, Wildcard, NotFound, - Normal, } -#[derive(Debug, Default, PartialEq, Eq, Deserialize, Serialize)] +pub struct RouteContainer { + inner: Route, + present_in_current_update: bool, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize)] pub struct Route { pub cfg: RtConfig, pub args: RtArgs, @@ -40,31 +54,147 @@ pub struct Route { pub end_pos: usize, } +lazy_lock! { + pub static ROUTES_INDEX: RtGlobalIndex = Arc::new(Mutex::new(DashMap::new())); +} + +fn match_route(route_template: &str, placeholders: &[&str], url: &str) -> Option> { + let mut matched_placeholders = Vec::new(); + + let route_segments: Vec<&str> = route_template.split('/').collect(); + let url_segments: Vec<&str> = url.split('/').collect(); + + if route_segments.len() != url_segments.len() { + return None; + } + + for (route_segment, url_segment) in route_segments.iter().zip(url_segments.iter()) { + if let Some(placeholder_value) = match_segment(route_segment, url_segment, placeholders) { + if !placeholder_value.is_empty() { + matched_placeholders.push(placeholder_value); + } + } else { + return None; + } + } + + Some(matched_placeholders) +} + +fn match_segment(route_segment: &str, url_segment: &str, placeholders: &[&str]) -> Option { + if route_segment.starts_with('{') && route_segment.ends_with('}') { + let placeholder = &route_segment[1..route_segment.len() - 1]; + if placeholders.contains(&placeholder) { + Some(url_segment.to_string()) + } else { + None + } + } else if route_segment == url_segment { + Some("".to_string()) + } else { + let route_parts: Vec<&str> = route_segment.split('.').collect(); + let url_parts: Vec<&str> = url_segment.split('.').collect(); + if route_parts.len() == url_parts.len() && route_parts.last() == url_parts.last() { + match_segment(route_parts[0], url_parts[0], placeholders) + } else { + None + } + } +} + impl Route { pub fn default() -> Self { Default::default() } + pub async fn cleanup() -> std::io::Result<()> { + let cache_dir = PathBuf::from(global!("base.cache")); + let routes = ROUTES_INDEX.lock().await; + + tracing::trace!("Cache directory: {:?}", cache_dir); + + let valid_cache_files: HashSet = routes + .iter() + .map(|item| { + let path = item.value().inner.cache.clone(); + tracing::trace!("Valid cache file: {:?}", path); + path + }) + .collect(); + + for entry in WalkDir::new(&cache_dir).into_iter().filter_map(|e| e.ok()) { + let path = entry.path().to_path_buf(); + if path.is_file() { + tracing::trace!("Checking file: {:?}", path); + + let should_keep = valid_cache_files.iter().any(|valid_path| { + let paths_match = path == *valid_path; + tracing::trace!("Comparing {:?} with {:?}: {}", path, valid_path, paths_match); + paths_match + }); + + if !should_keep { + tracing::debug!("Deleting file: {:?}", path); + remove_file(&path)?; + } else { + tracing::debug!("Keeping file: {:?}", path); + } + } else if entry.file_type().is_dir() { + if read_dir(path.to_owned())?.next().is_none() { + tracing::debug!("Removing empty directory: {:?}", path); + remove_dir(&path)?; + } + } + } + + Ok(()) + } + + pub async fn update_index(new_routes: Vec) { + let routes = ROUTES_INDEX.lock().await; + + for mut entry in routes.iter_mut() { + entry.present_in_current_update = false; + } + + for (key, value) in new_routes { + routes + .entry(key) + .and_modify(|e| { + e.inner = value.to_owned(); + e.present_in_current_update = true; + }) + .or_insert(RouteContainer { + inner: value, + present_in_current_update: true, + }); + } + + routes.retain(|_, v| v.present_in_current_update); + } + pub fn cache(&mut self, kind: RtKind) -> &Self { + let now = Utc::now(); let mut md5 = Md5::new(); let route_name = match kind { - RtKind::Index => "/index", RtKind::Wildcard => "/wildcard", RtKind::NotFound => "/not_found", RtKind::Normal => self.route.as_str(), }; + let cache_key = match kind { + RtKind::Wildcard => global!("dirs.handler", route_name), + RtKind::NotFound => global!("dirs.handler", route_name), + RtKind::Normal => global!("dirs.cache", route_name), + }; + let fn_name = match kind { - RtKind::Index => "index", RtKind::Wildcard => "wildcard", RtKind::NotFound => "not_found", RtKind::Normal => self.fn_name.as_str(), }; - let now = Utc::now(); - let cache_key = global!("dirs.cache", route_name); - self.route = route_name.into(); - self.fn_name = fn_name.into(); + self.fn_name = fn_name.replace("/", "_").replace(".", "_d").into(); md5.update(&self.route); md5.update(&self.fn_name); @@ -78,7 +208,8 @@ impl Route { return self; } - pub fn save(&mut self, kind: RtKind) { + // save functions that expired or dont exist + pub async fn save(&mut self, kind: RtKind) -> RtIndex { self.cache(kind); if let Some(parent) = self.cache.parent() { @@ -96,30 +227,58 @@ impl Route { } }; - if let Err(err) = write(self.cache.to_owned(), encoded) { + if let Err(err) = write(self.cache.to_owned(), encoded).await { tracing::error!(err = string!(err), "Error writing route"); std::process::exit(1); } + + ROUTES_INDEX.lock().await.insert( + self.hash.to_owned(), + RouteContainer { + inner: self.clone(), + present_in_current_update: true, + }, + ); + + return (self.hash.to_owned(), take(self)); } - pub fn get(key: &str) -> String { - let bytes = match read(&key) { - Ok(contents) => contents, - Err(err) => { - tracing::error!(err = string!(err), "Error reading route"); - std::process::exit(1); - } + pub async fn get(key: &str) -> Result { + let key = match key { + "/" => global!("dirs.cache", "/index"), + _ => global!("dirs.cache", key), }; - let data: Route = match ron::de::from_bytes(&bytes) { - Ok(parsed) => parsed, + let files = HashMap::from([ + ("not_found", global!("dirs.handler", "/not_found")), + ("wildcard", global!("dirs.handler", "/wildcard")), + ("server_error", global!("dirs.handler", "/internal_err")), + ]); + + let page_exists = |key| match key { + "not_found" => file_exists!(&files.get("not_found").unwrap()), + "wildcard" => file_exists!(&files.get("wildcard").unwrap()), + _ => false, + }; + + let bytes = match read(&key).await { + Ok(contents) => contents, Err(err) => { - tracing::error!(err = string!(err), "Error reading route"); - std::process::exit(1); + if page_exists("not_found") { + read(files.get("not_found").unwrap()).await? + } else if page_exists("wildcard") { + read(files.get("wildcard").unwrap()).await? + } else { + return Err(anyhow!(err)); + } } }; - let args = match data.args { + Ok(ron::de::from_bytes(&bytes)?) + } + + pub fn construct_fn(&self) -> String { + let args = match self.args.to_owned() { Some(args) => match args.len() { 0 => String::new(), 1 => args[0].to_string(), @@ -128,13 +287,10 @@ impl Route { None => "".into(), }; - format!("fn {}({args}){{{}", data.fn_name, data.fn_body) + format!("fn {}({args}){{{}}}", self.fn_name, self.fn_body) } } pub mod prelude { pub use super::Route; - pub use super::RtConfig; - pub use super::RtData; - pub use super::RtKind; } diff --git a/src/routes/grammar.peg b/src/routes/grammar.peg index bb8bd2f..96f2e37 100644 --- a/src/routes/grammar.peg +++ b/src/routes/grammar.peg @@ -1,4 +1,4 @@ -grammar = { SOI ~ (route_definition | function_def | index | not_found | wildcard)* ~ EOI } +grammar = { SOI ~ (route_definition | function_def | not_found | wildcard)* ~ EOI } route_definition = { route_attr? ~ function_def @@ -8,10 +8,6 @@ function_def = { route_name ~ ("(" ~ parameters? ~ ")")? ~ block } -index = { - "index" ~ block -} - wildcard = { "*" ~ block } diff --git a/src/routes/parse.rs b/src/routes/parse.rs index 634accf..407f25d 100644 --- a/src/routes/parse.rs +++ b/src/routes/parse.rs @@ -1,7 +1,10 @@ +use futures::future::join_all; use pest::iterators::Pair; use pest::Parser; use pest_derive::Parser; use std::collections::HashMap; +use std::future::Future; +use std::pin::Pin; #[derive(Parser)] #[grammar = "routes/grammar.peg"] @@ -93,27 +96,39 @@ fn extract_route_info(pair: Pair, input: &str) -> super::Route { route_info } -fn process_pair(pair: Pair, input: &str) { - match pair.as_rule() { - Rule::index => extract_route_info(pair, input).save(super::RtKind::Index), - Rule::route_definition => extract_route_info(pair, input).save(super::RtKind::Normal), - Rule::not_found => extract_route_info(pair, input).save(super::RtKind::NotFound), - Rule::wildcard => extract_route_info(pair, input).save(super::RtKind::Wildcard), +fn process_pair<'i>(pair: Pair<'i, Rule>, input: &'i str) -> Pin> + 'i>> { + Box::pin(async move { + let mut index: Vec<(String, super::Route)> = Vec::new(); - _ => { - for inner_pair in pair.into_inner() { - process_pair(inner_pair, input); + match pair.as_rule() { + Rule::route_definition => index.push(extract_route_info(pair, input).save(super::RtKind::Normal).await), + Rule::not_found => index.push(extract_route_info(pair, input).save(super::RtKind::NotFound).await), + Rule::wildcard => index.push(extract_route_info(pair, input).save(super::RtKind::Wildcard).await), + _ => { + for inner_pair in pair.into_inner() { + let mut inner_index = process_pair(inner_pair, input).await; + index.append(&mut inner_index); + } } } - } + + index + }) } -pub fn try_parse(input: &str) { +pub async fn try_parse(input: &str) { match RouteParser::parse(Rule::grammar, input) { Ok(pairs) => { - for pair in pairs { - process_pair(pair, input); - } + let futures: Vec<_> = pairs.into_iter().map(|pair| process_pair(pair, input)).collect(); + let results = join_all(futures).await; + let index: Vec<(String, super::Route)> = results.into_iter().flatten().collect(); + + super::Route::update_index(index).await; + + match super::Route::cleanup().await { + Ok(_) => tracing::trace!("Cache cleanup completed successfully"), + Err(err) => tracing::error!(err = err.to_string(), "Error during cache cleanup"), + }; } Err(e) => println!("Error: {}", e), } -- GitLab