From 3a27c7e38980fd7b8cfffa69b916dfb8e7f462fd Mon Sep 17 00:00:00 2001 From: C0ffeeCode Date: Wed, 1 May 2024 17:20:07 +0200 Subject: [PATCH 1/4] Fix (storage): Avoid bug creating file for in-mem sqlite --- src/storage.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storage.rs b/src/storage.rs index 123afb8..4a46653 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -8,7 +8,7 @@ pub async fn create_pool(db_url: String) -> AnyPool { sqlx::any::install_default_drivers(); // Create SQLite database file if it does not exist - if db_url.starts_with("sqlite:") { + if db_url.starts_with("sqlite:") && db_url != ("sqlite::memory:") { let path = db_url.replace("sqlite:", ""); if !Path::new(&path).exists() { warn!("Sqlite database does not exist, creating file {}", path); From be4e698d61b0ab2954d5b410bd9079b450c4be6c Mon Sep 17 00:00:00 2001 From: C0ffeeCode Date: Wed, 1 May 2024 20:04:21 +0200 Subject: [PATCH 2/4] tada --- .gitignore | 4 +++- go_client/Containerfile | 1 + go_client/tests/secret_test.go | 3 ++- migrations/20240428160456_init.sql | 2 ++ src/storage.rs | 5 +++++ 5 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index a76edee..2d669dc 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,10 @@ .vs/ .vscode/ .idea/ +.env *.pdf target/ go_client/openapi.json -crates/storage-sled/sled_db \ No newline at end of file +crates/storage-sled/sled_db +test.db diff --git a/go_client/Containerfile b/go_client/Containerfile index 6cbe75f..90ca186 100644 --- a/go_client/Containerfile +++ b/go_client/Containerfile @@ -7,6 +7,7 @@ RUN go mod download COPY . . # RUN go build -o /app +RUN go build CMD go test tests/* # FROM docker.io/library/alpine:3.19 diff --git a/go_client/tests/secret_test.go b/go_client/tests/secret_test.go index af50611..21cc769 100644 --- a/go_client/tests/secret_test.go +++ b/go_client/tests/secret_test.go @@ -13,7 +13,8 @@ import ( var client *vault.Client var ctx context.Context -var mountpath = "" +// Apparently used as a default if mountpath is an empty string (client library) +var mountpath = "/kv-v2" func TestMain(m *testing.M) { ctx = context.Background() diff --git a/migrations/20240428160456_init.sql b/migrations/20240428160456_init.sql index 91d683d..95f17bd 100644 --- a/migrations/20240428160456_init.sql +++ b/migrations/20240428160456_init.sql @@ -4,3 +4,5 @@ CREATE TABLE secret_engines ( mount_point TEXT PRIMARY KEY NOT NULL, engine_type TEXT NOT NULL ); + +INSERT INTO secret_engines (mount_point, engine_type) VALUES ('/kv-v2', 'kv'); diff --git a/src/storage.rs b/src/storage.rs index 4a46653..062e055 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -23,5 +23,10 @@ pub async fn create_pool(db_url: String) -> AnyPool { .await .expect(&db_url); + sqlx::migrate!() + .run(&pool) + .await + .expect("Failed to apply migrations"); + pool } From 2c355ef75d1b14aa15d19e8e498dbfdb3a5fc543 Mon Sep 17 00:00:00 2001 From: C0ffeeCode Date: Wed, 1 May 2024 20:05:18 +0200 Subject: [PATCH 3/4] engine routing experiment --- src/engines.rs | 93 +++++++++++++++++--------------------------------- 1 file changed, 31 insertions(+), 62 deletions(-) diff --git a/src/engines.rs b/src/engines.rs index 91d4ab9..d57a55a 100644 --- a/src/engines.rs +++ b/src/engines.rs @@ -1,77 +1,46 @@ pub mod kv; -use crate::engines::kv::logic::body_to_json; -use crate::engines::kv::structs::KvSecret; use axum::{ - extract::{Path, Request}, - http::StatusCode, - middleware::{self, Next}, + body::Body, + extract::{Request, State}, response::Response, - routing::*, Router, }; use log::*; use sqlx::{Any, Pool}; pub fn secrets_router(pool: Pool) -> Router> { - // Router::new().layer(map_request(handler)) - - Router::new() - .route("/:mount_path/data/:kv_path", post(baz)) - .with_state(pool) + Router::new().fallback(map_mount_points).with_state(pool) } -/// Routing handler for path "/v1/kv-v2/data/foo" -/// expects payload as JSON, prints payload into struct +async fn map_mount_points(State(_pool): State>, req: Request) -> Response { + // let mount_path = mount_path.trim_start_matches('/'); + // debug!("Mount path: {}", mount_path); + debug!("Mount path: {:?}", req); -async fn baz(Path(mount_path): Path, Path(kv_path): Path, body: String) -> String { - let mut body_json = body_to_json(body); + let mut mount_path: Vec<&str> = req.uri().path().split("/").collect(); - // TODO: If version field provided during a read, the value at the version number will be returned - let secret: KvSecret = KvSecret { - data: body_json.to_string(), - // content: body_json["data"]["password1"].take().to_string(), - version: body_json["data"]["version"].take().as_i64(), - }; - log::debug!( - "Secret: {}, Content: {}, Version: {:?}, path: {}", - kv_path, - secret.data, - secret.version, - mount_path, - ); - String::from("RoutingTest baz successful") + // let a = sqlx::query!("SELECT * FROM secret_engines WHERE path = $1", mount_path.join("/")) + // .fetch_one(&pool) + // .await + // .expect("Failed to fetch secret engine"); + + let tada = vec!["a/b/c/d".to_string(), "/a/b/c".to_string(), "/kv-v2".into()]; + + // Find longest matching existing mount path for the request + for _ in 1..mount_path.len() { + if tada.contains(&mount_path.join("/")) { + trace!( + "Mount path {} found for route request: {}", + mount_path.join("/"), + req.uri().path() + ); + + break; + } else { + mount_path.pop(); + } + } + + return Response::new(Body::from(format!("Mount path:"))); } - -// async fn handler(Host(hostname): Host, request: Request) -> &'static str { -// TODO: Find a solution for this mess -// async fn handler(request: Request) -> Result, StatusCode> { -// // let path: Vec<&str> = request.uri().path().split('/').clone().collect(); -// // log::info!("path, {:?}", path[1]); - -// let root = service_fn(|req: Request| async move { -// let res = Response::new("Hello, World!".to_string()); -// Ok::<_, Infallible>(res) -// }); -// let root = BoxService::new(root); - -// let mut routes = vec!["/abc", "/def"]; -// routes.sort_unstable_by(|a, b| a.len().cmp(&b.len())); - -// let mut app = Router::new(); -// app.as_service().call(request).await.unwrap(); - -// // match path[1] { -// // "test" => { -// // log::info!("test route"); -// // // TODO: Nest another Router here -// // return Ok(Request::new(Body::empty())); -// // } -// // _ => { -// // log::info!("default"); -// // return Err(StatusCode::NOT_FOUND); -// // } -// // } - -// Err(StatusCode::IM_A_TEAPOT) -// } From 0a0091c44ff4d570a74715d24f8a9c2657b920a7 Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 1 May 2024 20:17:16 +0200 Subject: [PATCH 4/4] + secret & metadata migration --- .gitignore | 1 + migrations/20240501152243_KvSecret.sql | 28 ++++++++++++++++++++++++++ src/engines/kv/structs.rs | 2 +- 3 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 migrations/20240501152243_KvSecret.sql diff --git a/.gitignore b/.gitignore index 2d669dc..8cac0ad 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ target/ go_client/openapi.json crates/storage-sled/sled_db test.db +src/storage/database.db diff --git a/migrations/20240501152243_KvSecret.sql b/migrations/20240501152243_KvSecret.sql new file mode 100644 index 0000000..190d7f5 --- /dev/null +++ b/migrations/20240501152243_KvSecret.sql @@ -0,0 +1,28 @@ +-- Add migration script here + +CREATE TABLE metadata ( + secret_path TEXT PRIMARY KEY NOT NULL, + + cas_required INTEGER NOT NULL, -- no bool datatype in sqlite + created_time TIMESTAMP NOT NULL, + delete_version_after TEXT, -- Maybe NOT NULL + max_versions INTEGER NOT NULL, + -- current_version INTEGER NOT NULL, + -- oldest_version INTEGER NOT NULL, + updated_time TIMESTAMP NOT NULL, + custom_data TEXT +); + +CREATE TABLE secret_versions ( + secret_data TEXT NOT NULL, + + created_time TIMESTAMP NOT NULL, + deletion_time TIMESTAMP, + + version_number INTEGER NOT NULL DEFAULT 0, + secret_path TEXT NOT NULL, + PRIMARY KEY (secret_path, version_number), + FOREIGN KEY (secret_path) REFERENCES metadata(secret_path) +); + +CREATE INDEX idx_secret_versions_secret_path ON secret_versions (secret_path); diff --git a/src/engines/kv/structs.rs b/src/engines/kv/structs.rs index 444cec4..563c2eb 100644 --- a/src/engines/kv/structs.rs +++ b/src/engines/kv/structs.rs @@ -1,6 +1,6 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; -use std::{collections::HashMap, hash::Hash, vec}; +use std::{collections::HashMap, vec}; #[derive(Serialize, Deserialize, Debug)] pub struct KvSecret {