+ update testdata

+ implement get_meta - refactor structs a bit
2024-06-02 00:58:28 -07:00 · 2024-06-02 00:58:28 -07:00 · a6de2133fd
commit a6de2133fd
parent 363cd11aa5
7 changed files with 241 additions and 73 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@ -1511,6 +1511,7 @@ dependencies = [
 "atoi",
 "byteorder",
 "bytes",
+ "chrono",
 "crc",
 "crossbeam-queue",
 "either",
@ -1574,6 +1575,7 @@ dependencies = [
 "sha2",
 "sqlx-core",
 "sqlx-mysql",
+ "sqlx-postgres",
 "sqlx-sqlite",
 "syn 1.0.109",
 "tempfile",
@ -1592,6 +1594,7 @@ dependencies = [
 "bitflags 2.5.0",
 "byteorder",
 "bytes",
+ "chrono",
 "crc",
 "digest",
 "dotenvy",
@ -1633,6 +1636,7 @@ dependencies = [
 "base64",
 "bitflags 2.5.0",
 "byteorder",
+ "chrono",
 "crc",
 "dotenvy",
 "etcetera",
@ -1668,6 +1672,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b244ef0a8414da0bed4bb1910426e890b19e5e9bccc27ada6b797d05c55ae0aa"
 dependencies = [
 "atoi",
+ "chrono",
 "flume",
 "futures-channel",
 "futures-core",
--- a/Cargo.toml
+++ b/Cargo.toml
@ -21,6 +21,7 @@ sqlx = { version = "0.7.4", features = [
    "sqlite",
    # "postgres",
    # "any",
+    "chrono",
    "macros",
    "runtime-tokio",
    "tls-rustls",
--- a/go_client/Containerfile
+++ b/go_client/Containerfile
@ -8,6 +8,7 @@ RUN go mod download
 COPY . .
 # RUN go build -o /app
 RUN go build
+# CMD export GOCACHE=off
 CMD go test tests/*

 # FROM docker.io/library/alpine:3.19
--- a/migrations/20240506135416_testdata.sql
+++ b/migrations/20240506135416_testdata.sql
@ -1,5 +1,6 @@
 -- Add migration script here

-INSERT INTO metadata VALUES ("bar", false, DateTime('now'), "123", 4, DateTime('now'), "customData");
+INSERT INTO metadata VALUES ("bar", false, DateTime('now'), "30d", 4, DateTime('now'), '{"foo": "customData"}');

-INSERT INTO secret_versions VALUES ("secret_data", DateTime('now'), DateTime('now'), 1, "bar");
+INSERT INTO secret_versions VALUES ("secret_data", DateTime('now'), DateTime('now'), 1, "bar");
+INSERT INTO secret_versions VALUES ("more_secret_data", DateTime('now'), datetime('now', '+30 day'), 2, "bar");
--- a/src/engines/kv.rs
+++ b/src/engines/kv.rs
@ -2,24 +2,30 @@
 #![allow(dead_code)]

 // pub mod logic; // TODO: Remove or correct errors
-pub mod http_structs;
 pub mod db_structs;
+pub mod http_structs;

 // #[cfg(test)]
 // mod tests;

-use std::{collections::HashMap, convert::Infallible};
-use serde_json;
 use crate::{
-    engines::kv::http_structs::*,
+    engines::kv::{self, http_structs::*},
    storage::DatabaseDriver,
 };
 use axum::{
-    extract::{self, Path, State}, http::StatusCode, response::IntoResponse, routing::*, Json, Router
+    extract::{self, Path, State},
+    http::StatusCode,
+    response::IntoResponse,
+    routing::*,
+    Json, Router,
 };
+
 use chrono::{DateTime, Utc};
-use log::{info, error};
+use db_structs::*;
+use log::{error, info};
+use serde_json;
 use sqlx::{error, Row};
+use std::{collections::HashMap, convert::Infallible};

 pub fn kv_router(pool: DatabaseDriver) -> Router {
    Router::new()
@ -68,7 +74,13 @@ async fn get_data(
                ("secret_path".to_string(), v.get("secret_path")),
            ]);
            let return_secret = KvSecretRes::new(KvSecretResData {
-                created_time: DateTime::parse_from_rfc3339(secret_content.get("created_time").unwrap_or(&"".to_string())).unwrap_or_default().to_utc(), // TODO
+                created_time: DateTime::parse_from_rfc3339(
+                    secret_content
+                        .get("created_time")
+                        .unwrap_or(&"".to_string()),
+                )
+                .unwrap_or_default()
+                .to_utc(), // TODO
                custom_metadata: None,
                deletion_time: None, // TODO
                destroyed: false,
@ -81,12 +93,12 @@ async fn get_data(
        Err(e) => match e {
            sqlx::Error::RowNotFound => {
                error!("{:?}", e);
-                let error_struct : ErrorStruct = ErrorStruct{err: e.to_string() };
+                let error_struct: ErrorStruct = ErrorStruct { err: e.to_string() };
                error!("{:?}", error_struct.err);
-                Ok(error_struct.into_response())    // TODO: API doesn't specify return value in case of error. Error struct correct? Else send empty secret back?
-                // let error_secret = KvSecretRes{data: None, options: None}; 
-                // Ok(Json())
-            },
+                Ok(error_struct.into_response()) // TODO: API doesn't specify return value in case of error. Error struct correct? Else send empty secret back?
+                                                 // let error_secret = KvSecretRes{data: None, options: None};
+                                                 // Ok(Json())
+            }
            _ => panic!("{:?}", e),
        },
    }
@ -99,7 +111,6 @@ async fn post_data(
 ) -> &'static str {
    // Insert Metadata first -> Else: Error because of foreign key constraint

-    
    log::debug!(
        "Secret: {}, Content: {:?}, Version: {:?}, path: {}",
        path,
@ -107,23 +118,20 @@ async fn post_data(
        payload.options,
        path
    );
-    

    let data = serde_json::to_string(&payload.data).unwrap();
    log::debug!("Received data: {:?}", data);
    let created_time = Utc::now().to_string();
-    let deletion_time = "12-12-2024 12:00:00"; // TODO: 
+    let deletion_time = "12-12-2024 12:00:00"; // TODO:
    let version = "1";
-    match sqlx::query(
-        "INSERT INTO secret_versions VALUES ($1, $2, $3, $4, $5)",
-    )
-    .bind(data)
-    .bind(created_time)
-    .bind (deletion_time)
-    .bind(version)
-    .bind(path)
-    .execute(&pool)
-    .await
+    match sqlx::query("INSERT INTO secret_versions VALUES ($1, $2, $3, $4, $5)")
+        .bind(data)
+        .bind(created_time)
+        .bind(deletion_time)
+        .bind(version)
+        .bind(path)
+        .execute(&pool)
+        .await
    {
        Ok(v) => {
            info!("{:?}", v);
@ -178,8 +186,96 @@ async fn destroy_path() -> &'static str {
    todo!("not implemented")
 }

-async fn get_meta() -> &'static str {
-    todo!("not implemented")
+async fn get_meta(
+    State(pool): State<DatabaseDriver>,
+    Path(kv_path): Path<String>,
+) -> Result<impl IntoResponse, Infallible> {
+    log::debug!("Path: {}", kv_path);
+
+    let mut metadata_res: KvMetaRes = KvMetaRes::default();
+
+    let dbmeta = sqlx::query_as::<_, DbSecretMeta>("SELECT * FROM metadata where secret_path = $1")
+        .bind(&kv_path)
+        .fetch_optional(&pool)
+        .await;
+
+    match dbmeta {
+        Ok(Some(dbmeta)) => {
+            metadata_res.data = KvMetaResData {
+                created_time: dbmeta.created_time,
+
+                // map the custom_data to a Hashmap
+                custom_metadata: dbmeta.custom_data.map(|data| {
+                    serde_json::from_str::<HashMap<String, String>>(&data)
+                        .unwrap_or_else(|_| HashMap::new())
+                }),
+
+                cas_required: dbmeta.cas_required,
+                max_versions: dbmeta.max_versions,
+                updated_time: dbmeta.updated_time,
+                delete_version_after: dbmeta.delete_version_after,
+                current_version: 0,
+                oldest_version: 0,
+                versions: HashMap::new(),
+            };
+
+            let version_data = sqlx::query_as::<_, DbSecretVersionMeta>("SELECT version_number, created_time, deletion_time FROM secret_versions WHERE secret_path = $1")
+            .bind(&kv_path)
+            .fetch_all(&pool)
+            .await;
+            log::debug!("found version_data: {:?}", version_data);
+
+            if let Ok(version_data) = version_data {
+                // 1. iterate through all version data
+                // 2. put all version numbers as keys in the hashmap. the rest of the values values should be the value
+                let mut parsed_versions: HashMap<i64, KvMetaResVersionData> = HashMap::new();
+                let now = Utc::now();
+
+                for curr_ver in version_data {
+                    let curr_num = curr_ver.version_number;
+                    let data = KvMetaResVersionData {
+                        created_time: curr_ver.created_time,
+                        deletion_time: curr_ver.deletion_time,
+                        destroyed: if curr_ver.deletion_time < now {
+                            true
+                        } else {
+                            false
+                        },
+                    };
+
+                    if metadata_res.data.current_version < curr_num {
+                        // should be the max of the available version numbers
+                        metadata_res.data.current_version = curr_num;
+                    }
+                    if metadata_res.data.oldest_version > curr_num {
+                        // should be the min of the available version numbers
+                        metadata_res.data.oldest_version = curr_num;
+                    }
+
+                    parsed_versions.insert(curr_num, data);
+                }
+
+                metadata_res.data.versions = parsed_versions;
+            }
+        }
+
+        Ok(None) => {
+            return Ok((StatusCode::BAD_REQUEST, Json("No metadata found")).into_response());
+        }
+        Err(e) => {
+            log::error!("Database error: {}", e);
+            return Ok((
+                StatusCode::INTERNAL_SERVER_ERROR,
+                Json("Internal server error"),
+            )
+                .into_response());
+        }
+    }
+
+    let json_string = serde_json::to_string(&metadata_res).unwrap();
+    log::debug!("Returning response: {}", json_string);
+
+    Ok((StatusCode::OK, Json(metadata_res)).into_response())
 }

 async fn post_meta(
--- a/src/engines/kv/db_structs.rs
+++ b/src/engines/kv/db_structs.rs
@ -1,53 +1,63 @@
-use std::collections::HashMap;
-
 use chrono::{DateTime, Utc};
-use log::*;
+use serde::Serialize;
+use sqlx::FromRow;

-#[derive(Debug)]
-#[deprecated(note = "Add Req or Res respecively if AND move to http file if intended; remove deprecation once used")]
-pub struct VersionMeta {
-    pub created_time: DateTime<Utc>,
-    pub deletion_time: Option<DateTime<Utc>>, // optional deletion time
-    pub destroyed: bool,
-}
+// #[derive(Debug)]
+// #[deprecated(note = "Add Req or Res respecively if AND move to http file if intended; remove deprecation once used")]
+// pub struct SecretMeta {
+//     pub cas_required: bool,
+//     pub created_time: DateTime<Utc>,
+//     pub current_version: i64,
+//     /// In Hashicorp:
+//     /// If not set, the backend's configured delete_version_after is used.
+//     /// Cannot be greater than the backend's delete_version_after
+//     // TODO: implement duration type
+//     pub delete_version_after: String,
+//     // TODO https://developer.hashicorp.com/vault/docs/concepts/duration-format
+//     pub max_versions: i64,
+//     pub oldest_version: i64,
+//     pub updated_time: DateTime<Utc>,
+//     /// User-provided key-value pairs that are used to describe arbitrary and version-agnostic information about a secret.
+//     pub custom_metadata: Option<HashMap<String, String>>,
+//     pub versions: Vec<VersionMeta>,
+// }

+#[derive(FromRow)]
 #[derive(Debug)]
-#[deprecated(note = "Add Req or Res respecively if AND move to http file if intended; remove deprecation once used")]
-pub struct SecretMeta {
+pub struct DbSecretMeta {
+    pub secret_path: String,
    pub cas_required: bool,
    pub created_time: DateTime<Utc>,
-    pub current_version: i64,
+    // Consider: implement duration type
+    // https://developer.hashicorp.com/vault/docs/concepts/duration-format
+
    /// In Hashicorp:
    /// If not set, the backend's configured delete_version_after is used.
    /// Cannot be greater than the backend's delete_version_after
-    // TODO: implement duration type
-    pub delete_version_after: String,
-    // TODO https://developer.hashicorp.com/vault/docs/concepts/duration-format
+    pub delete_version_after: Option<String>,
+    ///In Hashicorp:
+    /// The number of versions to keep per key.
+    /// If not set, the backend’s configured max version is used.
+    /// Once a key has more than the configured allowed versions, 
+    /// the oldest version will be permanently deleted.
    pub max_versions: i64,
-    pub oldest_version: i64,
    pub updated_time: DateTime<Utc>,
    /// User-provided key-value pairs that are used to describe arbitrary and version-agnostic information about a secret.
-    pub custom_metadata: Option<HashMap<String, String>>,
-    pub versions: Vec<VersionMeta>,
+    
+    pub custom_data: Option<String>,
+
+    // TODO: AS HASHMAP
+    // pub custom_data: Option<HashMap<String, String>>,
+
+    // pub current_version: i64,
+    // pub oldest_version: i64,
 }

-impl Default for SecretMeta {
-    /// Use [Serde field defaults](https://serde.rs/attr-default.html) instead
-    /// TODO: remove this function
-    fn default() -> Self {
-        warn!("DEPRECATED: Default of SecretMeta was used, to be removed");
-
-        let current = Utc::now();
-        SecretMeta {
-            cas_required: false,
-            created_time: current,
-            current_version: 1,
-            delete_version_after: "24h00m00s".to_string(),
-            max_versions: 10,
-            oldest_version: 1,
-            updated_time: current,
-            custom_metadata: None,
-            versions: vec![],
-        }
-    }
-}
+#[derive(Serialize,Debug, FromRow)]
+/// Metadata concerning a specific secret version
+/// contained by [KvMetaRes]
+pub struct DbSecretVersionMeta {    
+    pub version_number: i64,
+    pub created_time: DateTime<Utc>,
+    pub deletion_time: DateTime<Utc>,
+}
--- a/src/engines/kv/http_structs.rs
+++ b/src/engines/kv/http_structs.rs
@ -1,10 +1,12 @@
-use axum::{body::Body, http::{Response, StatusCode}, response::IntoResponse, Error};
+use axum::{
+    body::Body,
+    http::{Response, StatusCode},
+    response::IntoResponse,
+};
 use chrono::{DateTime, Utc};
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;

-use crate::common::HttpError;
-
 pub type KvSecretData = HashMap<String, String>;

 // This file contains structures for serializing HTTP Responses (Res) and deserializing Requests (Req) for the KV engine
@ -48,11 +50,63 @@ impl KvSecretRes {

 #[derive(Serialize)]
 pub struct ErrorStruct {
-    pub err: String
+    pub err: String,
 }
 impl ErrorStruct {
    pub fn into_response(self) -> Response<Body> {
        let body = self.err;
        (StatusCode::NOT_FOUND, body).into_response()
    }
-}
+}
+
+#[derive(Serialize, Debug)]
+/// HTTP Response to Reading a Secret metadata
+/// Container of [`KvMetaResData`]
+pub struct KvMetaRes {
+    pub data: KvMetaResData,
+}
+
+impl Default for KvMetaRes {
+    fn default() -> Self {
+        let now = Utc::now();
+        Self {
+            data: KvMetaResData {
+                cas_required: false,
+                created_time: now,
+                delete_version_after: Some("".to_string()),
+                max_versions: 0,
+                updated_time: now,
+                custom_metadata: Some(HashMap::new()),
+                current_version: 0,
+                oldest_version: 0,
+
+                versions: HashMap::new(),
+            },
+        }
+    }
+}
+
+#[derive(Serialize, Debug)]
+/// Metadata concerning a specific secret version
+/// contained by [KvMetaRes]
+pub struct KvMetaResVersionData {
+    pub created_time: DateTime<Utc>,
+    pub deletion_time: DateTime<Utc>,
+    pub destroyed: bool,
+}
+
+#[derive(Serialize, Debug)]
+/// contained by [KvMetaRes]
+pub struct KvMetaResData {
+    pub cas_required: bool,
+    pub created_time: DateTime<Utc>,
+    pub current_version: i64,
+    pub delete_version_after: Option<String>,
+    pub max_versions: i64,
+    pub oldest_version: i64,
+    pub updated_time: DateTime<Utc>,
+    // pub custom_metadata: Option<String>, // TODO as hashmap
+    pub custom_metadata: Option<HashMap<String, String>>,
+    pub versions: HashMap<i64, KvMetaResVersionData>,
+    // the key to a version is the version number
+}