Store_map and some improvements

.devcontainer/Dockerfile

@@ -1,5 +1,5 @@
 # DevContainer image
-FROM rust:1.84-slim
+FROM rust:1.85-slim
 RUN \
     adduser --system --disabled-password --shell /bin/bash --home /home/vscode vscode && \
     # install docker
@@ -15,7 +15,7 @@ RUN \
 RUN \
     # dev setup
     apt update && \
-    apt-get install sudo jq git bash-completion graphviz -y && \
+    apt-get install sudo jq git bash-completion graphviz rsync -y && \
     usermod -aG sudo vscode && \
     echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers && \
     echo '. /etc/bash_completion' >> /home/vscode/.bashrc && \
@@ -35,4 +35,4 @@ RUN \
     rustup component add llvm-tools-preview && \
     # rust crate structure diagram
     cargo install cargo-modules
-CMD ["bash", "-c", "sudo rm /var/run/docker.pid; sudo dockerd"]
+CMD ["bash", "-c", "sudo rm /var/run/docker.pid; sudo dockerd"]

.devcontainer/devcontainer.json

@@ -12,7 +12,7 @@
 		"LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}",
 		"RUST_BACKTRACE": "1"
 	},
-	"postStartCommand": "docker system prune -fa && docker volume prune -f",
+	"postStartCommand": "mkdir -p tests/.tmp && docker system prune -fa && docker volume prune -f",
 	"hostRequirements": {
 		"cpus": 2,
 		"memory": "8gb",
@@ -39,4 +39,4 @@
 			]
 		}
 	}
-}
+}

.devcontainer/gpu/Dockerfile

@@ -16,7 +16,7 @@ RUN \
 RUN \
     # dev setup
     apt update && \
-    apt-get install build-essential sudo jq git bash-completion -y && \
+    apt-get install build-essential sudo jq git bash-completion rsync -y && \
     echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers && \
     echo '. /etc/bash_completion' >> /root/.bashrc && \
     echo 'export PS1="\[\e[32;1m\]\u\[\e[m\]@\[\e[34;1m\]\H\[\e[m\]:\[\e[33;1m\]\w\[\e[m\]$ "' >> /root/.bashrc && \
@@ -34,4 +34,4 @@ RUN apt-get update && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs
     apt-get clean
 
 ENV PATH=${PATH}:/root/.local/bin
-CMD ["bash", "-c", "sudo rm /var/run/docker.pid; sudo dockerd"]
+CMD ["bash", "-c", "sudo rm /var/run/docker.pid; sudo dockerd"]

.devcontainer/gpu/devcontainer.json

@@ -12,7 +12,7 @@
 		"LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}",
 		"RUST_BACKTRACE": "1"
 	},
-	"postStartCommand": "docker system prune -fa && docker volume prune -f && apt update && apt install fish -y && chsh -s /usr/bin/fish",
+	"postStartCommand": "mkdir -p tests/.tmp && docker system prune -fa && docker volume prune -f",
 	"hostRequirements": {
 		"cpus": 2,
 		"memory": "8gb",
@@ -37,5 +37,8 @@
 				"streetsidesoftware.code-spell-checker" // Catch spelling errors in docs
 			]
 		}
+	},
+	"features": {
+		"ghcr.io/meaningful-ooo/devcontainer-features/fish:2": {} // Fish shell
 	}
-}
+}

.github/workflows/tests.yaml

@@ -11,7 +11,7 @@ jobs:
       - name: Install Rust + components
         uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
-          toolchain: 1.84
+          toolchain: 1.85
           components: rustfmt,clippy
       - name: Install Rust code coverage
         uses: taiki-e/install-action@cargo-llvm-cov
@@ -22,4 +22,6 @@ jobs:
       - name: Run integration tests w/ coverage report
         env:
           RUST_BACKTRACE: full
-        run: cargo llvm-cov -- --nocapture
+        run: |
+          mkdir -p tests/.tmp
+          cargo llvm-cov -- --nocapture

.gitignore

@@ -4,5 +4,5 @@ Cargo.lock
 target
 lcov.info
 *.tar.gz
-test_store
-*.dot
+*.dot
+tests/.tmp

.vscode/launch.json

@@ -103,4 +103,4 @@
             "cwd": "${workspaceFolder}",
         },
     ]
-}
+}

.vscode/settings.json

@@ -1,6 +1,7 @@
 {
     "editor.formatOnPaste": false,
     "editor.formatOnSave": true,
+    "files.insertFinalNewline": true,
     "notebook.formatOnSave.enabled": true,
     "editor.rulers": [
         100
@@ -17,4 +18,4 @@
     "gitlens.showWelcomeOnInstall": false,
     "gitlens.showWhatsNewAfterUpgrades": false,
     "lldb.consoleMode": "evaluate",
-}
+}

Cargo.toml

@@ -32,9 +32,7 @@ glob = "0.3.1" # recursive walk of filesystem in filestore
 heck = "0.5.0" # strings to snake_case
 names = "0.14.0" # random name generator
 regex = "1.11.0" # complex pattern matching in strings
-serde = { version = "1.0.210", features = [
-    "derive",
-] } # serialization/deserialization to/from filestore
+serde = { version = "1.0.210", features = ["derive"] } # serialization/deserialization to/from filestore
 serde_json = "1.0.137" # JSON in sharing memory with local docker orchestrator
 serde_yaml = "0.9.34" # YAML in filestore
 sha2 = "0.10.8" # checksums based on SHA256 
@@ -76,7 +74,6 @@ float_arithmetic = { level = "allow", priority = 127 }                 # allow f
 host_endian_bytes = { level = "allow", priority = 127 }                # allow to_ne_bytes / from_ne_bytes
 impl_trait_in_params = { level = "allow", priority = 127 }             # impl in params ok
 implicit_return = { level = "allow", priority = 127 }                  # missing return ok
-indexing_slicing = { level = "allow", priority = 127 }                 # allow since guaranteed and error handling is overkill
 inline_asm_x86_intel_syntax = { level = "allow", priority = 127 }      # intel syntax ok
 integer_division = { level = "allow", priority = 127 }                 # allow discarding remainder
 little_endian_bytes = { level = "allow", priority = 127 }              # allow to_le_bytes / from_le_bytes

cspell.json

@@ -61,4 +61,4 @@
         "peaceiris",
         "tintinweb"
     ]
-}
+}

src/crypto.rs

@@ -1,13 +1,26 @@
-use crate::error::Result;
+use crate::{
+    error::Result,
+    model::{Blob, BlobKind},
+    util::get,
+};
 use serde_yaml;
 use sha2::{Digest as _, Sha256};
-use std::{collections::BTreeMap, fs::File, io::Read, path::Path};
-
+use std::{
+    collections::{BTreeMap, HashMap},
+    fs::File,
+    hash::RandomState,
+    io::Read,
+    path::{Path, PathBuf},
+};
 /// Evaluate checksum hash of streamed data i.e. chunked buffers.
 ///
 /// # Errors
 ///
 /// Will return error if unable to read from stream.
+#[expect(
+    clippy::indexing_slicing,
+    reason = "Reading less than 0 is impossible."
+)]
 pub fn hash_stream(stream: &mut impl Read) -> Result<String> {
     const BUFFER_SIZE: usize = 8 << 10; // 8KB chunks to match with page size typically found
     let mut hash = Sha256::new();
@@ -22,12 +35,10 @@ pub fn hash_stream(stream: &mut impl Read) -> Result<String> {
 
     Ok(format!("{:x}", hash.finalize()))
 }
-
 /// Evaluate checksum hash of raw data in memory.
 pub fn hash_buffer(buffer: impl AsRef<[u8]>) -> String {
     format!("{:x}", Sha256::digest(buffer.as_ref()))
 }
-
 /// Evaluate checksum hash of a file.
 ///
 /// # Errors
@@ -36,8 +47,7 @@ pub fn hash_buffer(buffer: impl AsRef<[u8]>) -> String {
 pub fn hash_file(filepath: impl AsRef<Path>) -> Result<String> {
     hash_stream(&mut File::open(filepath)?)
 }
-
-/// Evaluate checksum hash of a folder.
+/// Evaluate checksum hash of a directory.
 ///
 /// # Errors
 ///
@@ -64,3 +74,21 @@ pub fn hash_dir(dirpath: impl AsRef<Path>) -> Result<String> {
 
     Ok(hash_buffer(serde_yaml::to_string(&summary)?))
 }
+/// Evaluate checksum hash of a blob.
+///
+/// # Errors
+///
+/// Will return error if hashing fails on file or directory.
+pub fn hash_blob(
+    namespace_lookup: &HashMap<String, PathBuf, RandomState>,
+    blob: Blob,
+) -> Result<Blob> {
+    let blob_path = get(namespace_lookup, &blob.location.namespace)?.join(&blob.location.path);
+    Ok(Blob {
+        checksum: match blob.kind {
+            BlobKind::File => hash_file(blob_path)?,
+            BlobKind::Directory => hash_dir(blob_path)?,
+        },
+        ..blob
+    })
+}

src/error.rs

@@ -1,7 +1,5 @@
 use bollard::errors::Error as BollardError;
-use colored::Colorize as _;
 use glob;
-use regex;
 use serde_json;
 use serde_yaml;
 use std::{
@@ -16,8 +14,16 @@ pub type Result<T> = result::Result<T, OrcaError>;
 /// Possible errors you may encounter.
 #[derive(Error, Debug)]
 pub(crate) enum Kind {
-    #[error("File `{}` already exists.", path.to_string_lossy().bright_cyan())]
-    FileExists { path: PathBuf },
+    #[error("Received an empty response when attempting to load the alternate container image file: {path}.")]
+    EmptyResponseWhenLoadingContainerAltImage { path: PathBuf },
+    #[error("Out of generated random names.")]
+    GeneratedNamesOverflow,
+    #[error("Path missing a file or directory name: {path}.")]
+    InvalidPath { path: PathBuf },
+    #[error("An invalid datetime was set for pod result for pod job (hash: {pod_job_hash}).")]
+    InvalidPodResultTerminatedDatetime { pod_job_hash: String },
+    #[error("Key '{key}' was not found in map.")]
+    KeyMissing { key: String },
     #[error("No annotation found for `{name}:{version}` {class}.")]
     NoAnnotationFound {
         class: String,
@@ -26,30 +32,22 @@ pub(crate) enum Kind {
     },
     #[error("No known container names.")]
     NoContainerNames,
-    #[error("Out of generated random names.")]
-    GeneratedNamesOverflow,
     #[error("No corresponding pod run found for pod job (hash: {pod_job_hash}).")]
     NoMatchingPodRun { pod_job_hash: String },
-    #[error("An invalid datetime was set for pod result for pod job (hash: {pod_job_hash}).")]
-    InvalidPodResultTerminatedDatetime { pod_job_hash: String },
-    #[error("Received an empty response when attempting to load the alternate container image file: {path}.")]
-    EmptyResponseWhenLoadingContainerAltImage { path: PathBuf },
     #[error("No tags found in provided container alternate image: {path}.")]
     NoTagFoundInContainerAltImage { path: PathBuf },
     #[error(transparent)]
+    BollardError(#[from] BollardError),
+    #[error(transparent)]
     GlobPatternError(#[from] glob::PatternError),
     #[error(transparent)]
-    RegexError(#[from] regex::Error),
+    IoError(#[from] io::Error),
     #[error(transparent)]
-    SerdeYamlError(#[from] serde_yaml::Error),
+    PathPrefixError(#[from] path::StripPrefixError),
     #[error(transparent)]
     SerdeJsonError(#[from] serde_json::Error),
     #[error(transparent)]
-    IoError(#[from] io::Error),
-    #[error(transparent)]
-    BollardError(#[from] BollardError),
-    #[error(transparent)]
-    PathPrefixError(#[from] path::StripPrefixError),
+    SerdeYamlError(#[from] serde_yaml::Error),
 }
 /// A stable error API interface.
 #[derive(Error, Debug)]
@@ -71,24 +69,31 @@ impl Display for OrcaError {
         write!(f, "{}", self.kind)
     }
 }
+impl From<BollardError> for OrcaError {
+    fn from(error: BollardError) -> Self {
+        Self {
+            kind: Kind::BollardError(error),
+        }
+    }
+}
 impl From<glob::PatternError> for OrcaError {
     fn from(error: glob::PatternError) -> Self {
         Self {
             kind: Kind::GlobPatternError(error),
         }
     }
 }
-impl From<regex::Error> for OrcaError {
-    fn from(error: regex::Error) -> Self {
+impl From<io::Error> for OrcaError {
+    fn from(error: io::Error) -> Self {
         Self {
-            kind: Kind::RegexError(error),
+            kind: Kind::IoError(error),
         }
     }
 }
-impl From<serde_yaml::Error> for OrcaError {
-    fn from(error: serde_yaml::Error) -> Self {
+impl From<path::StripPrefixError> for OrcaError {
+    fn from(error: path::StripPrefixError) -> Self {
         Self {
-            kind: Kind::SerdeYamlError(error),
+            kind: Kind::PathPrefixError(error),
         }
     }
 }
@@ -99,24 +104,10 @@ impl From<serde_json::Error> for OrcaError {
         }
     }
 }
-impl From<io::Error> for OrcaError {
-    fn from(error: io::Error) -> Self {
-        Self {
-            kind: Kind::IoError(error),
-        }
-    }
-}
-impl From<BollardError> for OrcaError {
-    fn from(error: BollardError) -> Self {
-        Self {
-            kind: Kind::BollardError(error),
-        }
-    }
-}
-impl From<path::StripPrefixError> for OrcaError {
-    fn from(error: path::StripPrefixError) -> Self {
+impl From<serde_yaml::Error> for OrcaError {
+    fn from(error: serde_yaml::Error) -> Self {
         Self {
-            kind: Kind::PathPrefixError(error),
+            kind: Kind::SerdeYamlError(error),
         }
     }
 }