Update README

Change organization links

.cargo/config.toml

@@ -0,0 +1,2 @@
+[build]
+rustdocflags = ["--html-in-header", "assets/header.html", "--default-theme", "light", "--extend-css","assets/doc.css"]

.gitignore

@@ -1,4 +1,5 @@
 target
+tmp
 Cargo.lock
 .DS_Store
 ._.DS_Store

ABOUT.md

@@ -0,0 +1,51 @@
+
+# Backend internal interview (rust) - improved  
+
+## Verification
+
+From [input.txt](input.txt) compare output with [output_expected.txt](output_expected.txt)
+
+```bash
+cargo run -q -- -i input.txt -q
+```
+
+From tests
+
+```rust
+cargo test test_expected_metrics
+```
+
+**test_expectd_metrics** can be found at the end of [tests.rs](src/tests.rs)
+
+> DEFAULT_INPUT_PATH and DEFAULT_REG_EXP values can be found in [defs.rs](src/defs.rs)
+
+### For parallel processing
+
+A second file [input_2.txt](input_2.txt) has been created and included in [config.toml](config.toml)
+
+```rust
+cargo run -q
+```
+
+It the same as:
+
+```rust
+cargo run -q -- -c config.toml
+```
+
+> [!NOTE]
+> [config.toml](config.toml) is DEFAULT_CONFIG_PATH if does not exists it will only use DEFAULT_INPUT_PATH [input.txt](input.txt)
+
+## Read Documentation
+
+Build documentation and browse to **main** page
+
+```rust
+cargo doc --no-deps --open
+```
+
+Source code contains doc text.  
+
+## How to use
+
+For more explanations use [how to use](howto.md)

CHANGES.md

@@ -0,0 +1,14 @@
+# Backend internal interview (rust)
+
+## CHANGES: improved branch
+
+- Create abstractions / models to load and parse metrics, can be easily extended 
+- Define basic operations via [`MetricParser`] trait path **defs/metrics.rs**
+- Use **structs** to group metrics attributes and implement operations like [`metrics_consumer::MetricsConsumerTarget`]
+- Use **enums** with associated values like [`defs::metric_data::MetricsConsumerData`] path  **defs/metric_data** to group attributes and values
+- Remove **Maps collections**, use [Vectors](https://doc.rust-lang.org/std/vec/struct.Vec.html)
+- Use **const** for DEFAULT values on [`defs`] module 
+- Remove all <i>hardcoded</i> values to [`Config`] settings, a **TOML config file** (see config.toml) can be used to define several metrics input and customize output
+- **Output** write to file (append o rewrite mode) is working (for now I/O is sync)
+- **Command line** arguments are processed [`parse_args`] with [`defs::CliSettings`] via [clap](https://docs.rs/clap/latest/clap/) and **overide** the ones loaded in **config files**
+- **Tests** have been accommodated from previous [Branch basic-fixed](https://repo.jesusperez.pro/NewRelic/be-technical-interview-rust/src/branch/basic-fixed) version to **imporved** approach

Cargo.toml

@@ -1,8 +1,20 @@
 [package]
 name = "be-technical-interview-rust"
+description = "Backend internal interview (rust) October 2024"
 version = "0.1.0"
 edition = "2021"
+authors = ["Jesús Pérez <jpl@jesusperez.pro>"]
+license = "MIT OR Apache-2.0"
+publish = false
 
 [dependencies]
 regex = "1.10.4"
 chrono = "0.4"
+
+# These are to parse and load toml [`Config`]  files 
+serde = { version = "1.0.210", features = ["derive"] }
+serde_derive = "1.0.210"
+toml = "0.8.19"
+
+# This is for command line options [`Cli`]
+clap = {version = "4.5.20", features = [ "derive"] } 

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2024 jesus
+Copyright (c) 2024 Jesús Pérez
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 

README.md

@@ -1,15 +1,66 @@
-# Backend internal interview (rust)
+# Backend internal interview (rust) - improved  
 
-A **Refactor metric-consumer** task
+This **Improved** branch is a rather disruptive approach to the [initial proposal](https://repo.jesusperez.pro/jesus/be-technical-interview-rust)<br>
+[Branch basic-fixed](https://repo.jesusperez.pro/jesus/be-technical-interview-rust/src/branch/basic-fixed) tried to solve proposal from initial code 
+as a **continuity effort** with the necessary changes and some improvement adjustments such as the **parallel input processing**
+
+> [!NOTE]
+> A full refactoring done for <u>better quality, maintenance and readability</u>. (Structs, implementaitions, settings for multiple inputs, etc). <br>
+> It is able to **process multiple metrics in parallel**. <br>
+
+## In summary
+
+- [x]  Define a basic model, easily to extend and modify. **Abstraction / Generic**.
+- [x]  Structs and implementations to specific metricis traitments. **Modular appoach**.
+- [x]  Settings and configuration for interactive and non interactive processing (batch mode) **Customize on context**.
+
+See [main changes](/jesus/be-technical-interview-rust/src/branch/improved/CHANGES.md)
+
+> [!IMPORTANT]  
+> Use [ABOUT](ABOUT.md) content for quick [Verification](ABOUT.md) (it requires download and build)
+
+## Benefits
+
+Elements items come from  [main changes](/jesus/be-technical-interview-rust/src/branch/improved/CHANGES.md)
+
+| Element          | Benefit |
+|----------------- |---------|
+| Generic traits   | Group generic task for metric processing, steps / tasks separation |
+| Structs          | Customize atributes and implementation for specific target or patterns  |
+| Enums with values| Associate attributes metrics and values, easy to add new attributes or combine with different values at once |
+| Vectors          | Simplify types / grouped in structs, priorize vectors type, easy to iterate, filter, sort, etc |
+| Const and Config | Group main const, define metric targes and operations in declarative mode for non intective traitment |
+| Command line args| Help to run in terminal as a cli |
+| Unit Tests       | Verify some operations results |
+
+> [!TIP]  
+> After download repository and build:
+
+> - Use [howto](howto.md) for command, options, etc.
+> - Build documentation and browse content whit source code (instruction in [about](ABOUT.md))
+> - Files layout notes in [layout.md](layout.md)
+
+
+## Ideas not included
+
+- **Async I/O** to scale and performance ?
+- Other Thread alternatives like [Tokio](https://tokio.rs/) or/and [Coroutines](https://doc.rust-lang.org/std/ops/trait.Coroutine.html)
+- Benchmarking for optimization
+- More **tests**
+- Run as **API mode** not only as batch processing
+
+> [!NOTE]
+> Code is in a private repository with several other branches.<br>
+> Link to [branch repository improved](https://repo.jesusperez.pro/jesus/be-technical-interview-rust/src/branch/improved)
+
+---
 
 [Home Task exercise description](Home-Task_exercise_interview_Rust.pdf)
 
 There are several branches developed as proposal:
 
-- [Basic fixed one](basic-fixed)
-  > A basic code review, fixed with minor and essential changes to work as expected.
-  > Tests are included for verification.
+- [Basic fixed one](/jesus/be-technical-interview-rust/src/branch/basic-fixed)
+  > A basic code review, fixed with minor and essential changes to work as expected. <br>
+  > Tests are included for verification. <br>
 
-- [Improve one](improved)
-
-- [Multiple input one](multi-input)
+- [Improve one](/jesus/be-technical-interview-rust/src/branch/improved)

assets/doc.css

@@ -0,0 +1,3 @@
+.logo-container img {
+  width: 100% !important;
+}

assets/howto.md

@@ -0,0 +1,80 @@
+
+## How to used
+
+**cargo run --** can be replaced for generated binaries with
+ 
+- target/debug/be-technical-interview-rust (for debug build)
+- target/release/be-technical-interview-rust (for release build)
+
+### For help
+
+```bash
+cargo run -- -h
+```
+
+### For input file **input.txt**
+
+```bash
+cargo run -- -i input.txt
+```
+
+### In quiet mode
+
+```bash
+cargo run -- -i input.txt -q
+```
+
+if **output path** is provided, results will be saved in provided path,<br>
+without **quiet mode** info is printed to terminal
+
+```bash
+cargo run -- -i input.txt -o /tmp/output.txt
+```
+
+### Use config path. **Batch processing in parallel** (one thread for each target) 
+
+```bash
+cargo run -- -c config.toml
+```
+
+### Config file content
+
+```toml
+be_quiet = false
+[[targets]]
+input = "input.txt"
+
+[[targets]]
+input = "input_2.txt"
+```
+
+If **output** path is provided, **out_overwrite** can be used (true or false) to append conten or rewrite.
+
+By default:
+
+- **out_overwrite** is **true**
+- *reg_exp** is set as
+
+```rust
+pub const DEFAULT_REG_EXP: &str = r"(\d+) (\w+) (\d+)";
+```
+
+**reg_exp** can be provided but as a **regex** expresion from **string**, if it can not be converted parser exit.
+
+To solve this
+
+```toml
+be_quiet = false
+[[targets]]
+input = "input.txt"
+# \ has to be escaped
+reg_exp = "(\\d+) (\\w+) (\\d+)"
+
+[[targets]]
+input = "input_2.txt"
+```
+
+#### CAUTION
+
+Command line options have precedence over **config path** settings. <br>
+Be careful with the combinations

config.toml

@@ -0,0 +1,8 @@
+be_quiet = false 
+[[targets]]
+input = "input.txt"
+# \ has to be escaped 
+reg_exp = "(\\d+) (\\w+) (\\d+)"
+
+[[targets]]
+input = "input_2.txt"

howto.md

@@ -0,0 +1,81 @@
+
+# Backend internal interview (rust) - improved
+
+## How to used
+
+**cargo run --** can be replaced for generated binaries with
+ 
+- target/debug/be-technical-interview-rust (for debug build)
+- target/release/be-technical-interview-rust (for release build)
+
+### For help
+
+```bash
+cargo run -- -h
+```
+
+### For input file **input.txt**
+
+```bash
+cargo run -- -i input.txt
+```
+
+### In quiet mode
+
+```bash
+cargo run -- -i input.txt -q
+```
+
+if **output path** is provided, results will be saved in provided path,<br>
+without **quiet mode** info is printed to terminal
+
+```bash
+cargo run -- -i input.txt -o /tmp/output.txt
+```
+
+### Use config path. **Batch processing in parallel** (one thread for each target) 
+
+```bash
+cargo run -- -c config.toml
+```
+
+### Config file content
+
+```toml
+be_quiet = false
+[[targets]]
+input = "input.txt"
+
+[[targets]]
+input = "input_2.txt"
+```
+
+If **output** path is provided, **out_overwrite** can be used (true or false) to append conten or rewrite.
+
+By default:
+
+- **out_overwrite** is **true**
+- *reg_exp** is set as
+
+```rust
+pub const DEFAULT_REG_EXP: &str = r"(\d+) (\w+) (\d+)";
+```
+
+**reg_exp** can be provided but as a **regex** expresion from **string**, if it can not be converted parser exit.
+
+To solve this 
+
+```toml
+be_quiet = false 
+[[targets]]
+input = "input.txt"
+# \ has to be escaped 
+reg_exp = "(\\d+) (\\w+) (\\d+)"
+
+[[targets]]
+input = "input_2.txt"
+```
+
+> [!CAUTION]
+> Command line options have precedence over **config path** settings. <br>
+> Be careful with the combinations

input_2.txt

@@ -0,0 +1,19 @@
+1650973147 mem 1761992
+1650973159 cpu 49
+1650973171 mem 1858502
+1650973183 cpu 51
+1650973195 cpu 55
+1650973207 mem 1076203
+1650973219 cpu 60
+1650973231 mem 640005
+1650973243 mem 324911
+1650973255 mem 1024
+1650973267 cpu 56
+1650973279 cpu 58
+1650973291 mem 1024
+1650973303 mem 1024
+1650973315 mem 1024
+1650973327 mem 1024
+1650973339 cpu 49
+1650973351 mem 1024
+1650973363 cpu 49

layout.md

@@ -0,0 +1,38 @@
+
+# Backend internal interview (rust) - improved  
+
+Files layout 
+
+```markdown
+.
+├── ABOUT.md           [Verification](ABOUT.md)
+├── Cargo.lock
+├── Cargo.toml
+├── CHANGES.md         [Changes](CHANGES.md)
+├── Home-Task_exercise_interview_Rust.pdf
+├── LICENSE
+├── README.md 
+├── assets             [Assets files for rustdoc](assets)
+│   ├── doc.css
+│   ├── header.html
+│   └── howto.md
+├── howto.md           [How to use](howto.md)
+├── input.txt
+├── input_2.txt        [Second input for parallel](input2.txt)
+├── layout.md          [Files layout](layout.md)
+├── output_expected.txt
+└── src                [source code](src)
+   ├── defs     
+   │   ├── cli.rs
+   │   ├── config.rs
+   │   ├── metric_data.rs
+   │   └── metrics.rs
+   ├── defs.rs
+   ├── main.rs
+   ├── metrics_consumer.rs
+   └── tests.rs
+```
+
+**.cargo** for **rustdoc** documentation build.
+**.gitignore** to exclude paths like **target**
+

src/defs.rs

@@ -0,0 +1,20 @@
+//! ## Definitions (settings and in common types)
+//! - Group some types definitions in a directory
+//! - Includes global **const** as **&str**
+//! - Export / shared to other code files in crate or public
+mod cli;
+mod config;
+pub mod metric_data;
+pub mod metrics;
+
+pub(crate) use cli::{parse_args, CliSettings};
+pub(crate) use config::{load_from_file, Config};
+pub(crate) use metric_data::MetricsConsumerData;
+pub(crate) use metrics::MetricParser;
+
+pub const PKG_NAME: &str = env!("CARGO_PKG_NAME");
+pub const PKG_VERSION: &str = env!("CARGO_PKG_VERSION");
+pub const CFG_FILE_EXTENSION: &str = ".toml";
+pub const DEFAULT_CONFIG_PATH: &str = "config.toml";
+pub const DEFAULT_INPUT_PATH: &str = "input.txt";
+pub const DEFAULT_REG_EXP: &str = r"(\d+) (\w+) (\d+)";

src/defs/cli.rs

@@ -0,0 +1,60 @@
+//! ## Cli definitions, arguments parsing
+//! - It uses [clap](https://docs.rs/clap/latest/clap/)
+//! - It includes a help options with **-h**
+//! - Alows to use differents **config paths** for batch processing
+//! - Alows to be used interactively via terminal as a command for a single input / output file
+
+use clap::Parser;
+
+use crate::{DEFAULT_CONFIG_PATH, PKG_NAME, PKG_VERSION};
+
+/// Use [clap](https://docs.rs/clap/latest/clap/) to parse command line options with **derive mode**
+#[derive(Parser, Debug)]
+pub struct Cli {
+    /// Config path to load targets settings (-c) command args override config values
+    #[clap(short = 'c', long = "config", value_parser, display_order = 1)]
+    pub config_path: Option<String>,
+
+    /// Quiet mode only data print (-q)
+    #[clap(short = 'q', long = "quiet", action, display_order = 3)]
+    pub be_quiet: bool,
+
+    /// Output path to load input data (-i)
+    #[clap(short = 'i', long = "input", value_parser, display_order = 3)]
+    pub input_path: Option<String>,
+
+    /// Output path to save metric aggreates (-o)
+    #[clap(short = 'o', long = "output", value_parser, display_order = 4)]
+    pub output_path: Option<String>,
+
+    /// Show version
+    #[clap(short = 'v', long = "version", action, display_order = 5)]
+    pub version: bool,
+}
+/// Collect settings for metric targets <br>
+/// Only one **input** and **output** item, for more than one it is much better to use **defs::Config** for better customization for each target.
+#[derive(Debug, Clone)]
+pub struct CliSettings {
+    pub config_path: String,
+    pub be_quiet: Option<bool>,
+    pub input: Option<String>,
+    pub output: Option<String>,
+}
+/// Runs some options from command line <br>
+/// Set TOML config-path to load settings
+pub fn parse_args() -> CliSettings {
+    let args = Cli::parse();
+    if args.version {
+        println!("{} version: {}", PKG_NAME, PKG_VERSION);
+        std::process::exit(0);
+    }
+    let config_path = args
+        .config_path
+        .unwrap_or(String::from(DEFAULT_CONFIG_PATH));
+    CliSettings {
+        config_path,
+        be_quiet: if args.be_quiet { Some(true) } else { None },
+        input: args.input_path,
+        output: args.output_path,
+    }
+}

src/defs/config.rs

@@ -0,0 +1,138 @@
+//! # Config settings definitions
+//! To load config values from TOML file path, it can be provided via command-line arguments <br>
+//! It use [serde](https://serde.rs/) via [`load_from_file`]
+//
+
+use serde::de::DeserializeOwned;
+use serde::{Deserialize, Serialize}; // ,Deserializer,Serializer};
+use std::io::{Error, ErrorKind, Result};
+
+use crate::{defs::CliSettings, CFG_FILE_EXTENSION, DEFAULT_INPUT_PATH, DEFAULT_REG_EXP};
+
+fn default_config_input() -> String {
+    String::from("input.txt")
+}
+fn default_config_output() -> String {
+    String::from("")
+}
+fn default_config_overwrite() -> bool {
+    true
+}
+fn default_config_be_quiet() -> bool {
+    false
+}
+fn default_config_reg_exp() -> String {
+    String::from(DEFAULT_REG_EXP)
+}
+fn default_config_targets() -> Vec<ConfigTarget> {
+    vec![ConfigTarget::default()]
+}
+/// Settings for each target metric defined in **config path**
+/// **config.toml** content example:
+/// ```toml
+/// be_quiet = false
+/// [[targets]]
+/// input = "input.txt"
+///
+/// [[targets]]
+/// input = "input_2.txt"
+/// ```
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ConfigTarget {
+    #[serde(default = "default_config_input")]
+    pub input: String,
+    #[serde(default = "default_config_output")]
+    pub output: String,
+    #[serde(default = "default_config_overwrite")]
+    pub out_overwrite: bool,
+    #[serde(default = "default_config_reg_exp")]
+    pub reg_exp: String,
+}
+
+impl Default for ConfigTarget {
+    fn default() -> Self {
+        Self {
+            input: String::from(DEFAULT_INPUT_PATH),
+            output: String::from(""),
+            out_overwrite: true,
+            reg_exp: String::from(DEFAULT_REG_EXP),
+        }
+    }
+}
+/// Config Settings with target metric settings [`ConfigTarget`]
+/// **config.toml** content example:
+/// ```toml
+/// be_quiet = false
+/// [[targets]]
+/// input = "input.txt"
+///
+/// [[targets]]
+/// input = "input_2.txt"
+/// ```
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Config {
+    #[serde(default = "default_config_be_quiet")]
+    pub be_quiet: bool,
+    #[serde(default = "default_config_targets")]
+    pub targets: Vec<ConfigTarget>,
+}
+impl Default for Config {
+    fn default() -> Self {
+        Self {
+            be_quiet: false,
+            targets: vec![ConfigTarget::default()],
+        }
+    }
+}
+impl Config {
+    // To override [`Config`] values with [`CliSettings`] provided via command line arguments and loaded via [`parse_args`]
+    pub fn add_cli_settings(&self, cli_settings: CliSettings) -> Self {
+        let be_quiet = if let Some(be_quiet) = cli_settings.be_quiet {
+            be_quiet
+        } else {
+            self.be_quiet
+        };
+        let targets = if let Some(input) = cli_settings.input {
+            let mut target = ConfigTarget {
+                input,
+                ..Default::default()
+            };
+            if let Some(output) = cli_settings.output {
+                target.output = output
+            }
+            vec![target]
+        } else {
+            self.targets.to_owned()
+        };
+        Self { be_quiet, targets }
+    }
+}
+/// To load config settings and **Deserialize** content to [`Config`] struct <br>
+/// It use **T** as generic, allowing to load from a file to a **T** type
+/// It use **fs::read_to_string** as it is expecting short files size, no need to buffers.
+pub fn load_from_file<T: DeserializeOwned>(file_cfg: &str) -> Result<T> {
+    let file_path = if file_cfg.contains(CFG_FILE_EXTENSION) {
+        file_cfg.to_string()
+    } else {
+        format!("{}{}", file_cfg, CFG_FILE_EXTENSION)
+    };
+    let config_content = match std::fs::read_to_string(&file_path) {
+        Ok(cfgcontent) => cfgcontent,
+        Err(e) => {
+            return Err(Error::new(
+                ErrorKind::InvalidInput,
+                format!("Error read {}: {}", &file_path, e),
+            ))
+        }
+    };
+    let item_cfg = match toml::from_str::<T>(&config_content) {
+        Ok(cfg) => cfg,
+        Err(e) => {
+            return Err(Error::new(
+                ErrorKind::InvalidInput,
+                format!("Error loading config {}: {}", &file_path, e),
+            ))
+        }
+    };
+    Ok(item_cfg)
+}

src/defs/metric_data.rs

@@ -0,0 +1,101 @@
+//! ## Metrics Data for "Consumer Metrics"
+//! - Collecting, grouping and differentiate name, values, etc.
+
+use chrono::{DateTime, Utc};
+use std::{cmp::Ordering, time::SystemTime};
+
+/// Associate type for metric name in [`MetricsConsumerData`] <br>
+/// It allows to collect several **values** related with same **time**
+#[derive(Clone, Debug)]
+pub struct MetricTimeData {
+    pub time: SystemTime,
+    pub values: Vec<f64>,
+}
+/// Magic rust **enum** to clasify metrics names and their associated values in [`MetricTimeData`] <br>
+/// - It can combine different items with different types with associated values
+/// - Can be extended easily, **rust** will enforce consistence and definitions
+#[derive(Clone, Debug, Default)]
+pub enum MetricsConsumerData {
+    Mem(MetricTimeData),
+    Cpu(MetricTimeData),
+    #[default]
+    Unknown,
+}
+/// As **sort** is needed for some use cases, like **output** <br>
+/// Some implementations has to be written here, are not auto generated via **derive** macros, associtated types with types like **f64** will not allow autogeneration
+impl Eq for MetricsConsumerData {}
+impl Ord for MetricsConsumerData {
+    fn cmp(&self, other: &Self) -> Ordering {
+        self.name().cmp(&other.name())
+    }
+}
+impl PartialOrd for MetricsConsumerData {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.name().cmp(&other.name()))
+    }
+}
+impl PartialEq for MetricsConsumerData {
+    fn eq(&self, other: &Self) -> bool {
+        self.name() == other.name()
+    }
+}
+/// Display per item here
+impl std::fmt::Display for MetricsConsumerData {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            MetricsConsumerData::Mem(data) => write!(
+                f,
+                "{} mem {}",
+                DateTime::<Utc>::from(data.time).format("%Y-%m-%dT%H:%M:%SZ"),
+                format!("{:?}", data.values)
+                    .replace("[", "")
+                    .replace("]", "")
+            ),
+            MetricsConsumerData::Cpu(data) => write!(
+                f,
+                "{} cpu {}",
+                DateTime::<Utc>::from(data.time).format("%Y-%m-%dT%H:%M:%SZ"),
+                format!("{:?}", data.values)
+                    .replace("[", "")
+                    .replace("]", "")
+            ),
+            MetricsConsumerData::Unknown => write!(f, "anonymous"),
+        }
+    }
+}
+/// Some implementations:
+/// - to get item **name** or **data** values
+/// - to **add_data** to exiting [`MetricTimeData`] **values**
+/// - **from_values** allows create an **enum** item
+impl MetricsConsumerData {
+    pub fn name(&self) -> String {
+        match self {
+            MetricsConsumerData::Mem(_) => String::from("mem"),
+            MetricsConsumerData::Cpu(_) => String::from("cpu"),
+            MetricsConsumerData::Unknown => String::from("unknown"),
+        }
+    }
+    pub fn time_data(&self) -> Option<MetricTimeData> {
+        match self {
+            MetricsConsumerData::Mem(data) => Some(data.to_owned()),
+            MetricsConsumerData::Cpu(data) => Some(data.to_owned()),
+            MetricsConsumerData::Unknown => None,
+        }
+    }
+    pub fn add_data(&mut self, value: f64) {
+        match self {
+            MetricsConsumerData::Mem(data) => data.values.push(value),
+            MetricsConsumerData::Cpu(data) => data.values.push(value),
+            MetricsConsumerData::Unknown => (),
+        }
+    }
+    pub fn from_values(name: &str, time: SystemTime, values: Vec<f64>) -> MetricsConsumerData {
+        let metric_time_data = MetricTimeData { time, values };
+        match name {
+            "mem" | "Mem" | "MEM" => MetricsConsumerData::Mem(metric_time_data),
+            "cpu" | "Cpu" | "CPU" => MetricsConsumerData::Cpu(metric_time_data),
+            "unknown" | "Unknown" => MetricsConsumerData::Unknown,
+            _ => MetricsConsumerData::default(),
+        }
+    }
+}

src/defs/metrics.rs

@@ -0,0 +1,17 @@
+//! ## metrics definitions generic models.
+//! - Abstraction via **traits**
+//! - Generic process / tasks / steps required for metrics traitment.
+
+use regex::Regex;
+use std::{error::Error, fs::File};
+
+pub trait MetricParser
+where
+    Self: Sized,
+{
+    fn input(&self) -> String;
+    fn load_input(&self) -> Result<Self, Box<dyn Error>>;
+    fn parse(&self, file: File, reg_exp: Regex) -> Self;
+    fn collect_aggregates(&mut self);
+    fn show_metrics(&self) -> Result<Vec<String>, std::io::Error>;
+}

src/main.rs

@@ -1,76 +1,168 @@
-use std::collections::HashMap;
-use std::error::Error;
-use std::fs::File;
-use std::io::Read;
-use std::time::{Duration, UNIX_EPOCH};
+//! # Backend internal interview (rust) - improved
+//!
+//! This **Improved** branch is a rather disruptive approach to the [initial proposal](https://repo.jesusperez.pro/NewRelic/be-technical-interview-rust)<br>
+//! [Branch basic-fixed](https://repo.jesusperez.pro/NewRelic/be-technical-interview-rust/src/branch/basic-fixed) tried to solve proposal from initial code
+//! as a **continuity effort** with the necessary changes and some improvement adjustments such as the **parallel input processing**
+//!
+//! ## In summary
+//! - [x]  Define a basic model, easily to extend and modify. **Abstraction / Generic**.
+//! - [x]  Structs and implementations to specific metricis traitments. **Modular appoach**.
+//! - [x]  Settings and configuration for interactive and non interactive processing (batch mode) **Customize on context**.
+//!
+//! ## Main improvements
+//! - Create abstractions / models to load and parse metrics, can be easily extended
+//! - Define basic operations via [`MetricParser`] trait path **defs/metrics.rs**
+//! - Use **structs** to group metrics attributes and implement operations like [`metrics_consumer::MetricsConsumerTarget`]
+//! - Use **enums** with associated values like [`defs::metric_data::MetricsConsumerData`] path  **defs/metric_data** to group attributes and values
+//! - Remove **Maps collections**, use [Vectors](https://doc.rust-lang.org/std/vec/struct.Vec.html)
+//! - Use **const** for DEFAULT values on [`defs`] module
+//! - Remove all <i>hardcoded</i> values to [`Config`] settings, a **TOML config file** (see config.toml) can be used to define several metrics input and customize output
+//! - **Output** write to file (append o rewrite mode) is working (for now I/O is sync)
+//! - **Command line** arguments are processed [`parse_args`] with [`defs::CliSettings`] via [clap](https://docs.rs/clap/latest/clap/) and **overide** the ones loaded in **config files**
+//! -  **Tests** have been accommodated from previous [Branch basic-fixed](https://repo.jesusperez.pro/NewRelic/be-technical-interview-rust/src/branch/basic-fixed) version to **imporved** approach
+//!
+//! ## Benefits
+//!
+//! | Element          | Benefit |
+//! |----------------- |---------|
+//! | Generic traits   | Group generic task for metric processing, steps / tasks separation |
+//! | Structs          | Customize atributes and implementation for specific target or patterns  |
+//! | Enums with values| Associate attributes metrics and values, easy to add new attributes or combine with different values at once |
+//! | Vectors          | Simplify types / grouped in structs, priorize vectors type, easy to iterate, filter, sort, etc |
+//! | Const and Config | Group main const, define metric targes and operations in declarative mode for non intective traitment |
+//! | Command line args| Help to run in terminal as a cli |
+//! | Unit Tests       | Verify some operations results |
+//!
+//! ## Ideas not included
+//! - **Async I/O** to scale and performance ?
+//! - Other Thread alternatives like [Tokio](https://tokio.rs/) or/and [Coroutines](https://doc.rust-lang.org/std/ops/trait.Coroutine.html)
+//! - Benchmarking for optimization
+//! - More **tests**
+//! - Run as **API mode** not only as batch processing
+//! <br>
+//!
+//! Code is in a private repository with several branches[^note].
+//! [^note]: Link to [branch repository improved](https://repo.jesusperez.pro/NewRelic/be-technical-interview-rust/src/branch/improved)
 
-fn parse(
-    file: File,
-) -> Result<HashMap<String, HashMap<std::time::SystemTime, f64>>, Box<dyn Error>> {
-    let mut file = file;
-    let mut contents = String::new();
-    file.read_to_string(&mut contents)?;
+#![doc(html_logo_url = "https://info.jesusperez.pro/img/jesusperez-logo-b.png")]
+#![doc = include_str!("../assets/howto.md")]
+use crate::defs::{
+    load_from_file, parse_args, Config, MetricParser, CFG_FILE_EXTENSION, DEFAULT_CONFIG_PATH,
+    DEFAULT_INPUT_PATH, DEFAULT_REG_EXP, PKG_NAME, PKG_VERSION,
+};
+use crate::metrics_consumer::MetricsConsumerTarget;
+#[doc = include_str!("../README.md")]
+use std::{
+    sync::{
+        mpsc,
+        mpsc::{Receiver, Sender},
+    },
+    thread::spawn,
+    time::Instant,
+};
 
-    let mut metrics: HashMap<String, HashMap<std::time::SystemTime, Vec<f64>>> = HashMap::new();
+mod defs;
+mod metrics_consumer;
 
-    for line in contents.lines() {
-        let re = regex::Regex::new(r"(\d+) (\w+) (\d+)").unwrap();
-        if let Some(caps) = re.captures(line) {
-            let timestamp_raw = &caps[1];
-            let metric_name = &caps[2];
-            let metric_value_raw = &caps[3];
+// Tests are in a separated module for easy access
+#[cfg(test)]
+mod tests;
 
-            let timestamp = timestamp_raw.parse::<i64>().unwrap();
-            let metric_value = metric_value_raw.parse::<f64>().unwrap();
-
-            if !metrics.contains_key(metric_name) {
-                metrics.insert(metric_name.to_string(), HashMap::new());
+/// Main threads control flow for each `target_list` item <br>
+/// All process are collected and finally <br>
+/// [`MetricsConsumerTarget`] as **T** to **show_metrics** is called to get **print** or **write** results <br>
+/// **be_quiet** attribute is just to avoid (true) all messages around processing and parsing operations <br>
+/// ## For paralellism
+/// This can be done with [Tokio](https://tokio.rs/) as alternative to [std::thread](https://doc.rust-lang.org/std/thread/) <br>
+/// It will require load other **crates** and feature customizations<br>
+/// Another alternative could be [Coroutines](https://doc.rust-lang.org/std/ops/trait.Coroutine.html) <br>
+/// As experimental features is a **nightly-only** (October 2024)
+fn generate_metrics<T: MetricParser + Sync + Send + 'static>(targets_list: Vec<T>, be_quiet: bool) {
+    let n_items = targets_list.len();
+    let mut input_threads = Vec::with_capacity(n_items);
+    let (tx, rx): (Sender<Option<T>>, Receiver<Option<T>>) = mpsc::channel();
+    targets_list.into_iter().for_each(|metrics_item| {
+        let thread_tx = tx.clone();
+        input_threads.push(spawn(move || {
+            let start = Instant::now();
+            match metrics_item.load_input() {
+                Ok(mut result) => {
+                    result.collect_aggregates();
+                    thread_tx.send(Some(result)).unwrap_or_default();
+                }
+                Err(err) => {
+                    eprint!("Error: {}", err);
+                    thread_tx.send(None).unwrap_or_default();
+                }
             }
-
-            let minute = UNIX_EPOCH + Duration::from_secs((timestamp - (timestamp % 60)) as u64);
-            metrics
-                .get_mut(metric_name)
-                .unwrap()
-                .insert(minute, vec![metric_value]);
-        } else {
-            println!("invalid line");
+            if !be_quiet {
+                println!(
+                    "Processing {} took: {:?} ms",
+                    &metrics_item.input(),
+                    start.elapsed().as_millis()
+                )
+            }
+        }));
+    });
+    let mut inputs_metrics = Vec::with_capacity(n_items);
+    for _ in 0..input_threads.len() {
+        match rx.recv() {
+            Ok(result) => inputs_metrics.push(result),
+            Err(e) => eprint!("Error: {}", e),
         }
     }
-
-    let mut aggregated_metrics: HashMap<String, HashMap<std::time::SystemTime, f64>> =
-        HashMap::new();
-
-    for (metric_name, time_val_list) in metrics {
-        aggregated_metrics.insert(metric_name.clone(), HashMap::new());
-
-        for (time, values) in time_val_list {
-            let mut sum = 0.0;
-            for v in values.iter() {
-                sum += *v
-            }
-            let average = sum / values.len() as f64;
-
-            aggregated_metrics
-                .get_mut(&metric_name)
-                .unwrap()
-                .insert(time, average);
-        }
+    for thread in input_threads {
+        let _ = thread.join();
     }
-
-    Ok(aggregated_metrics)
+    inputs_metrics.iter().for_each(|metrics_item| {
+        if let Some(metrics_data) = metrics_item {
+            if !be_quiet {
+                println!("\n{}: ---------------\n", &metrics_data.input());
+            }
+            let _ = metrics_data.show_metrics();
+        }
+    });
 }
-
+/// Keep it short:
+/// - Parse [`defs::CliSettings`] from command-line arguments
+/// - Load [`Config`] file settings from **config_path**
+/// - Override [`Config`] settings with cli arguments parsed
+/// - Create **targets list** as [`MetricsConsumerTarget`] vector
+/// - Call to [`generate_metrics`] to do the job
+/// - If not **be_quiet** mode print elapsed time in milliseconds
+///
+/// > This is not running async and not expect any **Result**.
 fn main() {
-    let file = File::open("input.txt").expect("Unable to open file");
-    let metrics = parse(file).expect("Unable to parse file");
-    for (metric_name, time_val) in metrics {
-        for (time, value) in time_val {
-            println!(
-                "{} {:?} {:.2}",
-                metric_name,
-                chrono::DateTime::<chrono::Utc>::from(time),
-                value
-            );
-        }
+    let main_start = Instant::now();
+    let args_settings = parse_args();
+    let config: Config = if std::path::Path::new(&args_settings.config_path).exists() {
+        load_from_file(&args_settings.config_path).unwrap_or_else(|e| {
+            eprintln!("Settings error: {}", e);
+            Config::default()
+        })
+    } else {
+        Config::default()
+    };
+    let config = config.add_cli_settings(args_settings.clone());
+    if !config.be_quiet {
+        println!("Loaded config from: {}", &args_settings.config_path);
+    }
+    let targets_list = config
+        .targets
+        .iter()
+        .map(|item| MetricsConsumerTarget {
+            input: String::from(&item.input),
+            output: String::from(&item.output),
+            out_overwrite: item.out_overwrite,
+            reg_exp: String::from(&item.reg_exp),
+            ..Default::default()
+        })
+        .collect();
+    generate_metrics(targets_list, config.be_quiet);
+    if !config.be_quiet {
+        println!(
+            "\nALL Processing took: {:?} ms",
+            main_start.elapsed().as_millis()
+        )
     }
 }

src/metrics_consumer.rs

@@ -0,0 +1,185 @@
+//! ## MetricsConsumerTarget definitions and implementations
+//! Specific metric class **Consumer Metric** using generic metric operations <br>
+//! Save source / result paths, parsing regular_expression and collect metrics values and their aggreates using [`MetricsConsumerData`]
+//! From this modular approach other metrics classes can be defined and implemented
+
+use regex::Regex;
+use std::{
+    error::Error,
+    fs::File,
+    io::{prelude::*, BufReader, BufWriter},
+    time::{Duration, UNIX_EPOCH},
+};
+
+use crate::defs::{MetricParser, MetricsConsumerData};
+
+/// Attributes definition
+/// - **reg_exp** has some difficulties to be used with [regex](https://docs.rs/regex/latest/regex/), it works better with [string literals](https://doc.rust-lang.org/reference/expressions/literal-expr.html#string-literal-expressions)
+/// - **metrics** and **aggregates** are vectors of [`MetricsConsumerData`] enums values to group input lines or save aggregates values
+/// - **metrics** and **aggregates** have same type for implementation simplification, **aggregates** only use <u>first vector value</u> <br>
+///   it can be easily used or extended to also save other computed values like: max, min, etc.
+#[derive(Debug, Clone, Default)]
+pub(crate) struct MetricsConsumerTarget {
+    pub input: String,
+    pub output: String,
+    pub out_overwrite: bool,
+    pub reg_exp: String,
+    pub metrics: Vec<MetricsConsumerData>,
+    pub aggregates: Vec<MetricsConsumerData>,
+}
+
+/// Implement generic metrics operations / tasks for [`MetricsConsumerData`]
+///
+impl MetricParser for MetricsConsumerTarget {
+    fn parse(&self, file: File, reg_exp: Regex) -> Self {
+        let buf = BufReader::new(file);
+        let mut consumer_target = self.clone();
+        buf.lines()
+            .enumerate()
+            .for_each(|(index, read_line)| match read_line {
+                Ok(line) => {
+                    if let Some(metric_line) = self.parse_line(&line, index, &reg_exp) {
+                        let (timestamp, name, value) = metric_line;
+                        let minute =
+                            UNIX_EPOCH + Duration::from_secs((timestamp - (timestamp % 60)) as u64);
+                        let mut not_found = true;
+                        for metric in consumer_target.metrics.iter_mut() {
+                            if metric.name() == name.to_lowercase() {
+                                if let Some(metric_data) = metric.time_data() {
+                                    if metric_data.time == minute {
+                                        metric.add_data(value);
+                                        not_found = false;
+                                        break;
+                                    }
+                                }
+                            }
+                        }
+                        if not_found {
+                            consumer_target
+                                .metrics
+                                .push(MetricsConsumerData::from_values(
+                                    &name,
+                                    minute,
+                                    Vec::from([value]),
+                                ));
+                        }
+                    }
+                }
+                Err(e) => {
+                    eprintln!("Error reading line {}: {}", index, e);
+                }
+            });
+        consumer_target
+    }
+    fn load_input(&self) -> Result<Self, Box<dyn Error>> {
+        let file = File::open(&self.input)
+            .map_err(|err| format!("Error reading file: {} {}", &self.input, err))?;
+        if self.reg_exp.is_empty() {
+            return Err(String::from("Error invalid reg expression").into());
+        }
+        let reg_exp = Regex::new(&self.reg_exp)?;
+
+        Ok(self.parse(file, reg_exp))
+    }
+    fn collect_aggregates(&mut self) {
+        self.metrics.iter().for_each(|metric_data| {
+            let name = metric_data.name();
+            if let Some(metric_time_data) = metric_data.time_data() {
+                let average = metric_time_data.values.iter().sum::<f64>()
+                    / metric_time_data.values.len() as f64;
+                let mut not_found = true;
+                for metric in self.aggregates.iter_mut() {
+                    if metric.name() == name.to_lowercase() {
+                        if let Some(metric_data) = metric.time_data() {
+                            if metric_data.time == metric_time_data.time {
+                                metric.add_data(average);
+                                not_found = false;
+                                break;
+                            }
+                        }
+                    }
+                }
+                if not_found {
+                    self.aggregates.push(MetricsConsumerData::from_values(
+                        &name,
+                        metric_time_data.time,
+                        Vec::from([average]),
+                    ));
+                }
+            }
+        })
+    }
+    fn show_metrics(&self) -> Result<Vec<String>, std::io::Error> {
+        let mut output = Vec::new();
+        let mut aggregates = self.aggregates.clone();
+        aggregates.sort();
+        aggregates.iter().for_each(|metric_data| {
+            if metric_data.time_data().is_some() {
+                let output_line = format!("{}", metric_data);
+                match self.output.as_str() {
+                    "vec" => output.push(output_line),
+                    "print" | "" => println!("{}", output_line),
+                    _ => output.push(output_line),
+                }
+            }
+        });
+        match self.output.as_str() {
+            "vec" | "print" | "" => return Ok(output.to_owned()),
+            _ => {
+                if self.out_overwrite && std::path::Path::new(&self.output).exists() {
+                    std::fs::remove_file(&self.output)?;
+                }
+                if !std::path::Path::new(&self.output).exists() {
+                    File::create(&self.output)?;
+                }
+                let file = File::options().append(true).open(&self.output)?;
+                let mut writer = BufWriter::new(file);
+                writer.write_all(output.join("\n").as_bytes())?;
+                let text_overwrite = if self.out_overwrite {
+                    String::from("overwriten")
+                } else {
+                    String::from("")
+                };
+                println!(
+                    "Metrics for '{}' are saved in '{}' {}",
+                    &self.input, &self.output, &text_overwrite
+                );
+            }
+        };
+        Ok(output.to_owned())
+    }
+    fn input(&self) -> String {
+        self.input.to_string()
+    }
+}
+/// Specific implementations like **parse_line**
+impl MetricsConsumerTarget {
+    fn show_invalid_line(&self, index: usize, line: &str) {
+        println!("invalid line: {} {}", index, line);
+    }
+    /// Check metric line values
+    pub fn parse_line(&self, line: &str, index: usize, re: &Regex) -> Option<(i64, String, f64)> {
+        if let Some(caps) = re.captures(line) {
+            let timestamp = match caps[1].parse::<i64>() {
+                Ok(value) => value,
+                Err(e) => {
+                    println!("Parse timestamp {} error {}", &caps[1], e);
+                    self.show_invalid_line(index, line);
+                    return None;
+                }
+            };
+            let metric_value = match caps[3].parse::<f64>() {
+                Ok(value) => value,
+                Err(e) => {
+                    println!("Parse metric_value {} error {}", &caps[3], e);
+                    self.show_invalid_line(index, line);
+                    return None;
+                }
+            };
+            Some((timestamp, caps[2].to_string(), metric_value))
+        } else {
+            self.show_invalid_line(index, line);
+            None
+        }
+    }
+}

src/tests.rs

@@ -0,0 +1,73 @@
+//! ## Tests
+//! Some unitary tests grouped here <br>
+//! [`test_expected_metrics`] the more important one to verify results with **output_expected.txt**
+//!
+use super::*;
+#[test]
+fn test_load_input() -> Result<(), String> {
+    let metrics_target = MetricsConsumerTarget {
+        input: String::from(DEFAULT_INPUT_PATH),
+        reg_exp: String::from(DEFAULT_REG_EXP),
+        ..Default::default()
+    };
+    match metrics_target.load_input() {
+        Ok(_) => Ok(()),
+        Err(e) => Err(format!("Error: {}", e).into()),
+    }
+}
+#[test]
+fn test_invalid_line_value() -> Result<(), String> {
+    let metrics_target = MetricsConsumerTarget {
+        input: String::from(DEFAULT_INPUT_PATH),
+        reg_exp: String::from(DEFAULT_REG_EXP),
+        ..Default::default()
+    };
+    let contents = String::from("1650973075 cpu A47\n");
+    let re = regex::Regex::new(&metrics_target.reg_exp)
+        .map_err(|err| format!("Error regex: {}", err))?;
+    match metrics_target.parse_line(&contents, 1, &re) {
+        Some(_) => Err(format!("Error invalid line value: {}", contents).into()),
+        None => Ok(()),
+    }
+}
+#[test]
+fn test_invalid_line_time() -> Result<(), String> {
+    let metrics_target = MetricsConsumerTarget {
+        input: String::from(DEFAULT_INPUT_PATH),
+        reg_exp: String::from(DEFAULT_REG_EXP),
+        ..Default::default()
+    };
+    let contents = String::from("1650973075A cpu 47\n");
+    let re = regex::Regex::new(&metrics_target.reg_exp)
+        .map_err(|err| format!("Error regex: {}", err))?;
+    match metrics_target.parse_line(&contents, 1, &re) {
+        Some(_) => Err(format!("Error invalid line value: {}", contents).into()),
+        None => Ok(()),
+    }
+}
+#[test]
+fn test_expected_metrics() {
+    use std::{
+        fs::File,
+        io::{prelude::*, BufReader},
+    };
+    let metrics_target = MetricsConsumerTarget {
+        input: String::from(DEFAULT_INPUT_PATH),
+        output: String::from("vec"),
+        reg_exp: String::from(DEFAULT_REG_EXP),
+        ..Default::default()
+    };
+    let mut result = metrics_target.load_input().unwrap_or_default();
+    result.collect_aggregates();
+    let data_metrics = result.show_metrics().unwrap_or_default();
+
+    let expected_output = String::from("output_expected.txt");
+    let file = File::open(expected_output.clone())
+        .expect(format!("no such file: {}", expected_output).as_str());
+    let buf = BufReader::new(file);
+    let lines: Vec<String> = buf
+        .lines()
+        .map(|l| l.expect("Could not parse line"))
+        .collect();
+    assert_eq!(lines.join("\n"), data_metrics.join("\n"));
+}