jesus/be-technical-interview-rust
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: fmt, bound generic for generate_metrics, adjust types to generic, remove some mut and fix tests

Browse Source
This commit is contained in:
Jesús Pérez 2024-10-22 12:46:22 +01:00
parent c29aed3316
commit de89086e90
No known key found for this signature in database
8 changed files with 104 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/defs.rs
View File

@ -1,5 +1,5 @@
//! ## Definitions (settings and in common types)
//! - Group some types definitions in a directory
//! - Group some types definitions in a directory
//! - Includes global **const** as **&str**
//! - Export / shared to other code files in crate or public
mod cli;

4
src/defs/cli.rs
View File

@ -1,7 +1,7 @@
//! ## Cli definitions, arguments parsing
//! ## Cli definitions, arguments parsing
//! - It uses [clap](https://docs.rs/clap/latest/clap/)
//! - It includes a help options with **-h**
//! - Alows to use differents **config paths** for batch processing
//! - Alows to use differents **config paths** for batch processing
//! - Alows to be used interactively via terminal as a command for a single input / output file
use clap::Parser;

16
src/defs/config.rs
View File

@ -1,6 +1,6 @@
//! # Config settings definitions
//! To load config values from TOML file path, it can be provided via command-line arguments <br>
//! It use [serde](https://serde.rs/) via [`load_from_file`]
//! It use [serde](https://serde.rs/) via [`load_from_file`]
//
use serde::de::DeserializeOwned;
@ -28,12 +28,12 @@ fn default_config_targets() -> Vec<ConfigTarget> {
vec![ConfigTarget::default()]
}
/// Settings for each target metric defined in **config path**
/// **config.toml** content example:
/// **config.toml** content example:
/// ```toml
/// be_quiet = false
/// be_quiet = false
/// [[targets]]
/// input = "input.txt"
///
///
/// [[targets]]
/// input = "input_2.txt"
/// ```
@ -60,12 +60,12 @@ impl Default for ConfigTarget {
}
}
/// Config Settings with target metric settings [`ConfigTarget`]
/// **config.toml** content example:
/// **config.toml** content example:
/// ```toml
/// be_quiet = false
/// be_quiet = false
/// [[targets]]
/// input = "input.txt"
///
///
/// [[targets]]
/// input = "input_2.txt"
/// ```
@ -109,7 +109,7 @@ impl Config {
}
/// To load config settings and **Deserialize** content to [`Config`] struct <br>
/// It use **T** as generic, allowing to load from a file to a **T** type
/// It use **fs::read_to_string** as it is expecting short files size, no need to buffers.
/// It use **fs::read_to_string** as it is expecting short files size, no need to buffers.
pub fn load_from_file<T: DeserializeOwned>(file_cfg: &str) -> Result<T> {
let file_path = if file_cfg.contains(CFG_FILE_EXTENSION) {
file_cfg.to_string()

6
src/defs/metric_data.rs
View File

@ -1,11 +1,11 @@
//! ## Metrics Data for "Consumer Metrics"
//! ## Metrics Data for "Consumer Metrics"
//! - Collecting, grouping and differentiate name, values, etc.
use chrono::{DateTime, Utc};
use std::{cmp::Ordering, time::SystemTime};
/// Associate type for metric name in [`MetricsConsumerData`] <br>
/// It allows to collect several **values** related with same **time**
/// It allows to collect several **values** related with same **time**
#[derive(Clone, Debug)]
pub struct MetricTimeData {
pub time: SystemTime,
@ -66,7 +66,7 @@ impl std::fmt::Display for MetricsConsumerData {
/// Some implementations:
/// - to get item **name** or **data** values
/// - to **add_data** to exiting [`MetricTimeData`] **values**
/// - **from_values** allows create an **enum** item
/// - **from_values** allows create an **enum** item
impl MetricsConsumerData {
pub fn name(&self) -> String {
match self {

16
src/defs/metrics.rs
View File

@ -1,13 +1,17 @@
//! ## metrics definitions generic models.
//! ## metrics definitions generic models.
//! - Abstraction via **traits**
//! - Generic process / tasks / steps required for metrics traitment.
use std::{error::Error, fs::File};
use regex::Regex;
use std::{error::Error, fs::File};
pub trait MetricParser {
fn load_input(&mut self) -> Result<bool, Box<dyn Error>>;
fn parse(&mut self, file: File, reg_exp: Regex);
pub trait MetricParser
where
Self: Sized,
{
fn input(&self) -> String;
fn load_input(&self) -> Result<Self, Box<dyn Error>>;
fn parse(&self, file: File, reg_exp: Regex) -> Self;
fn collect_aggregates(&mut self);
fn show_metrics(&mut self) -> Result<Vec<String>, std::io::Error>;
fn show_metrics(&self) -> Result<Vec<String>, std::io::Error>;
}

79
src/main.rs
View File

@ -1,46 +1,46 @@
//! # Backend internal interview (rust) - improved
//!
//!
//! This **Improved** branch is a rather disruptive approach to the [initial proposal](https://repo.jesusperez.pro/NewRelic/be-technical-interview-rust)<br>
//! [Branch basic-fixed](https://repo.jesusperez.pro/NewRelic/be-technical-interview-rust/src/branch/basic-fixed) tried to solve proposal from initial code
//! [Branch basic-fixed](https://repo.jesusperez.pro/NewRelic/be-technical-interview-rust/src/branch/basic-fixed) tried to solve proposal from initial code
//! as a **continuity effort** with the necessary changes and some improvement adjustments such as the **parallel input processing**
//!
//!
//! ## In summary
//! - [x] Define a basic model, easily to extend and modify. **Abstraction / Generic**.
//! - [x] Structs and implementations to specific metricis traitments. **Modular appoach**.
//! - [x] Settings and configuration for interactive and non interactive processing (batch mode) **Customize on context**.
//!
//!
//! ## Main improvements
//! - Create abstractions / models to load and parse metrics, can be easily extended
//! - Create abstractions / models to load and parse metrics, can be easily extended
//! - Define basic operations via [`MetricParser`] trait path **defs/metrics.rs**
//! - Use **structs** to group metrics attributes and implement operations like [`metrics_consumer::MetricsConsumerTarget`]
//! - Use **enums** with associated values like [`defs::metric_data::MetricsConsumerData`] path **defs/metric_data** to group attributes and values
//! - Remove **Maps collections**, use [Vectors](https://doc.rust-lang.org/std/vec/struct.Vec.html)
//! - Use **const** for DEFAULT values on [`defs`] module
//! - Use **const** for DEFAULT values on [`defs`] module
//! - Remove all <i>hardcoded</i> values to [`Config`] settings, a **TOML config file** (see config.toml) can be used to define several metrics input and customize output
//! - **Output** write to file (append o rewrite mode) is working (for now I/O is sync)
//! - **Command line** arguments are processed [`parse_args`] with [`defs::CliSettings`] via [clap](https://docs.rs/clap/latest/clap/) and **overide** the ones loaded in **config files**
//! - **Tests** have been accommodated from previous [Branch basic-fixed](https://repo.jesusperez.pro/NewRelic/be-technical-interview-rust/src/branch/basic-fixed) version to **imporved** approach
//!
//! ## Benefits
//!
//!
//! ## Benefits
//!
//! | Element | Benefit |
//! |----------------- |---------|
//! | Generic traits | Group generic task for metric processing, steps / tasks separation |
//! | Structs | Customize atributes and implementation for specific target or patterns |
//! | Enums with values| Associate attributes metrics and values, easy to add new attributes or combine with different values at once |
//! | Vectors | Simplify types / grouped in structs, priorize vectors type, easy to iterate, filter, sort, etc |
//! | Vectors | Simplify types / grouped in structs, priorize vectors type, easy to iterate, filter, sort, etc |
//! | Const and Config | Group main const, define metric targes and operations in declarative mode for non intective traitment |
//! | Command line args| Help to run in terminal as a cli |
//! | Unit Tests | Verify some operations results |
//!
//! ## Ideas not included
//! - **Async I/O** to scale and performance ?
//! - Other Thread alternatives like [Tokio](https://tokio.rs/) or/and [Coroutines](https://doc.rust-lang.org/std/ops/trait.Coroutine.html)
//!
//! ## Ideas not included
//! - **Async I/O** to scale and performance ?
//! - Other Thread alternatives like [Tokio](https://tokio.rs/) or/and [Coroutines](https://doc.rust-lang.org/std/ops/trait.Coroutine.html)
//! - Benchmarking for optimization
//! - More **tests**
//! - More **tests**
//! - Run as **API mode** not only as batch processing
//! <br>
//!
//! <br>
//!
//! Code is in a private repository with several branches[^note].
//! [^note]: Link to [branch repository improved](https://repo.jesusperez.pro/NewRelic/be-technical-interview-rust/src/branch/improved)
@ -70,47 +70,40 @@ mod tests;
/// Main threads control flow for each `target_list` item <br>
/// All process are collected and finally <br>
/// [`MetricsConsumerTarget`] to **show_metrics** is called to get **print** or **write** results <br>
/// [`MetricsConsumerTarget`] as **T** to **show_metrics** is called to get **print** or **write** results <br>
/// **be_quiet** attribute is just to avoid (true) all messages around processing and parsing operations <br>
/// ## For paralellism
/// ## For paralellism
/// This can be done with [Tokio](https://tokio.rs/) as alternative to [std::thread](https://doc.rust-lang.org/std/thread/) <br>
/// It will require load other **crates** and feature customizations<br>
/// Another alternative could be [Coroutines](https://doc.rust-lang.org/std/ops/trait.Coroutine.html) <br>
/// As experimental features is a **nightly-only** (October 2024)
fn generate_metrics(targets_list: Vec<MetricsConsumerTarget>, be_quiet: bool) {
fn generate_metrics<T: MetricParser + Sync + Send + 'static>(targets_list: Vec<T>, be_quiet: bool) {
let n_items = targets_list.len();
let mut input_threads = Vec::with_capacity(n_items);
let (tx, rx): (
Sender<MetricsConsumerTarget>,
Receiver<MetricsConsumerTarget>,
) = mpsc::channel();
for mut metrics_consumer_data in targets_list.clone() {
let (tx, rx): (Sender<Option<T>>, Receiver<Option<T>>) = mpsc::channel();
targets_list.into_iter().for_each(|metrics_item| {
let thread_tx = tx.clone();
input_threads.push(spawn(move || {
let start = Instant::now();
match metrics_consumer_data.load_input() {
Ok(_) => {
metrics_consumer_data.collect_aggregates();
thread_tx
.send(metrics_consumer_data.clone())
.unwrap_or_default();
match metrics_item.load_input() {
Ok(mut result) => {
result.collect_aggregates();
thread_tx.send(Some(result)).unwrap_or_default();
}
Err(err) => {
eprint!("Error: {}", err);
thread_tx
.send(metrics_consumer_data.clone())
.unwrap_or_default();
thread_tx.send(None).unwrap_or_default();
}
}
if !be_quiet {
println!(
"Processing {} took: {:?} ms",
&metrics_consumer_data.input,
&metrics_item.input(),
start.elapsed().as_millis()
)
}
}));
}
});
let mut inputs_metrics = Vec::with_capacity(n_items);
for _ in 0..input_threads.len() {
match rx.recv() {
@ -121,11 +114,13 @@ fn generate_metrics(targets_list: Vec<MetricsConsumerTarget>, be_quiet: bool) {
for thread in input_threads {
let _ = thread.join();
}
inputs_metrics.iter_mut().for_each(|metrics_consumer_data| {
if !be_quiet {
println!("\n{}: ---------------\n", &metrics_consumer_data.input);
inputs_metrics.iter().for_each(|metrics_item| {
if let Some(metrics_data) = metrics_item {
if !be_quiet {
println!("\n{}: ---------------\n", &metrics_data.input());
}
let _ = metrics_data.show_metrics();
}
let _ = metrics_consumer_data.show_metrics();
});
}
/// Keep it short:
@ -135,7 +130,7 @@ fn generate_metrics(targets_list: Vec<MetricsConsumerTarget>, be_quiet: bool) {
/// - Create **targets list** as [`MetricsConsumerTarget`] vector
/// - Call to [`generate_metrics`] to do the job
/// - If not **be_quiet** mode print elapsed time in milliseconds
///
///
/// > This is not running async and not expect any **Result**.
fn main() {
let main_start = Instant::now();
@ -152,7 +147,7 @@ fn main() {
if !config.be_quiet {
println!("Loaded config from: {}", &args_settings.config_path);
}
let targets_list: Vec<MetricsConsumerTarget> = config
let targets_list = config
.targets
.iter()
.map(|item| MetricsConsumerTarget {

59
src/metrics_consumer.rs
View File

@ -1,6 +1,6 @@
//! ## MetricsConsumerTarget definitions and implementations
//! ## MetricsConsumerTarget definitions and implementations
//! Specific metric class **Consumer Metric** using generic metric operations <br>
//! Save source / result paths, parsing regular_expression and collect metrics values and their aggreates using [`MetricsConsumerData`]
//! Save source / result paths, parsing regular_expression and collect metrics values and their aggreates using [`MetricsConsumerData`]
//! From this modular approach other metrics classes can be defined and implemented
use regex::Regex;
@ -13,11 +13,11 @@ use std::{
use crate::defs::{MetricParser, MetricsConsumerData};
/// Attributes definition
/// Attributes definition
/// - **reg_exp** has some difficulties to be used with [regex](https://docs.rs/regex/latest/regex/), it works better with [string literals](https://doc.rust-lang.org/reference/expressions/literal-expr.html#string-literal-expressions)
/// - **metrics** and **aggregates** are vectors of [`MetricsConsumerData`] enums values to group input lines or save aggregates values
/// - **metrics** and **aggregates** have same type for implementation simplification, **aggregates** only use <u>first vector value</u> <br>
/// it can be easily used or extended to also save other computed values like: max, min, etc.
/// it can be easily used or extended to also save other computed values like: max, min, etc.
#[derive(Debug, Clone, Default)]
pub(crate) struct MetricsConsumerTarget {
pub input: String,
@ -28,11 +28,12 @@ pub(crate) struct MetricsConsumerTarget {
pub aggregates: Vec<MetricsConsumerData>,
}
/// Implement generic metrics operations / tasks for [`MetricsConsumerData`]
///
/// Implement generic metrics operations / tasks for [`MetricsConsumerData`]
///
impl MetricParser for MetricsConsumerTarget {
fn parse(&mut self, file: File, reg_exp: Regex) {
fn parse(&self, file: File, reg_exp: Regex) -> Self {
let buf = BufReader::new(file);
let mut consumer_target = self.clone();
buf.lines()
.enumerate()
.for_each(|(index, read_line)| match read_line {
@ -42,7 +43,7 @@ impl MetricParser for MetricsConsumerTarget {
let minute =
UNIX_EPOCH + Duration::from_secs((timestamp - (timestamp % 60)) as u64);
let mut not_found = true;
for metric in self.metrics.iter_mut() {
for metric in consumer_target.metrics.iter_mut() {
if metric.name() == name.to_lowercase() {
if let Some(metric_data) = metric.time_data() {
if metric_data.time == minute {
@ -54,11 +55,13 @@ impl MetricParser for MetricsConsumerTarget {
}
}
if not_found {
self.metrics.push(MetricsConsumerData::from_values(
&name,
minute,
Vec::from([value]),
));
consumer_target
.metrics
.push(MetricsConsumerData::from_values(
&name,
minute,
Vec::from([value]),
));
}
}
}
@ -66,16 +69,17 @@ impl MetricParser for MetricsConsumerTarget {
eprintln!("Error reading line {}: {}", index, e);
}
});
consumer_target
}
fn load_input(&mut self) -> Result<bool, Box<dyn Error>> {
fn load_input(&self) -> Result<Self, Box<dyn Error>> {
let file = File::open(&self.input)
.map_err(|err| format!("Error reading file: {} {}", &self.input, err))?;
if self.reg_exp.is_empty() {
return Err(String::from("Error invalid reg expression").into());
}
let reg_exp = Regex::new(&self.reg_exp)?;
self.parse(file, reg_exp);
Ok(true)
Ok(self.parse(file, reg_exp))
}
fn collect_aggregates(&mut self) {
self.metrics.iter().for_each(|metric_data| {
@ -105,10 +109,11 @@ impl MetricParser for MetricsConsumerTarget {
}
})
}
fn show_metrics(&mut self) -> Result<Vec<String>, std::io::Error> {
fn show_metrics(&self) -> Result<Vec<String>, std::io::Error> {
let mut output = Vec::new();
self.aggregates.sort();
self.aggregates.iter().for_each(|metric_data| {
let mut aggregates = self.aggregates.clone();
aggregates.sort();
aggregates.iter().for_each(|metric_data| {
if metric_data.time_data().is_some() {
let output_line = format!("{}", metric_data);
match self.output.as_str() {
@ -130,19 +135,29 @@ impl MetricParser for MetricsConsumerTarget {
let file = File::options().append(true).open(&self.output)?;
let mut writer = BufWriter::new(file);
writer.write_all(output.join("\n").as_bytes())?;
let text_overwrite = if self.out_overwrite { String::from ("overwriten")} else { String::from("") };
println!("Metrics for '{}' are saved in '{}' {}", &self.input, &self.output, &text_overwrite);
let text_overwrite = if self.out_overwrite {
String::from("overwriten")
} else {
String::from("")
};
println!(
"Metrics for '{}' are saved in '{}' {}",
&self.input, &self.output, &text_overwrite
);
}
};
Ok(output.to_owned())
}
fn input(&self) -> String {
self.input.to_string()
}
}
/// Specific implementations like **parse_line**
impl MetricsConsumerTarget {
fn show_invalid_line(&self, index: usize, line: &str) {
println!("invalid line: {} {}", index, line);
}
/// Check metric line values
/// Check metric line values
pub fn parse_line(&self, line: &str, index: usize, re: &Regex) -> Option<(i64, String, f64)> {
if let Some(caps) = re.captures(line) {
let timestamp = match caps[1].parse::<i64>() {

12
src/tests.rs
View File

@ -1,11 +1,11 @@
//! ## Tests
//! Some unitary tests grouped here <br>
//! [`test_expected_metrics`] the more important one to verify results with **output_expected.txt**
//!
//!
use super::*;
#[test]
fn test_load_input() -> Result<(), String> {
let mut metrics_target = MetricsConsumerTarget {
let metrics_target = MetricsConsumerTarget {
input: String::from(DEFAULT_INPUT_PATH),
reg_exp: String::from(DEFAULT_REG_EXP),
..Default::default()
@ -51,15 +51,15 @@ fn test_expected_metrics() {
fs::File,
io::{prelude::*, BufReader},
};
let mut metrics_target = MetricsConsumerTarget {
let metrics_target = MetricsConsumerTarget {
input: String::from(DEFAULT_INPUT_PATH),
output: String::from("vec"),
reg_exp: String::from(DEFAULT_REG_EXP),
..Default::default()
};
metrics_target.load_input().unwrap_or_default();
metrics_target.collect_aggregates();
let data_metrics = metrics_target.show_metrics().unwrap_or_default();
let mut result = metrics_target.load_input().unwrap_or_default();
result.collect_aggregates();
let data_metrics = result.show_metrics().unwrap_or_default();
let expected_output = String::from("output_expected.txt");
let file = File::open(expected_output.clone())