From 06f9a4add6c75c64551e4006da25c9a4b0011532 Mon Sep 17 00:00:00 2001 From: Chris Holcombe Date: Thu, 7 Feb 2019 11:15:06 -0800 Subject: [PATCH] Pool and volume creation for hitachi --- src/hitachi.rs | 592 +++++++++++++++++++++++++++++++++++++- tests/hitachi/pool.json | 71 +++++ tests/hitachi/volume.json | 78 +++++ 3 files changed, 738 insertions(+), 3 deletions(-) create mode 100644 tests/hitachi/pool.json create mode 100644 tests/hitachi/volume.json diff --git a/src/hitachi.rs b/src/hitachi.rs index f7b6dc8..15097cd 100644 --- a/src/hitachi.rs +++ b/src/hitachi.rs @@ -2,6 +2,7 @@ use crate::error::{MetricsResult, StorageError}; use std::collections::HashMap; +use std::fmt; use std::str; use std::str::FromStr; @@ -10,20 +11,444 @@ use chrono::offset::Utc; use chrono::DateTime; use csv::Reader; use reqwest::header::ACCEPT; +use serde_json::{Number, Value}; + +#[serde(rename_all = "UPPERCASE")] +#[derive(Deserialize, Debug)] +pub enum BlockingMode { + /// Full or blockade + Fb, + /// No blocking + Nb, + /// Pool full + Pf, + /// Pool vol Blockade + Pb, +} + +impl fmt::Display for BlockingMode { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + BlockingMode::Fb => write!(f, "FB"), + BlockingMode::Nb => write!(f, "NB"), + BlockingMode::Pb => write!(f, "PB"), + BlockingMode::Pf => write!(f, "PF"), + } + } +} #[derive(Deserialize, Debug)] pub struct Collection { pub items: Vec>, } +#[serde(rename_all = "snake_case")] +#[derive(Debug, Serialize)] +pub enum DataReductionMode { + Compression, + CompressionDeduduplication, + Disabled, +} + +impl fmt::Display for DataReductionMode { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + DataReductionMode::Compression => write!(f, "compression"), + DataReductionMode::CompressionDeduduplication => write!(f, "compression_deduplication"), + DataReductionMode::Disabled => write!(f, "disabled"), + } + } +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "UPPERCASE")] +pub enum DriveType { + Sas, + Sata, + #[serde(rename = "SSD(FMC)")] + SsdFmc, + #[serde(rename = "SSD(FMD)")] + SsdFmd, + #[serde(rename = "SSD(MLC)")] + SsdMlc, + #[serde(rename = "SSD(SLC)")] + SsdSlc, + Ssd, +} + +impl fmt::Display for DriveType { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + DriveType::Sas => write!(f, "SAS"), + DriveType::Sata => write!(f, "SATA"), + DriveType::SsdFmc => write!(f, "SSD(FMC)"), + DriveType::SsdFmd => write!(f, "SSD(FMD)"), + DriveType::SsdMlc => write!(f, "SSD(MLC)"), + DriveType::SsdSlc => write!(f, "SSD(SLC)"), + DriveType::Ssd => write!(f, "SSD"), + } + } +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "UPPERCASE")] +pub enum DetailInfoType { + Fmc, + #[serde(rename = "SSD(FMC)")] + SsdFmc, +} + +impl fmt::Display for DetailInfoType { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + DetailInfoType::Fmc => write!(f, "FMC"), + DetailInfoType::SsdFmc => write!(f, "SSD(FMC)"), + } + } +} +/* +#[derive(Debug, Deserialize)] +pub struct RaidType { + pub data: (u8, Option), + pub parity: Option, + /* + Specify one of the following values: + 2D+2D + 3D+1P + 4D+1P + 6D+1P + 7D+1P + 6D+2P + 12D+2P + 14D+2P + */ +} + +impl serde::Serialize for RaidType { +fn serialize(&self, serializer: S) -> Result +where +S: serde::ser::Serializer, +{ +//serializer.serialize_str(format!("{}D+{}P")) +} +} +*/ + +#[derive(Deserialize, Debug)] +pub struct ParityGroup { + parityGroupId: String, + numOfLdevs: i32, + usedCapacityRate: i32, + availableVolumeCapacity: u64, + raidLevel: String, + raidType: String, + clprId: i32, + driveType: String, + driveTypeName: DriveType, + driveSpeed: i32, + totalCapacity: u64, + physicalCapacity: u64, + isAcceleratedCompressionEnabled: bool, +} + +#[derive(Deserialize, Debug)] +pub struct ParityGroupResp { + pub data: Vec, +} + +#[serde(rename_all = "camelCase")] +#[derive(Debug, Deserialize)] +pub struct DataIncludingSystemDatum { + pub is_reduction_capacity_available: bool, + pub is_reduction_rate_available: bool, + pub reduction_capacity: u64, + pub reduction_rate: i64, +} + +#[serde(rename_all = "camelCase")] +#[derive(Debug, Deserialize)] +pub struct DataExcludingSystemDatum { + pub compressed_capacity: u64, + pub deduped_capacity: u64, + pub pre_used_capacity: u64, + pub pre_compressed_capacity: u64, + pub pre_dedupred_capacity: u64, + pub reclaimed_capacity: u64, + pub system_data_capacity: u64, + pub used_virtual_volume_capacity: u64, +} + +#[serde(rename_all = "camelCase")] +#[derive(Deserialize, Debug)] +pub struct DataPort { + pub port_id: String, + pub host_group_number: i64, + pub host_group_name: String, + pub lun: i64, +} + +#[serde(rename_all = "camelCase")] +#[derive(Debug, Deserialize)] +pub struct Datum { + pub available_physical_volume_capacity: u64, + pub available_volume_capacity: u64, + pub blocking_mode: BlockingMode, + pub capacities_excluding_system_data: DataExcludingSystemDatum, + pub compression_rate: i64, + pub data_reduction_accelerate_comp_capacity: u64, + pub data_reduction_accelerate_comp_including_system_data: DataIncludingSystemDatum, + pub data_reduction_accelerate_comp_rate: i64, + pub data_reduction_before_capacity: u64, + pub data_reduction_capacity: u64, + pub data_reduction_including_system_data: DataIncludingSystemDatum, + pub data_reduction_rate: i64, + pub depletion_threshold: i64, + pub duplication_ldev_ids: Vec, + pub duplication_number: i64, + pub duplication_rate: i64, + pub first_ldev_id: i64, + pub is_mainframe: bool, + pub is_shrinking: bool, + pub located_volume_count: i64, + pub num_of_ldevs: i64, + pub pool_id: i64, + pub pool_name: String, + pub pool_status: PoolStatus, + pub pool_type: PoolType, + pub reserved_volume_count: i64, + pub snapshot_count: i64, + /// Total size of snapshot data mapped to pool (MB) + pub snapshot_used_capacity: u64, + pub suspend_snapshot: bool, + /// Total capacity of the DP Volumes mapped to the pool (MB) + pub total_located_capacity: u64, + pub total_physical_capacity: u64, + pub total_pool_capacity: u64, + pub total_reserved_capacity: u64, + /// Usage rate of logical capacity in % + pub used_capacity_rate: i64, + /// Usage rate of physical capacity in % + pub used_physical_capacity_rate: i64, + pub virtual_volume_capacity_rate: i64, + pub warning_threshold: i64, +} + +#[derive(Debug)] +pub enum LdevOption { + /// Gets information about implemented LDEVs. + Defined, + /// Gets information about DP volumes. + DpVolume, + /// Gets information about external volumes. + ExternalVolume, + ///Gets information about LDEVs for which LU paths are defined. + LuMapped, + /// Gets information about LDEVs for which LU paths are undefined. + LuUnmapped, + /// Gets information about LDEVs that are not implemented. + Undefined, +} + +impl fmt::Display for LdevOption { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + LdevOption::Defined => write!(f, "defined"), + LdevOption::DpVolume => write!(f, "dpVolume"), + LdevOption::ExternalVolume => write!(f, "externalVolume"), + LdevOption::LuMapped => write!(f, "luMapped"), + LdevOption::LuUnmapped => write!(f, "luUnmapped"), + LdevOption::Undefined => write!(f, "undefined"), + } + } +} + +#[derive(Debug, Deserialize)] +pub struct Pools { + pub data: Vec, +} +#[serde(rename_all = "UPPERCASE")] +#[derive(Debug, Deserialize)] +pub enum PoolStatus { + /// Pool is normal + PolN, + /// The pool is in overflow status exceeding the threshold + PolF, + /// The pool is in overflow status exceeding the threshold and is suspended + PolS, + /// The pool is suspended in failure status. If pool is POLE then + /// info cannot be obtained + PolE, +} + +impl fmt::Display for PoolStatus { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + PoolStatus::PolN => write!(f, "POLN"), + PoolStatus::PolF => write!(f, "POLF"), + PoolStatus::PolS => write!(f, "POLS"), + PoolStatus::PolE => write!(f, "POLE"), + } + } +} + +#[serde(rename_all = "UPPERCASE")] +#[derive(Debug, Deserialize)] +pub enum PoolType { + /// Data direct mapped HDP pool + Dm, + /// Dynamic Provisioning + Dp, + Hdp, + Hdt, + /// Hitachi thin image + Hti, + /// Active flash pool + Rt, +} + +impl fmt::Display for PoolType { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + PoolType::Dm => write!(f, "DM"), + PoolType::Dp => write!(f, "DP"), + PoolType::Hdp => write!(f, "HDP"), + PoolType::Hdt => write!(f, "HDT"), + PoolType::Hti => write!(f, "HTI"), + PoolType::Rt => write!(f, "RT"), + } + } +} + #[derive(Deserialize, Debug)] pub struct Version { - pub productName: String, - pub productVersion: String, - pub apiVersion: String, + pub product_name: String, + pub product_version: String, + pub api_version: String, pub description: String, } +#[serde(rename_all = "UPPERCASE")] +#[derive(Deserialize, Debug)] +pub enum LdevAttribute { + /// Volume Migration volume + Alun, + /// Cache LUN (DCR) + Clun, + /// Command device + Cmd, + /// CVS volume + Cvs, + /// External volume + Elun, + /// Encrypted disk + Encd, + /// global-active device volume + Gad, + /// HDP volume or Dynamic Provisioning for Mainframe volume + Hdp, + /// HDT volume + Hdt, + /// Volume used as the system LU of NAS Platform + Hnass, + /// Volume used as a user LU of NAS Platform + Hnasu, + /// Pair volume (P-VOL or S-VOL) for + /// remote copy (TrueCopy,TrueCopy for + /// Mainframe, Universal Replicator, Universal + /// Replicator for Mainframe) + Horc, + /// Thin Image volume (P-VOL or S-VOL) + Hti, + /// Journal volume + Jnl, + /// ShadowImage volume (P-VOL or S-VOL) + Mrcf, + /// OpenLDEV Guard volume + Olg, + /// Pool volume + Pool, + /// Quorum disk + Qrd, + /// Remote command device + Rcmd, + /// System disk + Sysd, + /// Volume for which the T10 PI attribute is enabled + T10pi, + /// HDP volume used for FCSE + Tse, + /// Virtual volume + Vvol, +} + +#[serde(rename_all = "camelCase")] +#[derive(Deserialize, Debug)] +pub struct VolumeDatum { + pub attributes: Vec, + pub block_capacity: i64, + pub byte_format_capacity: String, + pub clpr_id: i64, + pub data_reduction_mode: String, + pub data_reduction_status: String, + pub emulation_type: String, + pub is_full_allocation_enabled: bool, + pub is_alua_enabled: bool, + pub label: String, + pub ldev_id: i64, + pub mp_blade_id: i64, + pub num_of_ports: i64, + pub num_of_used_block: i64, + pub operation_type: Option, + pub pool_id: i64, + pub ports: Vec, + pub resource_group_id: i64, + pub ssid: String, + pub status: VolumeStatus, +} + +#[serde(rename_all = "UPPERCASE")] +#[derive(Deserialize, Debug)] +pub enum VolumeOperationType { + /// Collection access is in progress. + Caccs, + /// Collection copying is in progress. + Ccopy, + /// Formatting is in progress. + Fmt, + /// Quick formatting is in progress. + Qfmt, + /// Pools are being rebalanced. + Rbl, + /// Pools are being reallocated. + Rlc, + /// Shredding is in progress. + Shred, + ///Deletion from the pool is in progress. + Shrpl, + /// Pages are being released. + Zpd, +} + +#[serde(rename_all = "UPPERCASE")] +#[derive(Deserialize, Debug)] +pub enum VolumeStatus { + /// The LDEV is blocked. + Blk, + /// The LDEV status is being changed. + Bsy, + /// The LDEV is in normal status. + Nml, + /// The LDEV status is unknown (not supported). + #[serde(rename = "Unknown")] + Unknown, +} + +#[serde(rename_all = "camelCase")] +#[derive(Deserialize, Debug)] +pub struct Volumes { + pub data: Vec, +} + /// Convert a comma separated value vec of ByteRecords into timescaledb Value's /// The headers field is used to figure out which type to convert into fn into_values( @@ -312,6 +737,20 @@ fn test_nas_pi_hnhs() { println!("Result: {:?}", result); } +#[test] +fn test_pools() { + let f = include_str!("../tests/hitachi/pool.json"); + let v: Pools = serde_json::from_str(f).unwrap(); + println!("Result: {:?}", v); +} + +#[test] +fn test_volumes() { + let f = include_str!("../tests/hitachi/volume.json"); + let v: Volumes = serde_json::from_str(f).unwrap(); + println!("Result: {:?}", v); +} + #[derive(Clone, Deserialize, Debug)] pub struct HitachiConfig { /// The hitachi endpoint to use @@ -322,6 +761,44 @@ pub struct HitachiConfig { pub region: String, } +/// Capacity is specified in GB +pub fn create_pool( + client: &reqwest::Client, + config: &HitachiConfig, + storage_device_id: &str, + capacity: u64, + ldev_id: Option, + parity_group_id: Option<&str>, + pool_id: Option, + data_reduction_mode: Option, +) -> MetricsResult<()> { + let api_call = format!("v1/objects/storages/{}/ldevs", storage_device_id); + let mut params: HashMap = HashMap::new(); + params.insert( + "byteFormatCapacity".to_string(), + Value::String(format!("{}G", capacity)), + ); + if let Some(ldev_id) = ldev_id { + params.insert("ldevId".to_string(), Value::Number(::from(ldev_id))); + } + if let Some(parity_group_id) = parity_group_id { + params.insert( + "parityGroupId".to_string(), + Value::String(parity_group_id.to_string()), + ); + } + if let Some(pool_id) = pool_id { + params.insert("poolId".to_string(), Value::Number(::from(pool_id))); + } + if let Some(data_mode) = data_reduction_mode { + params.insert( + "dataReductionMode".to_string(), + Value::String(format!("{}", data_mode)), + ); + } + post_command_server_response(&client, &config, &api_call, &serde_json::to_value(params)?)?; + Ok(()) +} /// This request obtains the detailed version of the API pub fn get_version(client: &reqwest::Client, config: &HitachiConfig) -> MetricsResult { let version: Version = client @@ -374,6 +851,52 @@ pub fn get_agent_for_nas( Ok(agentnas) } +fn get_command_server_response( + client: &reqwest::Client, + config: &HitachiConfig, + api_call: &str, + params: Option<&HashMap>, +) -> MetricsResult +where + T: serde::de::DeserializeOwned, +{ + let content: T = if let Some(params) = params { + client + .get(&format!("http://{}/{}", config.endpoint, api_call)) + .query(params) + .basic_auth(&config.user, Some(&config.password)) + .send()? + .error_for_status()? + .json()? + } else { + client + .get(&format!("http://{}/{}", config.endpoint, api_call)) + .basic_auth(&config.user, Some(&config.password)) + .send()? + .error_for_status()? + .json()? + }; + Ok(content) +} + +fn post_command_server_response( + client: &reqwest::Client, + config: &HitachiConfig, + api_call: &str, + body: &B, +) -> MetricsResult<()> +where + B: serde::ser::Serialize, +{ + client + .post(&format!("http://{}/{}", config.endpoint, api_call)) + .basic_auth(&config.user, Some(&config.password)) + .json(body) + .send()? + .error_for_status()?; + Ok(()) +} + fn get_server_response( client: &reqwest::Client, config: &HitachiConfig, @@ -604,3 +1127,66 @@ pub fn get_nas_pi_hnhs( let points = csv_to_points(&result, "nas_pi_hnhs", Some(t))?; Ok(points) } + +pub fn get_parity_groups( + client: &reqwest::Client, + config: &HitachiConfig, + storage_device_id: &str, + clpr: Option, + drive_type: Option, + detail_info_type: Option, +) -> MetricsResult> { + let api_call = format!("v1/objects/storages/{}/parity-groups", storage_device_id); + let mut params: HashMap = HashMap::new(); + if let Some(clpr) = clpr { + params.insert("clprId".into(), clpr.to_string()); + } + if let Some(detail_info) = detail_info_type { + params.insert("detailInfoType".into(), detail_info.to_string()); + } + if let Some(drive_type) = drive_type { + params.insert("driveTypeName".into(), drive_type.to_string()); + } + let result: Vec = + get_command_server_response(&client, &config, &api_call, Some(¶ms))?; + Ok(result) +} + +pub fn get_pools( + client: &reqwest::Client, + config: &HitachiConfig, + storage_device_id: &str, + pool_type: Option, +) -> MetricsResult { + let api_call = format!("v1/objects/storages/{}/pools", storage_device_id); + let mut params: HashMap = HashMap::new(); + if let Some(pool_type) = pool_type { + params.insert("poolType".into(), pool_type.to_string()); + } + let result: Pools = get_command_server_response(&client, &config, &api_call, Some(¶ms))?; + Ok(result) +} + +pub fn get_volumes( + client: &reqwest::Client, + config: &HitachiConfig, + storage_device_id: &str, + count: Option, + // Default to all types + ldev_option: Option, + pool_id: Option, +) -> MetricsResult { + let api_call = format!("v1/objects/storages/{}/ldevs", storage_device_id); + let mut params: HashMap = HashMap::new(); + if let Some(count) = count { + params.insert("count".into(), count.to_string()); + } + if let Some(ldev_option) = ldev_option { + params.insert("ldevOption".into(), ldev_option.to_string()); + } + if let Some(pool_id) = pool_id { + params.insert("poolId".into(), pool_id.to_string()); + } + let result: Volumes = get_command_server_response(&client, &config, &api_call, Some(¶ms))?; + Ok(result) +} diff --git a/tests/hitachi/pool.json b/tests/hitachi/pool.json new file mode 100644 index 0000000..36348ac --- /dev/null +++ b/tests/hitachi/pool.json @@ -0,0 +1,71 @@ +{ + "data": [ + { + "poolId": 5, + "poolStatus": "POLN", + "usedCapacityRate": 1, + "usedPhysicalCapacityRate": 1, + "snapshotCount": 0, + "poolName": "pool_5", + "availableVolumeCapacity": 32042850, + "availablePhysicalVolumeCapacity": 20006364, + "totalPoolCapacity": 32066496, + "totalPhysicalCapacity": 20009724, + "numOfLdevs": 11, + "firstLdevId": 2304, + "warningThreshold": 70, + "depletionThreshold": 80, + "virtualVolumeCapacityRate": -1, + "isMainframe": false, + "isShrinking": false, + "locatedVolumeCount": 13, + "totalLocatedCapacity": 79951368, + "blockingMode": "NB", + "totalReservedCapacity": 0, + "reservedVolumeCount": 0, + "poolType": "HDP", + "duplicationLdevIds": [ + 65269, + 65268, + 65267, + 65266, + 65265, + 65264, + 65263, + 65262 + ], + "duplicationNumber": 8, + "dataReductionAccelerateCompCapacity": 206783585, + "dataReductionCapacity": 205901472, + "dataReductionBeforeCapacity": 210117216, + "dataReductionAccelerateCompRate": 87, + "duplicationRate": 42, + "compressionRate": 44, + "dataReductionRate": 97, + "dataReductionAccelerateCompIncludingSystemData": { + "isReductionCapacityAvailable": true, + "reductionCapacity": 228372480, + "isReductionRateAvailable": true, + "reductionRate": 97 + }, + "dataReductionIncludingSystemData": { + "isReductionCapacityAvailable": true, + "reductionCapacity": 186826752, + "isReductionRateAvailable": true, + "reductionRate": 79 + }, + "snapshotUsedCapacity": 0, + "suspendSnapshot": true, + "capacitiesExcludingSystemData": { + "usedVirtualVolumeCapacity": 235253760, + "compressedCapacity": 0, + "dedupedCapacity": 101035296, + "reclaimedCapacity": 129142560, + "systemDataCapacity": 43351104, + "preUsedCapacity": 234393600, + "preCompressedCapacity": 0, + "preDedupredCapacity": 105247408 + } + } + ] +} \ No newline at end of file diff --git a/tests/hitachi/volume.json b/tests/hitachi/volume.json new file mode 100644 index 0000000..6839183 --- /dev/null +++ b/tests/hitachi/volume.json @@ -0,0 +1,78 @@ +{ + "data": [ + { + "ldevId": 0, + "clprId": 0, + "emulationType": "OPEN-V-CVS", + "byteFormatCapacity": "1.00 G", + "blockCapacity": 2097152, + "numOfPorts": 2, + "ports": [ + { + "portId": "CL1-A", + "hostGroupNumber": 0, + "hostGroupName": "1A-G00", + "lun": 1 + }, + { + "portId": "CL2-A", + "hostGroupNumber": 0, + "hostGroupName": "2A-G00", + "lun": 1 + } + ], + "attributes": [ + "CVS", + "HDP" + ], + "label": "JH-26216_DP", + "status": "NML", + "mpBladeId": 2, + "ssid": "0012", + "poolId": 63, + "numOfUsedBlock": 86016, + "isFullAllocationEnabled": false, + "resourceGroupId": 0, + "dataReductionStatus": "ENABLED", + "dataReductionMode": "compression_deduplication", + "isAluaEnabled": false + }, + { + "ldevId": 1, + "clprId": 0, + "emulationType": "OPEN-V-CVS", + "byteFormatCapacity": "1.00 G", + "blockCapacity": 2097152, + "numOfPorts": 2, + "ports": [ + { + "portId": "CL1-A", + "hostGroupNumber": 0, + "hostGroupName": "1A-G00", + "lun": 2 + }, + { + "portId": "CL2-A", + "hostGroupNumber": 0, + "hostGroupName": "2A-G00", + "lun": 2 + } + ], + "attributes": [ + "CVS", + "HDP" + ], + "label": "JH-26216_DP", + "status": "NML", + "mpBladeId": 0, + "ssid": "0012", + "poolId": 63, + "numOfUsedBlock": 0, + "isFullAllocationEnabled": false, + "resourceGroupId": 0, + "dataReductionStatus": "DISABLED", + "dataReductionMode": "disabled", + "isAluaEnabled": false + } + ] +} \ No newline at end of file