scott/statscollect
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

removed old script, updated collect.sh to include topstat.py

Browse Source
This commit is contained in:
Marc Mance
2024-05-30 17:06:30 -04:00
parent 47cb2b7987
commit 43e05d6c58
2 changed files with 1 additions and 300 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
collect.sh
View File

@@ -22,6 +22,7 @@ for (( i=1; i<=$iterations; i++ )); do
# Execute the command
python3 zpooliostat.py
python3 iostat.py
python3 topstat.py
echo "minute: $i"
# Wait for 60 seconds before the next iteration

300
collect.zpool.iostat.py
View File

@@ -1,300 +0,0 @@
import subprocess
import csv
import os
import time
import argparse
# Configuration parameters
MAX_FILE_SIZE = 1 * 1024 * 1024 * 1024 # 1 GiB in bytes
INTERVAL = 5 # Interval in seconds between zpool iostat commands
COUNT = 1 # Number of samples in each cycle
UPDATE_INTERVAL = 60 # Interval in seconds between updates
OUTPUT_FILE = "/root/zpool_iostat.csv"
# Placeholder for your other functions (load_existing_data, update_json_file, rotate_file_if_needed)
def update_csv_file(output_file, data):
"""
Writes data to a CSV file and rotates the file if necessary.
Parameters:
output_file (str): Path to the CSV file where data will be written.
data (list): The data to be written to the file.
"""
try:
print(f"Attempting to write data to {output_file}...")
with open(output_file, "w", newline="") as csv_file:
fieldnames = data[0].keys() if data else []
csv_writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
csv_writer.writeheader()
for row in data:
csv_writer.writerow(row)
print(f"Data successfully written to {output_file}")
rotate_file_if_needed(output_file)
except Exception as e:
print(f"Error writing to file: {e}")
def rotate_file_if_needed(file_path):
"""
Rotates the file if it exceeds a specified size. The existing file is renamed to '.old'.
Parameters:
file_path (str): Path to the file that should be rotated.
"""
try:
file_size = os.path.getsize(file_path)
if file_size > MAX_FILE_SIZE:
old_file = file_path + ".old"
if os.path.exists(old_file):
print(f"Removing old file: {old_file}")
os.remove(old_file)
print(f"Rotating file: {file_path} to {old_file}")
os.rename(file_path, old_file)
except OSError as e:
print(f"Error during file rotation: {e}")
def load_existing_data(file_path):
"""
Loads existing data from a CSV file.
Parameters:
file_path (str): Path to the CSV file.
Returns:
list: The data loaded from the file, or an empty list if the file doesn't exist or an error occurs.
"""
data = []
try:
if os.path.exists(file_path):
with open(file_path, "r") as file:
print(f"Loading existing data from {file_path}.")
csv_reader = csv.DictReader(file)
for row in csv_reader:
data.append(row)
else:
print(f"No existing file found at {file_path}. Starting fresh.")
except Exception as e:
print(f"Error reading CSV file: {e}. Starting with empty data.")
return data
def remove_k(value_with_unit):
"""
Removes the 'K' unit from a value, if present, from the IOPS section.
Parameters:
value_with_unit (str): The string containing the value and possibly a 'K' unit.
Returns:
float: The numerical value, with 'K' removed if it was present.
"""
if value_with_unit.endswith("K"):
return float(value_with_unit[:-1])
else:
return float(value_with_unit)
def convert_to_mib(value_with_unit):
"""
Converts a value with a unit (K, M, G, T) to MiB.
Parameters:
value_with_unit (str): The string containing the value and unit.
Returns:
float: The value converted to MiB, or 0 if the input is invalid.
"""
# Check if the string is valid
if not value_with_unit or not value_with_unit[-1].isalpha():
return 0.0
# Extract the unit and value
unit = value_with_unit[-1].upper() # Ensure unit is uppercase for consistency
try:
value = float(value_with_unit[:-1])
except ValueError:
return 0.0
# Convert based on the unit
if unit == "K":
return round(value / 1024.0, 1) # Convert KiB to MiB
elif unit == "M":
return round(value, 1) # Already in MiB
elif unit == "G":
return round(value * 1024.0, 1) # Convert GiB to MiB
elif unit == "T":
return round(value * 1024.0 * 1024.0, 1) # Convert TiB to MiB
else:
print(f"Warning: Unrecognized unit '{unit}' in value '{value_with_unit}'.")
return 0.0
def convert_time_to_ms(value_with_unit):
"""
Converts a time value with a unit (ms, us, ns) to milliseconds (ms), rounded to three decimal places.
Returns 0 if the input is '-', indicating no data.
Parameters:
value_with_unit (str): The string containing the value and unit.
Returns:
float: The value converted to milliseconds (ms), rounded to three decimal places, or 0 if the input is invalid or '-'.
"""
if value_with_unit == "-":
# Handle no data case
return 0
try:
# Strip and lower case to normalize
value_with_unit = value_with_unit.strip().lower()
# Check for nanoseconds
if value_with_unit.endswith("ns"):
value = float(value_with_unit[:-2])
return round(value / 1e6, 1) # Convert ns to ms and round
# Check for microseconds
elif value_with_unit.endswith("us"):
value = float(value_with_unit[:-2])
return round(value / 1000.0, 1) # Convert us to ms and round
# Check for milliseconds
elif value_with_unit.endswith("ms"):
value = float(value_with_unit[:-2])
return round(value, 3) # Already in ms and round
# If no known unit, assume it's in milliseconds and round
else:
return round(float(value_with_unit), 1)
except ValueError:
# Log the error if conversion fails
print(f"Warning: Unrecognized or malformed time value '{value_with_unit}'.")
return 0.0
def parse_iostat_output(output):
"""
Parses the output of the `zpool iostat` command to include pool, vdev, disk level data,
and does not specifically skip the 'logs' line.
Parameters:
output (str): The output string from the `zpool iostat` command.
Returns:
list: A list of dictionaries, each representing an entity (pool, vdev, or disk) and its metrics.
"""
parsed_data = []
lines = output.strip().split("\n")
data_started = False
for line in lines:
if "capacity" in line and "operations" in line and "bandwidth" in line:
# This line indicates the end of the headers and the start of relevant data.
data_started = True
continue # Skip the header line itself
if data_started and "-----" in line or line.startswith("pool"):
# Skip separator lines and the column headers repeated in the output.
continue
if data_started:
# Process lines containing pool, vdev, and disk level data.
fields = line.split()
if len(fields) >= 7:
try:
# Check for numeric data in expected positions to avoid conversion errors.
if any(
not field.replace(".", "", 1).isdigit()
and not field.endswith("K")
for field in fields[3:6]
):
# Skip lines that do not contain numeric data where expected.
continue
data_entry = {
"time": time.strftime("%Y-%m-%dT%H:%M:%SZ"),
"name": fields[0],
"alloc": fields[1],
"free": fields[2],
"ops_read": remove_k(fields[3]),
"ops_write": remove_k(fields[4]),
"bandwidth_read": convert_to_mib(fields[5]),
"bandwidth_write": convert_to_mib(fields[6]),
"total_wait_read_ms": convert_time_to_ms(fields[7])
if len(fields) > 7
else 0,
"total_wait_write_ms": convert_time_to_ms(fields[8])
if len(fields) > 8
else 0,
"disk_wait_read_ms": convert_time_to_ms(fields[9])
if len(fields) > 9
else 0,
"disk_wait_write_ms": convert_time_to_ms(fields[10])
if len(fields) > 10
else 0,
"syncq_wait_read_ms": convert_time_to_ms(fields[11])
if len(fields) > 11
else 0,
"syncq_wait_write_ms": convert_time_to_ms(fields[12])
if len(fields) > 12
else 0,
"asyncq_wait_read_ms": convert_time_to_ms(fields[13])
if len(fields) > 13
else 0,
"asyncq_wait_write_ms": convert_time_to_ms(fields[14])
if len(fields) > 14
else 0,
"scrub_wait_ms": convert_time_to_ms(fields[15])
if len(fields) > 15 and fields[15] != "-"
else 0,
"trim_wait_ms": convert_time_to_ms(fields[16])
if len(fields) > 16 and fields[16] != "-"
else 0,
"rebuild_wait_ms": convert_time_to_ms(fields[17])
if len(fields) > 17 and fields[17] != "-"
else 0,
}
parsed_data.append(data_entry)
except Exception as e:
print(f"Error processing line: '{line}' - {e}")
return parsed_data
def run_zpool_iostat(interval, count, output_file, update_interval, TIMETORUN):
"""
Main function to run the zpool iostat command at regular intervals, parse its output, and update the CSV file.
Parameters:
interval (int): Interval in seconds between each command execution.
count (int): Number of samples to collect in each cycle.
output_file (str): Path to the output CSV file.
update_interval (int): Interval in seconds between updates to the CSV file.
"""
while not TIMETORUN == 0:
all_data = load_existing_data(output_file)
for _ in range(count):
command = "zpool iostat -vvyl 15 1"
try:
print("Running zpool iostat command...")
result = subprocess.run(
command, shell=True, capture_output=True, text=True
)
if result.returncode != 0:
print(f"Error running zpool iostat: {result.stderr}")
break
else:
parsed_data = parse_iostat_output(result.stdout)
if parsed_data:
all_data.extend(parsed_data)
else:
print("Error in parsing data. Skipping this sample.")
print("zpool iostat command executed successfully.")
except Exception as e:
print(f"Subprocess error occurred: {e}")
break
print(f"Completed sample {_+1}. Waiting for the next sample...")
time.sleep(interval)
update_csv_file(output_file, all_data)
print(
f"Completed data collection cycle. Waiting for the next update in {update_interval} seconds."
)
time.sleep(update_interval)
TIMETORUN = TIMETORUN - 1
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Collect for X minutes.")
parser.add_argument(
"--minutes", type=int, help="an integer for collected minutes", required=True
)
args = parser.parse_args()
TIMETORUN = args.minutes
try:
run_zpool_iostat(INTERVAL, COUNT, OUTPUT_FILE, UPDATE_INTERVAL, TIMETORUN)
except Exception as e:
print(f"An error occurred: {e}")