diff --git a/collect.py b/collect.py index 4a6b57a..b664c4c 100644 --- a/collect.py +++ b/collect.py @@ -6,6 +6,7 @@ import os import shutil import socket import platform +import tarfile from datetime import datetime @@ -16,10 +17,10 @@ def is_freebsd(): def is_debian(): # Debian-based systems often have 'debian' in their platform string - return "debian" in platform.platform().lower() + return "linux" in platform.platform().lower() -def scaleIfstat_disk(): +def scaleIostat_disk(): command = ["iostat", "-xyd", "1", "1"] headerRow = [ "timestamp", @@ -48,7 +49,24 @@ def scaleIfstat_disk(): "%utilread_s", ] - filename = "iostat.csv" + filename = "ioStat.csv" + collect = runCollect(command) + + if collect: + byline = re.split("\n", collect.strip()) + data = byline[4:] + + for line in data: + lineData = line.split() + + if lineData: + with open(filename, "a", newline="") as csvfile: + csv_writer = csv.writer(csvfile) + lineData.insert(0, getTimestamp()) + csv_writer.writerow(lineData) + + else: + print(f"Error running command: {collect}") def getTimestamp(): @@ -69,7 +87,7 @@ def runCollect(command): return result.stderr -def scaleIfstat_cpu(): +def scaleIostat_cpu(): command = ["iostat", "-c", "1", "1"] headerRow = [ @@ -88,8 +106,8 @@ def scaleIfstat_cpu(): collect = runCollect(command) if collect: - byline = re.split("\n", collect) - data = byline[2] + byline = re.split("\n", collect.strip()) + data = byline[3].strip() lineData = data.split() if lineData: @@ -103,7 +121,7 @@ def scaleIfstat_cpu(): print(f"Error running command: {collect}") -def coreIfstat(): +def ifstat(): command = ["ifstat", "-znq", "1", "1"] headerRow = [ "timestamp", @@ -207,19 +225,31 @@ def collect_data(minutes, interval): start_time = datetime.now() # Continuously collect data for the specified duration + gstat_command = ["gstat", "-C", "-s", "-d", "-o", "-p", "-I", "5s"] + if is_freebsd(): + with open("gstat.csv", "a") as output_file: + # Create a Popen object with stdout redirected to the file + process = subprocess.Popen(gstat_command, stdout=output_file) + for _ in range(minutes * 60 // interval): # Collect data using system commands zpoolIostat() + ifstat() - if is_freebsd(): - coreIfstat() + if is_debian(): + scaleIostat_cpu() + scaleIostat_disk() # Print progress indicator elapsed_time = (datetime.now() - start_time).total_seconds() // interval print(f"Minute {elapsed_time + 1}", flush=True) time.sleep(interval) + # kill gstat if freebsd + if is_freebsd(): + process.kill() + def run_debug(): print("Capturing new debug.") @@ -232,9 +262,35 @@ def run_debug(): return None +# def collect_csv(): +# source_dir = os.getcwd() +# tar_file_name = "dpkg.log" # Choose a desired name for the tar file +# tar_file_path = os.path.join(source_dir, tar_file_name) + +# with tarfile.open(tar_file_path, "w:gz") as tar: +# for filename in os.listdir(source_dir): +# if filename.endswith(".csv"): +# source_file = os.path.join(source_dir, filename) + +# try: +# tar.add(source_file, arcname=os.path.basename(source_file)) +# print(f"Added '{filename}' to tar archive successfully.") +# except FileNotFoundError: +# print(f"Error: File '{filename}' not found in '{source_dir}'.") +# except Exception as e: # Catch other potential errors +# print(f"Error adding '{filename}' to tar archive: {e}") + +# return tar_file_path + + def collect_csv(): source_dir = os.getcwd() - destination_dir = "/var/log" + + if is_debian(): + destination_dir = "/var/log/proftpd" + + if is_freebsd(): + destination_dir = "/var/log" for filename in os.listdir(source_dir): if filename.endswith(".csv"): # Check for .csv extension @@ -276,26 +332,32 @@ def upload_debug(): ) -def main(): - minutes = int(input("Enter the duration for data collection (minutes): ")) - interval = 60 # Default interval between data collection cycles (seconds) +def welcome(): + print("########################################################################") + print("# FreeNAS CORE/SCALE performance capture script v.02 #") + print("########################################################################") + print("# This script runs for x minutes and collects cpu/disk/network stats #") + print("# When it is completed, it will copy csv files to your /var/log folder #") + print("# It will then attempt to take a debug and upload both to our FTP #") + print("# If not connected to the internet, it will have to be manually d/l #") + print("########################################################################") + print("# Running this script repeatedly will append results to CSV files #") + print("# - https://gitlab.komputernerds.com/mmance/statscollect - #") + print("########################################################################") + print("") - gstat_command = ["gstat", "-C", "-s", "-d", "-o", "-p", "-I", "5s"] + +def main(): + welcome() + + minutes = int(input("Enter the duration in minutes: ")) + interval = 60 # Default interval between data collection cycles (seconds) print("Starting Collection") - if is_freebsd(): - with open("gstat.csv", "a") as output_file: - # Create a Popen object with stdout redirected to the file - process = subprocess.Popen(gstat_command, stdout=output_file) - # Collect data collect_data(minutes, interval) - # kill gstat if freebsd - if is_freebsd(): - process.kill() - # Copy data files to /var/log (replace with appropriate copying function) collect_csv()