import subprocess import time import re import csv import os import shutil import socket import platform from datetime import datetime def is_freebsd(): return platform.system() == "FreeBSD" def is_debian(): # Debian-based systems often have 'debian' in their platform string return "debian" in platform.platform().lower() def scaleIfstat_disk(): command = ["iostat", "-xyd", "1", "1"] headerRow = [ "timestamp", "name", "read_s", "rkB_s", "rrqm_s", "%rrqm", "r_await", "rareq-sz", "w_s", "wkB_s", "wrqm_s", "%wrqm", "w_await", "wareq-sz", "d_s", "dkB_s", "drqm_s", "%drqm", "d_await", "dareq-sz", "f_s", "f_await", "aqu-sz", "%utilread_s", ] filename = "iostat.csv" def getTimestamp(): timestamp = time.time() local_time = time.localtime(timestamp) # Format the time components for a more readable output return time.strftime("%Y-%m-%d %H:%M:%S", local_time) def runCollect(command): # Run the command and capture output result = subprocess.run(command, capture_output=True, text=True) # Check the return code (0 for success) if result.returncode == 0: # Access the captured output as a string return result.stdout return result.stderr def scaleIfstat_cpu(): command = ["iostat", "-c", "1", "1"] headerRow = [ "timestamp", "name", "%user", "%nice", "%system", "%iowait", "%steal", "%idle", ] filename = "cpuStat.csv" collect = runCollect(command) if collect: byline = re.split("\n", collect) data = byline[2] lineData = data.split() if lineData: with open(filename, "a", newline="") as csvfile: csv_writer = csv.writer(csvfile) lineData.insert(0, "cpu") lineData.insert(0, getTimestamp()) csv_writer.writerow(lineData) else: print(f"Error running command: {collect}") def coreIfstat(): command = ["ifstat", "-znq", "1", "1"] headerRow = [ "timestamp", "name", "Kbps_in", "Kbps_out", ] filename = "ifStat.csv" collect = runCollect(command) if collect: # process collection string bylines = collect.split("\n") interfaces = bylines[0].split() stats = bylines[2].split() for nic in interfaces: lineData = [getTimestamp(), nic, stats.pop(0), stats.pop(0)] if lineData: with open(filename, "a", newline="") as csvfile: csv_writer = csv.writer(csvfile) csv_writer.writerow(lineData) else: print(f"Error running command: {collect}") def zpoolIostat(): command = ["zpool", "iostat", "-Tu", "-l", "-p", "-v", "-y", "15", "1"] headerRow = [ "timestamp", "name", "capacity_alloc", "capacity_free", "ops_read", "ops_write", "bandwidth_read", "bandwidth_write", "total_wait_read", "total_wait_write", "disk_wait_read", "disk_wait_write", "syncq_wait_read", "syncq_wait_write", "asyncq_wait_read", "asyncq_wait_write", "scrub_wait", "trim_wait", "rebuld_wait", ] collect = runCollect(command) if collect: # split output by pools pools = re.split(r"^-", collect, flags=re.MULTILINE) i = 0 for pool in pools: if i == 0: pass else: # remove ------ lines pool = re.sub("^---.*", "", pool).strip() # turn - values into 0 pool = re.sub(" -", "0", pool) y = 0 poolbyline = re.split("\n", pool) for line in poolbyline: lineData = line.split() if not lineData: break if y == 0: poolname = lineData[0] filename = poolname + "-zio.csv" # Now open the file in append mode ('a') with open(filename, "a", newline="") as csvfile: csv_writer = csv.writer(csvfile) lineData.insert(0, getTimestamp()) csv_writer.writerow(lineData) y += 1 i += 1 else: print(f"Error running command: {collect}") def collect_data(minutes, interval): start_time = datetime.now() # Continuously collect data for the specified duration for _ in range(minutes * 60 // interval): # Collect data using system commands zpoolIostat() if is_freebsd(): coreIfstat() # Print progress indicator elapsed_time = (datetime.now() - start_time).total_seconds() // interval print(f"Minute {elapsed_time + 1}", flush=True) time.sleep(interval) def run_debug(): print("Capturing new debug.") command = ["midclt", "call", "system.debug_generate", "-job"] try: result = subprocess.run(command, capture_output=True, text=True, check=True) return result.stdout except subprocess.CalledProcessError as e: print(f"Error running midclt debug: {e}") return None def collect_csv(): source_dir = os.getcwd() destination_dir = "/var/log" for filename in os.listdir(source_dir): if filename.endswith(".csv"): # Check for .csv extension source_file = os.path.join(source_dir, filename) destination_file = os.path.join(destination_dir, filename) # Handle potential errors (e.g., source file not found, permission issues) try: shutil.copy2(source_file, destination_file) print(f"Copied '{filename}' to /var/log successfully.") except FileNotFoundError: print(f"Error: File '{filename}' not found in '{source_dir}'.") except PermissionError: print(f"Error: Insufficient permissions to copy '{filename}'.") except Exception as e: # Catch other potential errors print(f"Error copying '{filename}': {e}") def upload_debug(): """ Uploads debug file to a specified FTP server. """ hostname = socket.gethostname() # Generate debug file debug_file = run_debug().strip() # Upload data files subprocess.run( [ "curl", "--user", "customer:ixcustomer", "-T", debug_file, f"ftp.ixsystems.com/debug-{hostname}-{datetime.now().strftime('%Y%m%d%H%M')}.tgz", ] ) def main(): minutes = int(input("Enter the duration for data collection (minutes): ")) interval = 60 # Default interval between data collection cycles (seconds) gstat_command = ["gstat", "-C", "-s", "-d", "-o", "-p", "-I", "5s"] print("Starting Collection") if is_freebsd(): with open("gstat.csv", "a") as output_file: # Create a Popen object with stdout redirected to the file process = subprocess.Popen(gstat_command, stdout=output_file) # Collect data collect_data(minutes, interval) # kill gstat if freebsd if is_freebsd(): process.kill() # Copy data files to /var/log (replace with appropriate copying function) collect_csv() # Upload data (replace with credentials and actual upload logic if needed) upload_debug() print("Data collection and upload completed.") if __name__ == "__main__": main()