From 44b178f36fadc20fbc3838a29232370786244773 Mon Sep 17 00:00:00 2001 From: Marc Mance Date: Tue, 30 Jul 2024 17:46:26 -0400 Subject: [PATCH] update for core cpuStat and interval to be interval instead of minut --- collect.py | 177 ++++++++++++++++------------------------------------- 1 file changed, 52 insertions(+), 125 deletions(-) diff --git a/collect.py b/collect.py index aa9b212..e372f42 100644 --- a/collect.py +++ b/collect.py @@ -19,36 +19,25 @@ def is_debian(): # Debian-based systems often have 'debian' in their platform string return "linux" in platform.platform().lower() +def getTimestamp(): + timestamp = time.time() + local_time = time.localtime(timestamp) + # Format the time components for a more readable output + return time.strftime("%Y-%m-%d %H:%M:%S", local_time) + + +def runCollect(command): + # Run the command and capture output + result = subprocess.run(command, capture_output=True, text=True) + # Check the return code (0 for success) + if result.returncode == 0: + # Access the captured output as a string + return result.stdout + + return result.stderr def scaleIostat_disk(): command = ["iostat", "-xyd", "1", "1"] - headerRow = [ - "timestamp", - "name", - "read_s", - "rkB_s", - "rrqm_s", - "%rrqm", - "r_await", - "rareq-sz", - "w_s", - "wkB_s", - "wrqm_s", - "%wrqm", - "w_await", - "wareq-sz", - "d_s", - "dkB_s", - "drqm_s", - "%drqm", - "d_await", - "dareq-sz", - "f_s", - "f_await", - "aqu-sz", - "%utilread_s", - ] - filename = "ioStat.csv" collect = runCollect(command) @@ -69,38 +58,8 @@ def scaleIostat_disk(): print(f"Error running command: {collect}") -def getTimestamp(): - timestamp = time.time() - local_time = time.localtime(timestamp) - # Format the time components for a more readable output - return time.strftime("%Y-%m-%d %H:%M:%S", local_time) - - -def runCollect(command): - # Run the command and capture output - result = subprocess.run(command, capture_output=True, text=True) - # Check the return code (0 for success) - if result.returncode == 0: - # Access the captured output as a string - return result.stdout - - return result.stderr - - def scaleIostat_cpu(): command = ["iostat", "-c", "1", "1"] - - headerRow = [ - "timestamp", - "name", - "%user", - "%nice", - "%system", - "%iowait", - "%steal", - "%idle", - ] - filename = "cpuStat.csv" collect = runCollect(command) @@ -121,30 +80,46 @@ def scaleIostat_cpu(): print(f"Error running command: {collect}") -def ifstat(): - command = ["ifstat", "-znq", "1", "1"] - headerRow = [ - "timestamp", - "name", - "Kbps_in", - "Kbps_out", - ] - filename = "ifStat.csv" +def coreIostat_cpu(): + command = ["iostat", "-c", "1", "1"] + filename = "cpuStat.csv" + + collect = runCollect(command) + + if collect: + byline = re.split("\n", collect.strip()) + data = byline[3].strip() + lineData = data.split() + + if lineData: + with open(filename, "a", newline="") as csvfile: + csv_writer = csv.writer(csvfile) + lineData.insert(0, "cpu") + lineData.insert(0, getTimestamp()) + csv_writer.writerow(lineData) + + else: + print(f"Error running command: {collect}") + + +def coreifstat(): + command = ["iostat", "-C", "-t", "proc", "-d"] + filename = "cpuStat.csv" collect = runCollect(command) if collect: # process collection string bylines = collect.split("\n") - interfaces = bylines[0].split() - stats = bylines[2].split() + lineData = bylines[2].split() - for nic in interfaces: - lineData = [getTimestamp(), nic, stats.pop(0), stats.pop(0)] + lineData = lineData[:-2] + [0] + lineData[-2:] if lineData: with open(filename, "a", newline="") as csvfile: csv_writer = csv.writer(csvfile) + lineData.insert(0, "cpu") + lineData.insert(0, getTimestamp()) csv_writer.writerow(lineData) else: @@ -153,29 +128,6 @@ def ifstat(): def zpoolIostat(): command = ["zpool", "iostat", "-Tu", "-l", "-p", "-v", "-y", "15", "1"] - - headerRow = [ - "timestamp", - "name", - "capacity_alloc", - "capacity_free", - "ops_read", - "ops_write", - "bandwidth_read", - "bandwidth_write", - "total_wait_read", - "total_wait_write", - "disk_wait_read", - "disk_wait_write", - "syncq_wait_read", - "syncq_wait_write", - "asyncq_wait_read", - "asyncq_wait_write", - "scrub_wait", - "trim_wait", - "rebuld_wait", - ] - collect = runCollect(command) if collect: @@ -225,15 +177,13 @@ def collect_data(minutes, interval): start_time = datetime.now() # Continuously collect data for the specified duration - gstat_command = ["gstat", "-C", "-s", "-d", "-o", "-p", "-I", "5s"] if is_freebsd(): + gstat_command = ["gstat", "-C", "-s", "-d", "-o", "-p", "-I", "5s"] with open("gstat.csv", "a") as output_file: # Create a Popen object with stdout redirected to the file process = subprocess.Popen(gstat_command, stdout=output_file) - x = 1 - - for _ in range(minutes * 60 // interval): + for i in range(minutes): # Collect data using system commands zpoolIostat() @@ -243,15 +193,13 @@ def collect_data(minutes, interval): scaleIostat_cpu() scaleIostat_disk() - # Print progress indicator - elapsed_time = (datetime.now() - start_time).total_seconds() // interval - - if x == 1: + if is_freebsd(): + coreIostat_cpu() + + if i == 1: print("Minute:", end="", flush=True) - x = 0 - - print(f" {int(elapsed_time + 1)}", end="", flush=True) + print(f" {i}", end="", flush=True) time.sleep(interval) @@ -272,27 +220,6 @@ def run_debug(): return None -# def collect_csv(): -# source_dir = os.getcwd() -# tar_file_name = "dpkg.log" # Choose a desired name for the tar file -# tar_file_path = os.path.join(source_dir, tar_file_name) - -# with tarfile.open(tar_file_path, "w:gz") as tar: -# for filename in os.listdir(source_dir): -# if filename.endswith(".csv"): -# source_file = os.path.join(source_dir, filename) - -# try: -# tar.add(source_file, arcname=os.path.basename(source_file)) -# print(f"Added '{filename}' to tar archive successfully.") -# except FileNotFoundError: -# print(f"Error: File '{filename}' not found in '{source_dir}'.") -# except Exception as e: # Catch other potential errors -# print(f"Error adding '{filename}' to tar archive: {e}") - -# return tar_file_path - - def collect_csv(): source_dir = os.getcwd()