diff --git a/collect.sh b/collect.sh deleted file mode 100755 index 82c816e..0000000 --- a/collect.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -minutes=$1 - -echo "THIS SCRIPT IS FOR FREEBSD." -echo "" -echo "This script will scrape:" -echo "------------------------" -echo "- Network Interfaces via ifstat" -echo "- CPU/Memory via top" -echo "- Disk Stats via gstat" -echo "- ZFS Pool vai zpool iostat" -echo "" -echo "------------------------" -echo "./collect.sh 60 - will collect for 60 minutes." -echo "------------------------" -echo "" -echo "This script will auto uploaded data to ftp.ixsystems.com when completed." -echo "It will also copy the '*.csv' files to /var/log" -echo "Airgapped systems will need to upload the '*.csv' and/or take a debug." -echo "" -echo "Starting ['gstat','iostat','zpool','cpu','nic'] Collection" -echo "" -echo -n "START MINUTE COUNT:" - -echo "timestamp,name,q-depth,total_ops/s,read/s,read_sz-KiB,read-KiB/s,ms/read,write/s,write_sz-KiB,write-KiB/s,ms/write,delete/s,delete-sz-KiB,delete-KiB/s,ms/delete,other/s,ms/other,%busy" > gstat.csv -timeout ${minutes}m gstat -C -s -d -o -p -I 5s >> gstat.csv & - -for (( i=1; i<=$minutes; i++ )); do - - python3 zpooliostat.py - python3 iostat.py - python3 topstat.py - python3 ifstat.py - - echo -n " $i" - sleep 60 - -done - -cp *.csv /var/log - -DEBUGFILE=`midclt call system.debug_generate -job` && export DEBUGFILE && curl --user customer:ixcustomer -T $DEBUGFILE ftp.ixsystems.com/debug-`hostname`-`date +%Y%m%d%H%M`.tgz diff --git a/ifstat.py b/ifstat.py deleted file mode 100644 index 657f54f..0000000 --- a/ifstat.py +++ /dev/null @@ -1,63 +0,0 @@ -import subprocess -import re -import csv -import os -import time - -# Define your shell command -command = ["ifstat", "-znq", "1", "1"] - -headerRow = [ - "timestamp", - "name", - "Kbps_in", - "Kbps_out", -] - -filename = "ifStat.csv" - - -def runifstat(): - # Run the command and capture output - result = subprocess.run(command, capture_output=True, text=True) - - timestamp = time.time() - local_time = time.localtime(timestamp) - # Format the time components for a more readable output - timestamp = time.strftime("%Y-%m-%d %H:%M:%S", local_time) - - # Check the return code (0 for success) - if result.returncode == 0: - # Check if the file exists - if not os.path.exists(filename): - # Create the file in write mode ('w') if it doesn't exist - with open(filename, "w", newline="") as csvfile: - csv_writer = csv.writer(csvfile) - # Write the header row (optional) - csv_writer.writerow(headerRow) - - # Access the captured output as a string - output = result.stdout - - bylines = output.split("\n") - interfaces = bylines[0].split() - stats = bylines[2].split() - - for nic in interfaces: - lineData = [timestamp, nic, stats.pop(0), stats.pop(0)] - - if lineData: - with open(filename, "a", newline="") as csvfile: - csv_writer = csv.writer(csvfile) - csv_writer.writerow(lineData) - - else: - print(f"Error running command: {result.stderr}") - - -def main(): - runifstat() - - -if __name__ == "__main__": - main() diff --git a/iostat.py b/iostat.py deleted file mode 100644 index ea252ba..0000000 --- a/iostat.py +++ /dev/null @@ -1,66 +0,0 @@ -import subprocess -import re -import csv -import os -import time - -# Define your shell command -command = ["iostat", "-n", "0"] - -headerRow = [ - "timestamp", - "name", - "tty_in", - "tty_out", - "cpu_user", - "cpu_user_niced", - "cpu_system", - "cpu_interrupt", - "cpu_idle", -] - -filename = "cpuStat.csv" - - -def runIostat(): - # Run the command and capture output - result = subprocess.run(command, capture_output=True, text=True) - - timestamp = time.time() - local_time = time.localtime(timestamp) - # Format the time components for a more readable output - timestamp = time.strftime("%Y-%m-%d %H:%M:%S", local_time) - - # Check the return code (0 for success) - if result.returncode == 0: - # Check if the file exists - if not os.path.exists(filename): - # Create the file in write mode ('w') if it doesn't exist - with open(filename, "w", newline="") as csvfile: - csv_writer = csv.writer(csvfile) - # Write the header row (optional) - csv_writer.writerow(headerRow) - - # Access the captured output as a string - output = result.stdout - byline = re.split("\n", output) - data = byline[2] - lineData = data.split() - - if lineData: - with open(filename, "a", newline="") as csvfile: - csv_writer = csv.writer(csvfile) - lineData.insert(0, "cpu") - lineData.insert(0, timestamp) - csv_writer.writerow(lineData) - - else: - print(f"Error running command: {result.stderr}") - - -def main(): - runIostat() - - -if __name__ == "__main__": - main() diff --git a/test.sh b/test.sh deleted file mode 100755 index f85023f..0000000 --- a/test.sh +++ /dev/null @@ -1,23 +0,0 @@ - - -greeting=$(cat << EOF -.THIS SCRIPT IS FOR FREEBSD.\n\n -This script will scrape:\n -------------------------\n -- Network Interfaces via ifstat\n -- CPU/Memory via top\n -- Disk Stats via gstat\n -- ZFS Pool vai zpool iostat\n\n -Output will be a series of csv files.\n\n -------------------------\n -./collect.sh 60 - will collect for 60 minutes.\n -------------------------\n\n -This script will auto uploaded data to ftp.ixsystems.com when completed.\n -It will also copy the '*.csv' files to /var/log\n -Airgapped systems will need to upload the '*.csv' and/or take a debug.\n -EOF -) - -echo -n "hello" -echo "test" -echo "moteq" diff --git a/topstat.py b/topstat.py deleted file mode 100644 index bc41c24..0000000 --- a/topstat.py +++ /dev/null @@ -1,183 +0,0 @@ -import subprocess -import re -import csv -import os -import time - -# Define your shell command -command = ["top", "-n", "1"] - -headerRow = [ - "timestamp", - "name", - "loadaverage_1", - "loadaverage_5", - "loadaverage_15", - "total_processes", - "running", - "sleeping", - "zombie", - "cpu_user", - "cpu_nice", - "cpu_system", - "cpu_interrupt", - "cpu_idle", - "mem_active", - "mem_inactive", - "mem_wired", - "mem_free", - # "arc_total", - # "arc_MFU", - # "arc_MRU", - # "arc_anon", - # "arc_header", - # "arc_other", - # "swap_total", - # "swap_free", -] - -filename = "topStat.csv" - - -def convert_to_megabytes(data_size): - """ - Converts a string representing data size (number + unit) to size in megabytes. - - Args: - data_size (str): String representing data size (e.g., "10G", "2M"). - - Returns: - int: Size of data in megabytes, or None if the format is invalid. - """ - - try: - unit = data_size[-1].upper() - size = data_size[:-1] - size = float(size) - except (ValueError, AttributeError): - return None - - unit_map = { - "K": 1 / (1024**1), - "M": 1, - "G": 1024, - "T": 1024**2, - } - - if unit not in unit_map: - return None - - return int(size * unit_map[unit]) - - -def runtopstat(): - # Run the command and capture output - result = subprocess.run(command, capture_output=True, text=True) - - timestamp = time.time() - local_time = time.localtime(timestamp) - # Format the time components for a more readable output - timestamp = time.strftime("%Y-%m-%d %H:%M:%S", local_time) - - # Check the return code (0 for success) - if result.returncode == 0: - # Check if the file exists - if not os.path.exists(filename): - # Create the file in write mode ('w') if it doesn't exist - with open(filename, "w", newline="") as csvfile: - csv_writer = csv.writer(csvfile) - # Write the header row (optional) - csv_writer.writerow(headerRow) - - # Access the captured output as a string - output = result.stdout - - loadaverages = re.search("^last.*", output, re.MULTILINE).group() - loadaverages = re.sub("up.*", "", loadaverages.strip()) - loadaverages = re.sub("[a-z:;,]", "", loadaverages.strip()).split() - - loadaverage1 = loadaverages[0] - loadaverage5 = loadaverages[1] - loadaverage15 = loadaverages[2] - - processes = re.search(".*processes.*", output).group() - processes = re.sub("[a-z,:]", "", processes.strip()).split() - - total_processes = processes[0] - running = processes[1] - - sleeping = 0 - zombie = 0 - - try: - sleeping = processes[2] - except IndexError: - pass - - try: - zombie = processes[3] - except IndexError: - pass - - cpu = re.search("CPU:.*", output).group() - cpu = re.sub("CPU.", "", cpu.strip()) - cpu = re.sub("[a-z,%]", "", cpu.strip()).split() - - cpu_user = cpu[0] - cpu_nice = cpu[1] - cpu_system = cpu[2] - cpu_interrupt = cpu[3] - cpu_idle = cpu[4] - - mem = re.search("Mem:.*", output).group() - mem = re.sub("Mem.", "", mem.strip()) - mem = re.sub("[Active|Inact|Wired|Free|,]", "", mem.strip()).split() - - meminmeg = [] - - for x in mem: - meminmeg.append(convert_to_megabytes(x)) - - mem = meminmeg - - mem_active = mem[0] - mem_inactive = mem[1] - mem_wired = mem[2] - mem_free = mem[3] - - lineData = [ - timestamp, - "top", - loadaverage1, - loadaverage5, - loadaverage15, - total_processes, - running, - sleeping, - zombie, - cpu_user, - cpu_nice, - cpu_system, - cpu_interrupt, - cpu_idle, - mem_active, - mem_inactive, - mem_wired, - mem_free, - ] - - if lineData: - with open(filename, "a", newline="") as csvfile: - csv_writer = csv.writer(csvfile) - csv_writer.writerow(lineData) - - else: - print(f"Error running command: {result.stderr}") - - -def main(): - runtopstat() - - -if __name__ == "__main__": - main() diff --git a/zpooliostat.py b/zpooliostat.py deleted file mode 100644 index 24523b5..0000000 --- a/zpooliostat.py +++ /dev/null @@ -1,103 +0,0 @@ -import subprocess -import re -import csv -import os -import time - -# Define your shell command -command = ["zpool", "iostat", "-Tu", "-l", "-p", "-v", "-y", "15", "1"] - -headerRow = [ - "timestamp", - "name", - "capacity_alloc", - "capacity_free", - "ops_read", - "ops_write", - "bandwidth_read", - "bandwidth_write", - "total_wait_read", - "total_wait_write", - "disk_wait_read", - "disk_wait_write", - "syncq_wait_read", - "syncq_wait_write", - "asyncq_wait_read", - "asyncq_wait_write", - "scrub_wait", - "trim_wait", - "rebuld_wait", -] - - -def runZpoolIostat(): - # Run the command and capture output - result = subprocess.run(command, capture_output=True, text=True) - - # Check the return code (0 for success) - if result.returncode == 0: - # Access the captured output as a string - output = result.stdout - - # split output by pools - pools = re.split(r"^-", output, flags=re.MULTILINE) - - i = 0 - for pool in pools: - if i == 0: - # get timestamp - timestamp = int(pool.splitlines()[0].strip()) - # Convert to human-readable time using time.localtime() - local_time = time.localtime(timestamp) - # Format the time components for a more readable output - timestamp = time.strftime("%Y-%m-%d %H:%M:%S", local_time) - - else: - # remove ------ lines - pool = re.sub("^---.*", "", pool).strip() - - # turn - values into 0 - pool = re.sub(" -", "0", pool) - - y = 0 - - poolbyline = re.split("\n", pool) - - for line in poolbyline: - lineData = line.split() - - if not lineData: - break - - if y == 0: - poolname = lineData[0] - filename = poolname + "-zio.csv" - - # Check if the file exists - if not os.path.exists(filename): - # Create the file in write mode ('w') if it doesn't exist - with open(filename, "w", newline="") as csvfile: - csv_writer = csv.writer(csvfile) - # Write the header row (optional) - csv_writer.writerow(headerRow) - - # Now open the file in append mode ('a') - with open(filename, "a", newline="") as csvfile: - csv_writer = csv.writer(csvfile) - lineData.insert(0, timestamp) - csv_writer.writerow(lineData) - - y += 1 - - i += 1 - - else: - print(f"Error running command: {result.stderr}") - - -def main(): - runZpoolIostat() - - -if __name__ == "__main__": - main()