scott/statscollect
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cleaned up old files in multibuild

Browse Source
This commit is contained in:
Marc Mance
2024-07-29 14:14:54 -04:00
parent 2ff24b41dc
commit f691e55502
6 changed files with 0 additions and 480 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
collect.sh
View File

@@ -1,42 +0,0 @@
#!/bin/bash
minutes=$1
echo "THIS SCRIPT IS FOR FREEBSD."
echo ""
echo "This script will scrape:"
echo "------------------------"
echo "- Network Interfaces via ifstat"
echo "- CPU/Memory via top"
echo "- Disk Stats via gstat"
echo "- ZFS Pool vai zpool iostat"
echo ""
echo "------------------------"
echo "./collect.sh 60 - will collect for 60 minutes."
echo "------------------------"
echo ""
echo "This script will auto uploaded data to ftp.ixsystems.com when completed."
echo "It will also copy the '*.csv' files to /var/log"
echo "Airgapped systems will need to upload the '*.csv' and/or take a debug."
echo ""
echo "Starting ['gstat','iostat','zpool','cpu','nic'] Collection"
echo ""
echo -n "START MINUTE COUNT:"
echo "timestamp,name,q-depth,total_ops/s,read/s,read_sz-KiB,read-KiB/s,ms/read,write/s,write_sz-KiB,write-KiB/s,ms/write,delete/s,delete-sz-KiB,delete-KiB/s,ms/delete,other/s,ms/other,%busy" > gstat.csv
timeout ${minutes}m gstat -C -s -d -o -p -I 5s >> gstat.csv &
for (( i=1; i<=$minutes; i++ )); do
python3 zpooliostat.py
python3 iostat.py
python3 topstat.py
python3 ifstat.py
echo -n " $i"
sleep 60
done
cp *.csv /var/log
DEBUGFILE=`midclt call system.debug_generate -job` && export DEBUGFILE && curl --user customer:ixcustomer -T $DEBUGFILE ftp.ixsystems.com/debug-`hostname`-`date +%Y%m%d%H%M`.tgz

63
ifstat.py
View File

@@ -1,63 +0,0 @@
import subprocess
import re
import csv
import os
import time
# Define your shell command
command = ["ifstat", "-znq", "1", "1"]
headerRow = [
"timestamp",
"name",
"Kbps_in",
"Kbps_out",
]
filename = "ifStat.csv"
def runifstat():
# Run the command and capture output
result = subprocess.run(command, capture_output=True, text=True)
timestamp = time.time()
local_time = time.localtime(timestamp)
# Format the time components for a more readable output
timestamp = time.strftime("%Y-%m-%d %H:%M:%S", local_time)
# Check the return code (0 for success)
if result.returncode == 0:
# Check if the file exists
if not os.path.exists(filename):
# Create the file in write mode ('w') if it doesn't exist
with open(filename, "w", newline="") as csvfile:
csv_writer = csv.writer(csvfile)
# Write the header row (optional)
csv_writer.writerow(headerRow)
# Access the captured output as a string
output = result.stdout
bylines = output.split("\n")
interfaces = bylines[0].split()
stats = bylines[2].split()
for nic in interfaces:
lineData = [timestamp, nic, stats.pop(0), stats.pop(0)]
if lineData:
with open(filename, "a", newline="") as csvfile:
csv_writer = csv.writer(csvfile)
csv_writer.writerow(lineData)
else:
print(f"Error running command: {result.stderr}")
def main():
runifstat()
if __name__ == "__main__":
main()

66
iostat.py
View File

@@ -1,66 +0,0 @@
import subprocess
import re
import csv
import os
import time
# Define your shell command
command = ["iostat", "-n", "0"]
headerRow = [
"timestamp",
"name",
"tty_in",
"tty_out",
"cpu_user",
"cpu_user_niced",
"cpu_system",
"cpu_interrupt",
"cpu_idle",
]
filename = "cpuStat.csv"
def runIostat():
# Run the command and capture output
result = subprocess.run(command, capture_output=True, text=True)
timestamp = time.time()
local_time = time.localtime(timestamp)
# Format the time components for a more readable output
timestamp = time.strftime("%Y-%m-%d %H:%M:%S", local_time)
# Check the return code (0 for success)
if result.returncode == 0:
# Check if the file exists
if not os.path.exists(filename):
# Create the file in write mode ('w') if it doesn't exist
with open(filename, "w", newline="") as csvfile:
csv_writer = csv.writer(csvfile)
# Write the header row (optional)
csv_writer.writerow(headerRow)
# Access the captured output as a string
output = result.stdout
byline = re.split("\n", output)
data = byline[2]
lineData = data.split()
if lineData:
with open(filename, "a", newline="") as csvfile:
csv_writer = csv.writer(csvfile)
lineData.insert(0, "cpu")
lineData.insert(0, timestamp)
csv_writer.writerow(lineData)
else:
print(f"Error running command: {result.stderr}")
def main():
runIostat()
if __name__ == "__main__":
main()

23
test.sh
View File

@@ -1,23 +0,0 @@
greeting=$(cat << EOF
.THIS SCRIPT IS FOR FREEBSD.\n\n
This script will scrape:\n
------------------------\n
- Network Interfaces via ifstat\n
- CPU/Memory via top\n
- Disk Stats via gstat\n
- ZFS Pool vai zpool iostat\n\n
Output will be a series of csv files.\n\n
------------------------\n
./collect.sh 60 - will collect for 60 minutes.\n
------------------------\n\n
This script will auto uploaded data to ftp.ixsystems.com when completed.\n
It will also copy the '*.csv' files to /var/log\n
Airgapped systems will need to upload the '*.csv' and/or take a debug.\n
EOF
)
echo -n "hello"
echo "test"
echo "moteq"

183
topstat.py
View File

@@ -1,183 +0,0 @@
import subprocess
import re
import csv
import os
import time
# Define your shell command
command = ["top", "-n", "1"]
headerRow = [
"timestamp",
"name",
"loadaverage_1",
"loadaverage_5",
"loadaverage_15",
"total_processes",
"running",
"sleeping",
"zombie",
"cpu_user",
"cpu_nice",
"cpu_system",
"cpu_interrupt",
"cpu_idle",
"mem_active",
"mem_inactive",
"mem_wired",
"mem_free",
# "arc_total",
# "arc_MFU",
# "arc_MRU",
# "arc_anon",
# "arc_header",
# "arc_other",
# "swap_total",
# "swap_free",
]
filename = "topStat.csv"
def convert_to_megabytes(data_size):
"""
Converts a string representing data size (number + unit) to size in megabytes.
Args:
data_size (str): String representing data size (e.g., "10G", "2M").
Returns:
int: Size of data in megabytes, or None if the format is invalid.
"""
try:
unit = data_size[-1].upper()
size = data_size[:-1]
size = float(size)
except (ValueError, AttributeError):
return None
unit_map = {
"K": 1 / (1024**1),
"M": 1,
"G": 1024,
"T": 1024**2,
}
if unit not in unit_map:
return None
return int(size * unit_map[unit])
def runtopstat():
# Run the command and capture output
result = subprocess.run(command, capture_output=True, text=True)
timestamp = time.time()
local_time = time.localtime(timestamp)
# Format the time components for a more readable output
timestamp = time.strftime("%Y-%m-%d %H:%M:%S", local_time)
# Check the return code (0 for success)
if result.returncode == 0:
# Check if the file exists
if not os.path.exists(filename):
# Create the file in write mode ('w') if it doesn't exist
with open(filename, "w", newline="") as csvfile:
csv_writer = csv.writer(csvfile)
# Write the header row (optional)
csv_writer.writerow(headerRow)
# Access the captured output as a string
output = result.stdout
loadaverages = re.search("^last.*", output, re.MULTILINE).group()
loadaverages = re.sub("up.*", "", loadaverages.strip())
loadaverages = re.sub("[a-z:;,]", "", loadaverages.strip()).split()
loadaverage1 = loadaverages[0]
loadaverage5 = loadaverages[1]
loadaverage15 = loadaverages[2]
processes = re.search(".*processes.*", output).group()
processes = re.sub("[a-z,:]", "", processes.strip()).split()
total_processes = processes[0]
running = processes[1]
sleeping = 0
zombie = 0
try:
sleeping = processes[2]
except IndexError:
pass
try:
zombie = processes[3]
except IndexError:
pass
cpu = re.search("CPU:.*", output).group()
cpu = re.sub("CPU.", "", cpu.strip())
cpu = re.sub("[a-z,%]", "", cpu.strip()).split()
cpu_user = cpu[0]
cpu_nice = cpu[1]
cpu_system = cpu[2]
cpu_interrupt = cpu[3]
cpu_idle = cpu[4]
mem = re.search("Mem:.*", output).group()
mem = re.sub("Mem.", "", mem.strip())
mem = re.sub("[Active|Inact|Wired|Free|,]", "", mem.strip()).split()
meminmeg = []
for x in mem:
meminmeg.append(convert_to_megabytes(x))
mem = meminmeg
mem_active = mem[0]
mem_inactive = mem[1]
mem_wired = mem[2]
mem_free = mem[3]
lineData = [
timestamp,
"top",
loadaverage1,
loadaverage5,
loadaverage15,
total_processes,
running,
sleeping,
zombie,
cpu_user,
cpu_nice,
cpu_system,
cpu_interrupt,
cpu_idle,
mem_active,
mem_inactive,
mem_wired,
mem_free,
]
if lineData:
with open(filename, "a", newline="") as csvfile:
csv_writer = csv.writer(csvfile)
csv_writer.writerow(lineData)
else:
print(f"Error running command: {result.stderr}")
def main():
runtopstat()
if __name__ == "__main__":
main()

103
zpooliostat.py
View File

@@ -1,103 +0,0 @@
import subprocess
import re
import csv
import os
import time
# Define your shell command
command = ["zpool", "iostat", "-Tu", "-l", "-p", "-v", "-y", "15", "1"]
headerRow = [
"timestamp",
"name",
"capacity_alloc",
"capacity_free",
"ops_read",
"ops_write",
"bandwidth_read",
"bandwidth_write",
"total_wait_read",
"total_wait_write",
"disk_wait_read",
"disk_wait_write",
"syncq_wait_read",
"syncq_wait_write",
"asyncq_wait_read",
"asyncq_wait_write",
"scrub_wait",
"trim_wait",
"rebuld_wait",
]
def runZpoolIostat():
# Run the command and capture output
result = subprocess.run(command, capture_output=True, text=True)
# Check the return code (0 for success)
if result.returncode == 0:
# Access the captured output as a string
output = result.stdout
# split output by pools
pools = re.split(r"^-", output, flags=re.MULTILINE)
i = 0
for pool in pools:
if i == 0:
# get timestamp
timestamp = int(pool.splitlines()[0].strip())
# Convert to human-readable time using time.localtime()
local_time = time.localtime(timestamp)
# Format the time components for a more readable output
timestamp = time.strftime("%Y-%m-%d %H:%M:%S", local_time)
else:
# remove ------ lines
pool = re.sub("^---.*", "", pool).strip()
# turn - values into 0
pool = re.sub(" -", "0", pool)
y = 0
poolbyline = re.split("\n", pool)
for line in poolbyline:
lineData = line.split()
if not lineData:
break
if y == 0:
poolname = lineData[0]
filename = poolname + "-zio.csv"
# Check if the file exists
if not os.path.exists(filename):
# Create the file in write mode ('w') if it doesn't exist
with open(filename, "w", newline="") as csvfile:
csv_writer = csv.writer(csvfile)
# Write the header row (optional)
csv_writer.writerow(headerRow)
# Now open the file in append mode ('a')
with open(filename, "a", newline="") as csvfile:
csv_writer = csv.writer(csvfile)
lineData.insert(0, timestamp)
csv_writer.writerow(lineData)
y += 1
i += 1
else:
print(f"Error running command: {result.stderr}")
def main():
runZpoolIostat()
if __name__ == "__main__":
main()