cleaned up old files in multibuild
This commit is contained in:
Marc Mance
42
collect.sh
42
collect.sh
@@ -1,42 +0,0 @@
|
||||
#!/bin/bash
|
||||
minutes=$1
|
||||
|
||||
echo "THIS SCRIPT IS FOR FREEBSD."
|
||||
echo ""
|
||||
echo "This script will scrape:"
|
||||
echo "------------------------"
|
||||
echo "- Network Interfaces via ifstat"
|
||||
echo "- CPU/Memory via top"
|
||||
echo "- Disk Stats via gstat"
|
||||
echo "- ZFS Pool vai zpool iostat"
|
||||
echo ""
|
||||
echo "------------------------"
|
||||
echo "./collect.sh 60 - will collect for 60 minutes."
|
||||
echo "------------------------"
|
||||
echo ""
|
||||
echo "This script will auto uploaded data to ftp.ixsystems.com when completed."
|
||||
echo "It will also copy the '*.csv' files to /var/log"
|
||||
echo "Airgapped systems will need to upload the '*.csv' and/or take a debug."
|
||||
echo ""
|
||||
echo "Starting ['gstat','iostat','zpool','cpu','nic'] Collection"
|
||||
echo ""
|
||||
echo -n "START MINUTE COUNT:"
|
||||
|
||||
echo "timestamp,name,q-depth,total_ops/s,read/s,read_sz-KiB,read-KiB/s,ms/read,write/s,write_sz-KiB,write-KiB/s,ms/write,delete/s,delete-sz-KiB,delete-KiB/s,ms/delete,other/s,ms/other,%busy" > gstat.csv
|
||||
timeout ${minutes}m gstat -C -s -d -o -p -I 5s >> gstat.csv &
|
||||
|
||||
for (( i=1; i<=$minutes; i++ )); do
|
||||
|
||||
python3 zpooliostat.py
|
||||
python3 iostat.py
|
||||
python3 topstat.py
|
||||
python3 ifstat.py
|
||||
|
||||
echo -n " $i"
|
||||
sleep 60
|
||||
|
||||
done
|
||||
|
||||
cp *.csv /var/log
|
||||
|
||||
DEBUGFILE=`midclt call system.debug_generate -job` && export DEBUGFILE && curl --user customer:ixcustomer -T $DEBUGFILE ftp.ixsystems.com/debug-`hostname`-`date +%Y%m%d%H%M`.tgz
|
||||
63
ifstat.py
63
ifstat.py
@@ -1,63 +0,0 @@
|
||||
import subprocess
|
||||
import re
|
||||
import csv
|
||||
import os
|
||||
import time
|
||||
|
||||
# Define your shell command
|
||||
command = ["ifstat", "-znq", "1", "1"]
|
||||
|
||||
headerRow = [
|
||||
"timestamp",
|
||||
"name",
|
||||
"Kbps_in",
|
||||
"Kbps_out",
|
||||
]
|
||||
|
||||
filename = "ifStat.csv"
|
||||
|
||||
|
||||
def runifstat():
|
||||
# Run the command and capture output
|
||||
result = subprocess.run(command, capture_output=True, text=True)
|
||||
|
||||
timestamp = time.time()
|
||||
local_time = time.localtime(timestamp)
|
||||
# Format the time components for a more readable output
|
||||
timestamp = time.strftime("%Y-%m-%d %H:%M:%S", local_time)
|
||||
|
||||
# Check the return code (0 for success)
|
||||
if result.returncode == 0:
|
||||
# Check if the file exists
|
||||
if not os.path.exists(filename):
|
||||
# Create the file in write mode ('w') if it doesn't exist
|
||||
with open(filename, "w", newline="") as csvfile:
|
||||
csv_writer = csv.writer(csvfile)
|
||||
# Write the header row (optional)
|
||||
csv_writer.writerow(headerRow)
|
||||
|
||||
# Access the captured output as a string
|
||||
output = result.stdout
|
||||
|
||||
bylines = output.split("\n")
|
||||
interfaces = bylines[0].split()
|
||||
stats = bylines[2].split()
|
||||
|
||||
for nic in interfaces:
|
||||
lineData = [timestamp, nic, stats.pop(0), stats.pop(0)]
|
||||
|
||||
if lineData:
|
||||
with open(filename, "a", newline="") as csvfile:
|
||||
csv_writer = csv.writer(csvfile)
|
||||
csv_writer.writerow(lineData)
|
||||
|
||||
else:
|
||||
print(f"Error running command: {result.stderr}")
|
||||
|
||||
|
||||
def main():
|
||||
runifstat()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
66
iostat.py
66
iostat.py
@@ -1,66 +0,0 @@
|
||||
import subprocess
|
||||
import re
|
||||
import csv
|
||||
import os
|
||||
import time
|
||||
|
||||
# Define your shell command
|
||||
command = ["iostat", "-n", "0"]
|
||||
|
||||
headerRow = [
|
||||
"timestamp",
|
||||
"name",
|
||||
"tty_in",
|
||||
"tty_out",
|
||||
"cpu_user",
|
||||
"cpu_user_niced",
|
||||
"cpu_system",
|
||||
"cpu_interrupt",
|
||||
"cpu_idle",
|
||||
]
|
||||
|
||||
filename = "cpuStat.csv"
|
||||
|
||||
|
||||
def runIostat():
|
||||
# Run the command and capture output
|
||||
result = subprocess.run(command, capture_output=True, text=True)
|
||||
|
||||
timestamp = time.time()
|
||||
local_time = time.localtime(timestamp)
|
||||
# Format the time components for a more readable output
|
||||
timestamp = time.strftime("%Y-%m-%d %H:%M:%S", local_time)
|
||||
|
||||
# Check the return code (0 for success)
|
||||
if result.returncode == 0:
|
||||
# Check if the file exists
|
||||
if not os.path.exists(filename):
|
||||
# Create the file in write mode ('w') if it doesn't exist
|
||||
with open(filename, "w", newline="") as csvfile:
|
||||
csv_writer = csv.writer(csvfile)
|
||||
# Write the header row (optional)
|
||||
csv_writer.writerow(headerRow)
|
||||
|
||||
# Access the captured output as a string
|
||||
output = result.stdout
|
||||
byline = re.split("\n", output)
|
||||
data = byline[2]
|
||||
lineData = data.split()
|
||||
|
||||
if lineData:
|
||||
with open(filename, "a", newline="") as csvfile:
|
||||
csv_writer = csv.writer(csvfile)
|
||||
lineData.insert(0, "cpu")
|
||||
lineData.insert(0, timestamp)
|
||||
csv_writer.writerow(lineData)
|
||||
|
||||
else:
|
||||
print(f"Error running command: {result.stderr}")
|
||||
|
||||
|
||||
def main():
|
||||
runIostat()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
23
test.sh
23
test.sh
@@ -1,23 +0,0 @@
|
||||
|
||||
|
||||
greeting=$(cat << EOF
|
||||
.THIS SCRIPT IS FOR FREEBSD.\n\n
|
||||
This script will scrape:\n
|
||||
------------------------\n
|
||||
- Network Interfaces via ifstat\n
|
||||
- CPU/Memory via top\n
|
||||
- Disk Stats via gstat\n
|
||||
- ZFS Pool vai zpool iostat\n\n
|
||||
Output will be a series of csv files.\n\n
|
||||
------------------------\n
|
||||
./collect.sh 60 - will collect for 60 minutes.\n
|
||||
------------------------\n\n
|
||||
This script will auto uploaded data to ftp.ixsystems.com when completed.\n
|
||||
It will also copy the '*.csv' files to /var/log\n
|
||||
Airgapped systems will need to upload the '*.csv' and/or take a debug.\n
|
||||
EOF
|
||||
)
|
||||
|
||||
echo -n "hello"
|
||||
echo "test"
|
||||
echo "moteq"
|
||||
183
topstat.py
183
topstat.py
@@ -1,183 +0,0 @@
|
||||
import subprocess
|
||||
import re
|
||||
import csv
|
||||
import os
|
||||
import time
|
||||
|
||||
# Define your shell command
|
||||
command = ["top", "-n", "1"]
|
||||
|
||||
headerRow = [
|
||||
"timestamp",
|
||||
"name",
|
||||
"loadaverage_1",
|
||||
"loadaverage_5",
|
||||
"loadaverage_15",
|
||||
"total_processes",
|
||||
"running",
|
||||
"sleeping",
|
||||
"zombie",
|
||||
"cpu_user",
|
||||
"cpu_nice",
|
||||
"cpu_system",
|
||||
"cpu_interrupt",
|
||||
"cpu_idle",
|
||||
"mem_active",
|
||||
"mem_inactive",
|
||||
"mem_wired",
|
||||
"mem_free",
|
||||
# "arc_total",
|
||||
# "arc_MFU",
|
||||
# "arc_MRU",
|
||||
# "arc_anon",
|
||||
# "arc_header",
|
||||
# "arc_other",
|
||||
# "swap_total",
|
||||
# "swap_free",
|
||||
]
|
||||
|
||||
filename = "topStat.csv"
|
||||
|
||||
|
||||
def convert_to_megabytes(data_size):
|
||||
"""
|
||||
Converts a string representing data size (number + unit) to size in megabytes.
|
||||
|
||||
Args:
|
||||
data_size (str): String representing data size (e.g., "10G", "2M").
|
||||
|
||||
Returns:
|
||||
int: Size of data in megabytes, or None if the format is invalid.
|
||||
"""
|
||||
|
||||
try:
|
||||
unit = data_size[-1].upper()
|
||||
size = data_size[:-1]
|
||||
size = float(size)
|
||||
except (ValueError, AttributeError):
|
||||
return None
|
||||
|
||||
unit_map = {
|
||||
"K": 1 / (1024**1),
|
||||
"M": 1,
|
||||
"G": 1024,
|
||||
"T": 1024**2,
|
||||
}
|
||||
|
||||
if unit not in unit_map:
|
||||
return None
|
||||
|
||||
return int(size * unit_map[unit])
|
||||
|
||||
|
||||
def runtopstat():
|
||||
# Run the command and capture output
|
||||
result = subprocess.run(command, capture_output=True, text=True)
|
||||
|
||||
timestamp = time.time()
|
||||
local_time = time.localtime(timestamp)
|
||||
# Format the time components for a more readable output
|
||||
timestamp = time.strftime("%Y-%m-%d %H:%M:%S", local_time)
|
||||
|
||||
# Check the return code (0 for success)
|
||||
if result.returncode == 0:
|
||||
# Check if the file exists
|
||||
if not os.path.exists(filename):
|
||||
# Create the file in write mode ('w') if it doesn't exist
|
||||
with open(filename, "w", newline="") as csvfile:
|
||||
csv_writer = csv.writer(csvfile)
|
||||
# Write the header row (optional)
|
||||
csv_writer.writerow(headerRow)
|
||||
|
||||
# Access the captured output as a string
|
||||
output = result.stdout
|
||||
|
||||
loadaverages = re.search("^last.*", output, re.MULTILINE).group()
|
||||
loadaverages = re.sub("up.*", "", loadaverages.strip())
|
||||
loadaverages = re.sub("[a-z:;,]", "", loadaverages.strip()).split()
|
||||
|
||||
loadaverage1 = loadaverages[0]
|
||||
loadaverage5 = loadaverages[1]
|
||||
loadaverage15 = loadaverages[2]
|
||||
|
||||
processes = re.search(".*processes.*", output).group()
|
||||
processes = re.sub("[a-z,:]", "", processes.strip()).split()
|
||||
|
||||
total_processes = processes[0]
|
||||
running = processes[1]
|
||||
|
||||
sleeping = 0
|
||||
zombie = 0
|
||||
|
||||
try:
|
||||
sleeping = processes[2]
|
||||
except IndexError:
|
||||
pass
|
||||
|
||||
try:
|
||||
zombie = processes[3]
|
||||
except IndexError:
|
||||
pass
|
||||
|
||||
cpu = re.search("CPU:.*", output).group()
|
||||
cpu = re.sub("CPU.", "", cpu.strip())
|
||||
cpu = re.sub("[a-z,%]", "", cpu.strip()).split()
|
||||
|
||||
cpu_user = cpu[0]
|
||||
cpu_nice = cpu[1]
|
||||
cpu_system = cpu[2]
|
||||
cpu_interrupt = cpu[3]
|
||||
cpu_idle = cpu[4]
|
||||
|
||||
mem = re.search("Mem:.*", output).group()
|
||||
mem = re.sub("Mem.", "", mem.strip())
|
||||
mem = re.sub("[Active|Inact|Wired|Free|,]", "", mem.strip()).split()
|
||||
|
||||
meminmeg = []
|
||||
|
||||
for x in mem:
|
||||
meminmeg.append(convert_to_megabytes(x))
|
||||
|
||||
mem = meminmeg
|
||||
|
||||
mem_active = mem[0]
|
||||
mem_inactive = mem[1]
|
||||
mem_wired = mem[2]
|
||||
mem_free = mem[3]
|
||||
|
||||
lineData = [
|
||||
timestamp,
|
||||
"top",
|
||||
loadaverage1,
|
||||
loadaverage5,
|
||||
loadaverage15,
|
||||
total_processes,
|
||||
running,
|
||||
sleeping,
|
||||
zombie,
|
||||
cpu_user,
|
||||
cpu_nice,
|
||||
cpu_system,
|
||||
cpu_interrupt,
|
||||
cpu_idle,
|
||||
mem_active,
|
||||
mem_inactive,
|
||||
mem_wired,
|
||||
mem_free,
|
||||
]
|
||||
|
||||
if lineData:
|
||||
with open(filename, "a", newline="") as csvfile:
|
||||
csv_writer = csv.writer(csvfile)
|
||||
csv_writer.writerow(lineData)
|
||||
|
||||
else:
|
||||
print(f"Error running command: {result.stderr}")
|
||||
|
||||
|
||||
def main():
|
||||
runtopstat()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
103
zpooliostat.py
103
zpooliostat.py
@@ -1,103 +0,0 @@
|
||||
import subprocess
|
||||
import re
|
||||
import csv
|
||||
import os
|
||||
import time
|
||||
|
||||
# Define your shell command
|
||||
command = ["zpool", "iostat", "-Tu", "-l", "-p", "-v", "-y", "15", "1"]
|
||||
|
||||
headerRow = [
|
||||
"timestamp",
|
||||
"name",
|
||||
"capacity_alloc",
|
||||
"capacity_free",
|
||||
"ops_read",
|
||||
"ops_write",
|
||||
"bandwidth_read",
|
||||
"bandwidth_write",
|
||||
"total_wait_read",
|
||||
"total_wait_write",
|
||||
"disk_wait_read",
|
||||
"disk_wait_write",
|
||||
"syncq_wait_read",
|
||||
"syncq_wait_write",
|
||||
"asyncq_wait_read",
|
||||
"asyncq_wait_write",
|
||||
"scrub_wait",
|
||||
"trim_wait",
|
||||
"rebuld_wait",
|
||||
]
|
||||
|
||||
|
||||
def runZpoolIostat():
|
||||
# Run the command and capture output
|
||||
result = subprocess.run(command, capture_output=True, text=True)
|
||||
|
||||
# Check the return code (0 for success)
|
||||
if result.returncode == 0:
|
||||
# Access the captured output as a string
|
||||
output = result.stdout
|
||||
|
||||
# split output by pools
|
||||
pools = re.split(r"^-", output, flags=re.MULTILINE)
|
||||
|
||||
i = 0
|
||||
for pool in pools:
|
||||
if i == 0:
|
||||
# get timestamp
|
||||
timestamp = int(pool.splitlines()[0].strip())
|
||||
# Convert to human-readable time using time.localtime()
|
||||
local_time = time.localtime(timestamp)
|
||||
# Format the time components for a more readable output
|
||||
timestamp = time.strftime("%Y-%m-%d %H:%M:%S", local_time)
|
||||
|
||||
else:
|
||||
# remove ------ lines
|
||||
pool = re.sub("^---.*", "", pool).strip()
|
||||
|
||||
# turn - values into 0
|
||||
pool = re.sub(" -", "0", pool)
|
||||
|
||||
y = 0
|
||||
|
||||
poolbyline = re.split("\n", pool)
|
||||
|
||||
for line in poolbyline:
|
||||
lineData = line.split()
|
||||
|
||||
if not lineData:
|
||||
break
|
||||
|
||||
if y == 0:
|
||||
poolname = lineData[0]
|
||||
filename = poolname + "-zio.csv"
|
||||
|
||||
# Check if the file exists
|
||||
if not os.path.exists(filename):
|
||||
# Create the file in write mode ('w') if it doesn't exist
|
||||
with open(filename, "w", newline="") as csvfile:
|
||||
csv_writer = csv.writer(csvfile)
|
||||
# Write the header row (optional)
|
||||
csv_writer.writerow(headerRow)
|
||||
|
||||
# Now open the file in append mode ('a')
|
||||
with open(filename, "a", newline="") as csvfile:
|
||||
csv_writer = csv.writer(csvfile)
|
||||
lineData.insert(0, timestamp)
|
||||
csv_writer.writerow(lineData)
|
||||
|
||||
y += 1
|
||||
|
||||
i += 1
|
||||
|
||||
else:
|
||||
print(f"Error running command: {result.stderr}")
|
||||
|
||||
|
||||
def main():
|
||||
runZpoolIostat()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user