update for core cpuStat and interval to be interval instead of minut
This commit is contained in:
Marc Mance
175
collect.py
175
collect.py
@@ -19,36 +19,25 @@ def is_debian():
|
||||
# Debian-based systems often have 'debian' in their platform string
|
||||
return "linux" in platform.platform().lower()
|
||||
|
||||
def getTimestamp():
|
||||
timestamp = time.time()
|
||||
local_time = time.localtime(timestamp)
|
||||
# Format the time components for a more readable output
|
||||
return time.strftime("%Y-%m-%d %H:%M:%S", local_time)
|
||||
|
||||
|
||||
def runCollect(command):
|
||||
# Run the command and capture output
|
||||
result = subprocess.run(command, capture_output=True, text=True)
|
||||
# Check the return code (0 for success)
|
||||
if result.returncode == 0:
|
||||
# Access the captured output as a string
|
||||
return result.stdout
|
||||
|
||||
return result.stderr
|
||||
|
||||
def scaleIostat_disk():
|
||||
command = ["iostat", "-xyd", "1", "1"]
|
||||
headerRow = [
|
||||
"timestamp",
|
||||
"name",
|
||||
"read_s",
|
||||
"rkB_s",
|
||||
"rrqm_s",
|
||||
"%rrqm",
|
||||
"r_await",
|
||||
"rareq-sz",
|
||||
"w_s",
|
||||
"wkB_s",
|
||||
"wrqm_s",
|
||||
"%wrqm",
|
||||
"w_await",
|
||||
"wareq-sz",
|
||||
"d_s",
|
||||
"dkB_s",
|
||||
"drqm_s",
|
||||
"%drqm",
|
||||
"d_await",
|
||||
"dareq-sz",
|
||||
"f_s",
|
||||
"f_await",
|
||||
"aqu-sz",
|
||||
"%utilread_s",
|
||||
]
|
||||
|
||||
filename = "ioStat.csv"
|
||||
collect = runCollect(command)
|
||||
|
||||
@@ -69,38 +58,8 @@ def scaleIostat_disk():
|
||||
print(f"Error running command: {collect}")
|
||||
|
||||
|
||||
def getTimestamp():
|
||||
timestamp = time.time()
|
||||
local_time = time.localtime(timestamp)
|
||||
# Format the time components for a more readable output
|
||||
return time.strftime("%Y-%m-%d %H:%M:%S", local_time)
|
||||
|
||||
|
||||
def runCollect(command):
|
||||
# Run the command and capture output
|
||||
result = subprocess.run(command, capture_output=True, text=True)
|
||||
# Check the return code (0 for success)
|
||||
if result.returncode == 0:
|
||||
# Access the captured output as a string
|
||||
return result.stdout
|
||||
|
||||
return result.stderr
|
||||
|
||||
|
||||
def scaleIostat_cpu():
|
||||
command = ["iostat", "-c", "1", "1"]
|
||||
|
||||
headerRow = [
|
||||
"timestamp",
|
||||
"name",
|
||||
"%user",
|
||||
"%nice",
|
||||
"%system",
|
||||
"%iowait",
|
||||
"%steal",
|
||||
"%idle",
|
||||
]
|
||||
|
||||
filename = "cpuStat.csv"
|
||||
|
||||
collect = runCollect(command)
|
||||
@@ -121,30 +80,46 @@ def scaleIostat_cpu():
|
||||
print(f"Error running command: {collect}")
|
||||
|
||||
|
||||
def ifstat():
|
||||
command = ["ifstat", "-znq", "1", "1"]
|
||||
headerRow = [
|
||||
"timestamp",
|
||||
"name",
|
||||
"Kbps_in",
|
||||
"Kbps_out",
|
||||
]
|
||||
filename = "ifStat.csv"
|
||||
def coreIostat_cpu():
|
||||
command = ["iostat", "-c", "1", "1"]
|
||||
filename = "cpuStat.csv"
|
||||
|
||||
collect = runCollect(command)
|
||||
|
||||
if collect:
|
||||
byline = re.split("\n", collect.strip())
|
||||
data = byline[3].strip()
|
||||
lineData = data.split()
|
||||
|
||||
if lineData:
|
||||
with open(filename, "a", newline="") as csvfile:
|
||||
csv_writer = csv.writer(csvfile)
|
||||
lineData.insert(0, "cpu")
|
||||
lineData.insert(0, getTimestamp())
|
||||
csv_writer.writerow(lineData)
|
||||
|
||||
else:
|
||||
print(f"Error running command: {collect}")
|
||||
|
||||
|
||||
def coreifstat():
|
||||
command = ["iostat", "-C", "-t", "proc", "-d"]
|
||||
filename = "cpuStat.csv"
|
||||
|
||||
collect = runCollect(command)
|
||||
|
||||
if collect:
|
||||
# process collection string
|
||||
bylines = collect.split("\n")
|
||||
interfaces = bylines[0].split()
|
||||
stats = bylines[2].split()
|
||||
lineData = bylines[2].split()
|
||||
|
||||
for nic in interfaces:
|
||||
lineData = [getTimestamp(), nic, stats.pop(0), stats.pop(0)]
|
||||
lineData = lineData[:-2] + [0] + lineData[-2:]
|
||||
|
||||
if lineData:
|
||||
with open(filename, "a", newline="") as csvfile:
|
||||
csv_writer = csv.writer(csvfile)
|
||||
lineData.insert(0, "cpu")
|
||||
lineData.insert(0, getTimestamp())
|
||||
csv_writer.writerow(lineData)
|
||||
|
||||
else:
|
||||
@@ -153,29 +128,6 @@ def ifstat():
|
||||
|
||||
def zpoolIostat():
|
||||
command = ["zpool", "iostat", "-Tu", "-l", "-p", "-v", "-y", "15", "1"]
|
||||
|
||||
headerRow = [
|
||||
"timestamp",
|
||||
"name",
|
||||
"capacity_alloc",
|
||||
"capacity_free",
|
||||
"ops_read",
|
||||
"ops_write",
|
||||
"bandwidth_read",
|
||||
"bandwidth_write",
|
||||
"total_wait_read",
|
||||
"total_wait_write",
|
||||
"disk_wait_read",
|
||||
"disk_wait_write",
|
||||
"syncq_wait_read",
|
||||
"syncq_wait_write",
|
||||
"asyncq_wait_read",
|
||||
"asyncq_wait_write",
|
||||
"scrub_wait",
|
||||
"trim_wait",
|
||||
"rebuld_wait",
|
||||
]
|
||||
|
||||
collect = runCollect(command)
|
||||
|
||||
if collect:
|
||||
@@ -225,15 +177,13 @@ def collect_data(minutes, interval):
|
||||
start_time = datetime.now()
|
||||
# Continuously collect data for the specified duration
|
||||
|
||||
gstat_command = ["gstat", "-C", "-s", "-d", "-o", "-p", "-I", "5s"]
|
||||
if is_freebsd():
|
||||
gstat_command = ["gstat", "-C", "-s", "-d", "-o", "-p", "-I", "5s"]
|
||||
with open("gstat.csv", "a") as output_file:
|
||||
# Create a Popen object with stdout redirected to the file
|
||||
process = subprocess.Popen(gstat_command, stdout=output_file)
|
||||
|
||||
x = 1
|
||||
|
||||
for _ in range(minutes * 60 // interval):
|
||||
for i in range(minutes):
|
||||
# Collect data using system commands
|
||||
|
||||
zpoolIostat()
|
||||
@@ -243,15 +193,13 @@ def collect_data(minutes, interval):
|
||||
scaleIostat_cpu()
|
||||
scaleIostat_disk()
|
||||
|
||||
# Print progress indicator
|
||||
elapsed_time = (datetime.now() - start_time).total_seconds() // interval
|
||||
if is_freebsd():
|
||||
coreIostat_cpu()
|
||||
|
||||
if x == 1:
|
||||
if i == 1:
|
||||
print("Minute:", end="", flush=True)
|
||||
|
||||
x = 0
|
||||
|
||||
print(f" {int(elapsed_time + 1)}", end="", flush=True)
|
||||
print(f" {i}", end="", flush=True)
|
||||
|
||||
time.sleep(interval)
|
||||
|
||||
@@ -272,27 +220,6 @@ def run_debug():
|
||||
return None
|
||||
|
||||
|
||||
# def collect_csv():
|
||||
# source_dir = os.getcwd()
|
||||
# tar_file_name = "dpkg.log" # Choose a desired name for the tar file
|
||||
# tar_file_path = os.path.join(source_dir, tar_file_name)
|
||||
|
||||
# with tarfile.open(tar_file_path, "w:gz") as tar:
|
||||
# for filename in os.listdir(source_dir):
|
||||
# if filename.endswith(".csv"):
|
||||
# source_file = os.path.join(source_dir, filename)
|
||||
|
||||
# try:
|
||||
# tar.add(source_file, arcname=os.path.basename(source_file))
|
||||
# print(f"Added '{filename}' to tar archive successfully.")
|
||||
# except FileNotFoundError:
|
||||
# print(f"Error: File '{filename}' not found in '{source_dir}'.")
|
||||
# except Exception as e: # Catch other potential errors
|
||||
# print(f"Error adding '{filename}' to tar archive: {e}")
|
||||
|
||||
# return tar_file_path
|
||||
|
||||
|
||||
def collect_csv():
|
||||
source_dir = os.getcwd()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user