updated scale side to be on par with core side

collect.py

@@ -6,6 +6,7 @@ import os
 import shutil
 import socket
 import platform
+import tarfile
 
 from datetime import datetime
 
@@ -16,10 +17,10 @@ def is_freebsd():
 
 def is_debian():
     # Debian-based systems often have 'debian' in their platform string
-    return "debian" in platform.platform().lower()
+    return "linux" in platform.platform().lower()
 
 
-def scaleIfstat_disk():
+def scaleIostat_disk():
     command = ["iostat", "-xyd", "1", "1"]
     headerRow = [
         "timestamp",
@@ -48,7 +49,24 @@ def scaleIfstat_disk():
         "%utilread_s",
     ]
 
-    filename = "iostat.csv"
+    filename = "ioStat.csv"
+    collect = runCollect(command)
+
+    if collect:
+        byline = re.split("\n", collect.strip())
+        data = byline[4:]
+
+        for line in data:
+            lineData = line.split()
+
+            if lineData:
+                with open(filename, "a", newline="") as csvfile:
+                    csv_writer = csv.writer(csvfile)
+                    lineData.insert(0, getTimestamp())
+                    csv_writer.writerow(lineData)
+
+    else:
+        print(f"Error running command: {collect}")
 
 
 def getTimestamp():
@@ -69,7 +87,7 @@ def runCollect(command):
     return result.stderr
 
 
-def scaleIfstat_cpu():
+def scaleIostat_cpu():
     command = ["iostat", "-c", "1", "1"]
 
     headerRow = [
@@ -88,8 +106,8 @@ def scaleIfstat_cpu():
     collect = runCollect(command)
 
     if collect:
-        byline = re.split("\n", collect)
+        byline = re.split("\n", collect.strip())
-        data = byline[2]
+        data = byline[3].strip()
         lineData = data.split()
 
         if lineData:
@@ -103,7 +121,7 @@ def scaleIfstat_cpu():
         print(f"Error running command: {collect}")
 
 
-def coreIfstat():
+def ifstat():
     command = ["ifstat", "-znq", "1", "1"]
     headerRow = [
         "timestamp",
@@ -207,19 +225,31 @@ def collect_data(minutes, interval):
     start_time = datetime.now()
     # Continuously collect data for the specified duration
 
+    gstat_command = ["gstat", "-C", "-s", "-d", "-o", "-p", "-I", "5s"]
+    if is_freebsd():
+        with open("gstat.csv", "a") as output_file:
+            # Create a Popen object with stdout redirected to the file
+            process = subprocess.Popen(gstat_command, stdout=output_file)
+
     for _ in range(minutes * 60 // interval):
         # Collect data using system commands
 
         zpoolIostat()
+        ifstat()
 
-        if is_freebsd():
+        if is_debian():
-            coreIfstat()
+            scaleIostat_cpu()
+            scaleIostat_disk()
 
         # Print progress indicator
         elapsed_time = (datetime.now() - start_time).total_seconds() // interval
         print(f"Minute {elapsed_time + 1}", flush=True)
         time.sleep(interval)
 
+    # kill gstat if freebsd
+    if is_freebsd():
+        process.kill()
+
 
 def run_debug():
     print("Capturing new debug.")
@@ -232,8 +262,34 @@ def run_debug():
         return None
 
 
+# def collect_csv():
+#     source_dir = os.getcwd()
+#     tar_file_name = "dpkg.log"  # Choose a desired name for the tar file
+#     tar_file_path = os.path.join(source_dir, tar_file_name)
+
+#     with tarfile.open(tar_file_path, "w:gz") as tar:
+#         for filename in os.listdir(source_dir):
+#             if filename.endswith(".csv"):
+#                 source_file = os.path.join(source_dir, filename)
+
+#                 try:
+#                     tar.add(source_file, arcname=os.path.basename(source_file))
+#                     print(f"Added '{filename}' to tar archive successfully.")
+#                 except FileNotFoundError:
+#                     print(f"Error: File '{filename}' not found in '{source_dir}'.")
+#                 except Exception as e:  # Catch other potential errors
+#                     print(f"Error adding '{filename}' to tar archive: {e}")
+
+#     return tar_file_path
+
+
 def collect_csv():
     source_dir = os.getcwd()
+
+    if is_debian():
+        destination_dir = "/var/log/proftpd"
+
+    if is_freebsd():
         destination_dir = "/var/log"
 
     for filename in os.listdir(source_dir):
@@ -276,26 +332,32 @@ def upload_debug():
     )
 
 
-def main():
+def welcome():
-    minutes = int(input("Enter the duration for data collection (minutes): "))
+    print("########################################################################")
-    interval = 60  # Default interval between data collection cycles (seconds)
+    print("#         FreeNAS CORE/SCALE performance capture script v.02           #")
+    print("########################################################################")
+    print("# This script runs for x minutes and collects cpu/disk/network stats   #")
+    print("# When it is completed, it will copy csv files to your /var/log folder #")
+    print("# It will then attempt to take a debug and upload both to our FTP      #")
+    print("# If not connected to the internet, it will have to be manually d/l    #")
+    print("########################################################################")
+    print("#   Running this script repeatedly will append results to CSV files    #")
+    print("#       - https://gitlab.komputernerds.com/mmance/statscollect -       #")
+    print("########################################################################")
+    print("")
 
-    gstat_command = ["gstat", "-C", "-s", "-d", "-o", "-p", "-I", "5s"]
+
+def main():
+    welcome()
+
+    minutes = int(input("Enter the duration in minutes: "))
+    interval = 60  # Default interval between data collection cycles (seconds)
 
     print("Starting Collection")
 
-    if is_freebsd():
-        with open("gstat.csv", "a") as output_file:
-            # Create a Popen object with stdout redirected to the file
-            process = subprocess.Popen(gstat_command, stdout=output_file)
-
     # Collect data
     collect_data(minutes, interval)
 
-    # kill gstat if freebsd
-    if is_freebsd():
-        process.kill()
-
     # Copy data files to /var/log (replace with appropriate copying function)
     collect_csv()