Initial commit — bootstrapped from es24n-conf (TrueNAS/Linux edition)
Starting point for Windows packaging. Serial backend will be replaced with pyserial; PyInstaller used to produce a standalone .exe. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
327
modules/workflow_firmware.py
Normal file
327
modules/workflow_firmware.py
Normal file
@@ -0,0 +1,327 @@
|
||||
"""
|
||||
workflow_firmware.py — IOM and Fabric Card firmware update workflow.
|
||||
Connects to each IOM via its network IP address using the Redfish API.
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
|
||||
from redfish import (
|
||||
_redfish_upload_firmware,
|
||||
_redfish_trigger_update,
|
||||
_redfish_poll_tasks,
|
||||
_redfish_restart_iom,
|
||||
_redfish_reset_fabric,
|
||||
_wait_for_iom_online,
|
||||
_show_fw_versions,
|
||||
)
|
||||
from ui import (
|
||||
_c, C,
|
||||
banner, rule, draw_box,
|
||||
info, ok, warn, error,
|
||||
prompt, prompt_ip, prompt_yn, prompt_password,
|
||||
)
|
||||
|
||||
|
||||
# ── Firmware file selection helper ────────────────────────────────────────────
|
||||
def _prompt_fw_file(label: str) -> str:
|
||||
"""
|
||||
Scan the firmware/ directory (next to es24n_conf.py) for firmware files
|
||||
and let the user pick one by number, or enter a custom path as the last
|
||||
option. Files are sorted most-recently-modified first.
|
||||
"""
|
||||
fw_dir = os.path.normpath(
|
||||
os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "firmware")
|
||||
)
|
||||
FW_EXTS = {".bin", ".img", ".fw", ".fwc", ".hex", ".zip", ".tar", ".tgz", ".gz"}
|
||||
|
||||
try:
|
||||
candidates = sorted(
|
||||
[f for f in os.listdir(fw_dir)
|
||||
if not f.startswith(".")
|
||||
and os.path.isfile(os.path.join(fw_dir, f))
|
||||
and os.path.splitext(f)[1].lower() in FW_EXTS],
|
||||
key=lambda f: os.path.getmtime(os.path.join(fw_dir, f)),
|
||||
reverse=True,
|
||||
)
|
||||
except OSError:
|
||||
candidates = []
|
||||
|
||||
print()
|
||||
if candidates:
|
||||
info(f"Firmware files found in {fw_dir}:")
|
||||
for i, fname in enumerate(candidates, 1):
|
||||
sz = os.path.getsize(os.path.join(fw_dir, fname))
|
||||
print(f" {_c(C.BOLD, str(i))} {fname} {_c(C.DIM, f'({sz // 1024} KB)')}")
|
||||
custom_idx = len(candidates) + 1
|
||||
print(f" {_c(C.BOLD, str(custom_idx))} Enter a custom file path")
|
||||
print()
|
||||
|
||||
while True:
|
||||
choice = prompt(f"Select {label} [1-{custom_idx}]")
|
||||
if choice.isdigit():
|
||||
idx = int(choice)
|
||||
if 1 <= idx <= len(candidates):
|
||||
path = os.path.join(fw_dir, candidates[idx - 1])
|
||||
sz = os.path.getsize(path)
|
||||
ok(f"Selected: {candidates[idx - 1]} ({sz // 1024} KB)")
|
||||
return path
|
||||
if idx == custom_idx:
|
||||
break
|
||||
warn(f"Please enter a number between 1 and {custom_idx}.")
|
||||
else:
|
||||
info(f"No firmware files found in {fw_dir}.")
|
||||
|
||||
# Manual path entry
|
||||
while True:
|
||||
path = prompt(f"Path to {label}")
|
||||
if os.path.isfile(path):
|
||||
sz = os.path.getsize(path)
|
||||
ok(f"File: {path} ({sz // 1024} KB)")
|
||||
return path
|
||||
warn(f"File not found: {path}")
|
||||
|
||||
|
||||
# ── Multi-shelf IP collection ─────────────────────────────────────────────────
|
||||
def _collect_shelves() -> list:
|
||||
"""
|
||||
Prompt for the password and a single IOM IP address per shelf,
|
||||
offering to add more shelves after each entry.
|
||||
|
||||
Either IOM's IP can be used — both IOMs share the same Redfish endpoint
|
||||
and can reach each other's resources. Each shelf has its own password
|
||||
because the Admin password is the BMC serial number, unique per shelf.
|
||||
|
||||
Returns a list of (password, ip) tuples, one per shelf.
|
||||
"""
|
||||
shelves = []
|
||||
shelf_num = 1
|
||||
|
||||
while True:
|
||||
info(f"Shelf {shelf_num} — enter password and IOM IP address.")
|
||||
password = prompt_password()
|
||||
ip = prompt_ip(f" Shelf {shelf_num} IOM IP address (IOM1 or IOM2)")
|
||||
shelves.append((password, ip))
|
||||
print()
|
||||
|
||||
if not prompt_yn("Add another shelf?", default=False):
|
||||
break
|
||||
shelf_num += 1
|
||||
print()
|
||||
|
||||
return shelves
|
||||
|
||||
|
||||
def _make_targets(shelves: list, iom_choice: str) -> list:
|
||||
"""
|
||||
Convert the shelves structure into a flat list of (label, iom, ip, password)
|
||||
tuples suitable for _show_fw_versions(). When there is only one shelf the
|
||||
label is just the IOM name; for multiple shelves it includes the shelf number.
|
||||
All IOMs in a shelf share the same IP connection.
|
||||
"""
|
||||
ioms = []
|
||||
if iom_choice in ("1", "3"):
|
||||
ioms.append("IOM1")
|
||||
if iom_choice in ("2", "3"):
|
||||
ioms.append("IOM2")
|
||||
|
||||
multi = len(shelves) > 1
|
||||
return [
|
||||
(f"S{i} / {iom}" if multi else iom, iom, ip, password)
|
||||
for i, (password, ip) in enumerate(shelves, 1)
|
||||
for iom in ioms
|
||||
]
|
||||
|
||||
|
||||
# ── Per-IOM update helpers ────────────────────────────────────────────────────
|
||||
def _update_iom_fw(password: str, ip: str, iom: str, fw_path: str) -> bool:
|
||||
"""Upload and apply IOM firmware for one IOM, then restart it."""
|
||||
sz = os.path.getsize(fw_path)
|
||||
info(f"Uploading IOM firmware ({sz // 1024} KB) to {iom} at {ip}...")
|
||||
ok_flag, data = _redfish_upload_firmware(password, ip, fw_path)
|
||||
if not ok_flag:
|
||||
error(f"Upload failed: {data}")
|
||||
return False
|
||||
ok("Firmware file uploaded.")
|
||||
|
||||
info(f"Triggering {iom} firmware update...")
|
||||
ok_flag, data = _redfish_trigger_update(
|
||||
password, ip, f"/redfish/v1/Managers/{iom}",
|
||||
)
|
||||
if not ok_flag:
|
||||
error(f"Update trigger failed: {data}")
|
||||
return False
|
||||
ok("Update triggered.")
|
||||
|
||||
info("Monitoring update progress (this may take several minutes)...")
|
||||
ok_flag, msg = _redfish_poll_tasks(password, ip)
|
||||
if not ok_flag:
|
||||
warn(f"Task monitoring ended: {msg}")
|
||||
else:
|
||||
ok(msg)
|
||||
|
||||
info(f"Restarting {iom}...")
|
||||
_redfish_restart_iom(password, ip, iom) # connection drop on restart is normal
|
||||
ok(f"{iom} restart initiated. Waiting for IOM to come back online...")
|
||||
time.sleep(30) # allow time for the IOM to begin shutting down before polling
|
||||
if _wait_for_iom_online(password, ip):
|
||||
ok(f"{iom} is back online.")
|
||||
info("Allowing IOM services to fully initialize before next step...")
|
||||
time.sleep(60)
|
||||
else:
|
||||
warn(f"{iom} did not respond within 5 minutes — proceeding anyway.")
|
||||
return True
|
||||
|
||||
|
||||
def _update_fabric_fw(password: str, ip: str, iom: str, fw_path: str) -> bool:
|
||||
"""
|
||||
Upload and apply Fabric Card firmware for one IOM.
|
||||
Per the service guide, the firmware file must be re-uploaded even if it was
|
||||
already uploaded during the IOM firmware step.
|
||||
After the update: restart fabric card, then restart IOM.
|
||||
"""
|
||||
sz = os.path.getsize(fw_path)
|
||||
|
||||
# Retry the upload — after an IOM reboot the inter-IOM services can take
|
||||
# time to initialize, causing the first upload attempt to fail with
|
||||
# "Failed to send update package to other IOM".
|
||||
MAX_UPLOAD_ATTEMPTS = 3
|
||||
ok_flag, data = False, ""
|
||||
for attempt in range(1, MAX_UPLOAD_ATTEMPTS + 1):
|
||||
info(f"Uploading Fabric Card firmware ({sz // 1024} KB) to {iom} at {ip}"
|
||||
+ (f" (attempt {attempt}/{MAX_UPLOAD_ATTEMPTS})" if attempt > 1 else "") + "...")
|
||||
ok_flag, data = _redfish_upload_firmware(password, ip, fw_path)
|
||||
if ok_flag:
|
||||
break
|
||||
if attempt < MAX_UPLOAD_ATTEMPTS:
|
||||
warn(f"Upload failed: {data}")
|
||||
info("Waiting 60s for IOM services to finish initializing...")
|
||||
time.sleep(60)
|
||||
|
||||
if not ok_flag:
|
||||
error(f"Upload failed after {MAX_UPLOAD_ATTEMPTS} attempts: {data}")
|
||||
return False
|
||||
ok("Firmware file uploaded.")
|
||||
|
||||
info(f"Triggering {iom} Fabric Card firmware update...")
|
||||
ok_flag, data = _redfish_trigger_update(
|
||||
password, ip, f"/redfish/v1/Chassis/{iom}/NetworkAdapters/1",
|
||||
)
|
||||
if not ok_flag:
|
||||
error(f"Update trigger failed: {data}")
|
||||
return False
|
||||
ok("Update triggered.")
|
||||
|
||||
info("Monitoring update progress...")
|
||||
ok_flag, msg = _redfish_poll_tasks(password, ip)
|
||||
if not ok_flag:
|
||||
warn(f"Task monitoring ended: {msg}")
|
||||
else:
|
||||
ok(msg)
|
||||
|
||||
info(f"Restarting {iom} Fabric Card...")
|
||||
_redfish_reset_fabric(password, ip, iom)
|
||||
ok("Fabric Card restart initiated. Waiting 15s...")
|
||||
time.sleep(15)
|
||||
|
||||
info(f"Restarting {iom} after Fabric Card update...")
|
||||
_redfish_restart_iom(password, ip, iom)
|
||||
ok(f"{iom} restart initiated. Waiting for IOM to come back online...")
|
||||
time.sleep(30) # allow time for the IOM to begin shutting down before polling
|
||||
if _wait_for_iom_online(password, ip):
|
||||
ok(f"{iom} is back online.")
|
||||
else:
|
||||
warn(f"{iom} did not respond within 5 minutes — proceeding anyway.")
|
||||
return True
|
||||
|
||||
|
||||
# ── Firmware Update Workflow ──────────────────────────────────────────────────
|
||||
def firmware_update_workflow():
|
||||
"""
|
||||
Standalone firmware update for IOM and Fabric Card firmware.
|
||||
Connects to each IOM via its network IP (not serial loopback) — uploading
|
||||
firmware over 115200-baud serial would be impractically slow.
|
||||
Supports updating multiple shelves in a single run.
|
||||
"""
|
||||
banner()
|
||||
rule("IOM & Fabric Card Firmware Update")
|
||||
info("This procedure connects to each IOM via its network IP address.")
|
||||
info("Ensure this system has network access to the IOM management interface.")
|
||||
print()
|
||||
|
||||
print(" Which IOM(s) would you like to update?")
|
||||
print(f" {_c(C.BOLD, '1')} IOM1 only")
|
||||
print(f" {_c(C.BOLD, '2')} IOM2 only")
|
||||
print(f" {_c(C.BOLD, '3')} Both IOM1 and IOM2 {_c(C.YEL, '(Do not use on production systems)')}")
|
||||
print()
|
||||
|
||||
while True:
|
||||
iom_choice = prompt("Select option [1-3]")
|
||||
if iom_choice in ("1", "2", "3"):
|
||||
break
|
||||
warn("Please enter 1, 2, or 3.")
|
||||
print()
|
||||
|
||||
# ── Collect IPs for one or more shelves ───────────────────────────────────
|
||||
shelves = _collect_shelves()
|
||||
targets = _make_targets(shelves, iom_choice)
|
||||
|
||||
rule("Current Firmware Versions")
|
||||
_show_fw_versions(targets)
|
||||
|
||||
print(" What would you like to update?")
|
||||
print(f" {_c(C.BOLD, '1')} IOM Firmware only")
|
||||
print(f" {_c(C.BOLD, '2')} Fabric Card Firmware only")
|
||||
print(f" {_c(C.BOLD, '3')} Both IOM and Fabric Card Firmware")
|
||||
print(f" {_c(C.BOLD, '4')} Cancel")
|
||||
print()
|
||||
|
||||
while True:
|
||||
choice = prompt("Select option [1-4]")
|
||||
if choice in ("1", "2", "3", "4"):
|
||||
break
|
||||
warn("Please enter 1, 2, 3, or 4.")
|
||||
|
||||
if choice == "4":
|
||||
info("Firmware update cancelled.")
|
||||
return
|
||||
|
||||
update_iom = choice in ("1", "3")
|
||||
update_fabric = choice in ("2", "3")
|
||||
|
||||
fw_path = _prompt_fw_file("firmware file")
|
||||
|
||||
print()
|
||||
warn("For HA systems: update the passive IOM first.")
|
||||
if len(shelves) > 1:
|
||||
warn(f"Updating {len(shelves)} shelves sequentially — Shelf 1 first.")
|
||||
elif iom_choice == "3":
|
||||
warn("IOM1 will be updated first — adjust order if IOM2 is passive.")
|
||||
print()
|
||||
|
||||
if not prompt_yn("Proceed with firmware update?", default=True):
|
||||
info("Firmware update cancelled.")
|
||||
return
|
||||
|
||||
# ── Run updates per target (shelf × IOM) ─────────────────────────────────
|
||||
for label, iom, ip, password in targets:
|
||||
rule(f"{label} ({ip})")
|
||||
if update_iom:
|
||||
_update_iom_fw(password, ip, iom, fw_path)
|
||||
if update_fabric:
|
||||
_update_fabric_fw(password, ip, iom, fw_path)
|
||||
|
||||
rule("Post-Update Firmware Validation")
|
||||
_show_fw_versions(targets)
|
||||
|
||||
ok("Firmware update complete.")
|
||||
print()
|
||||
draw_box([
|
||||
f"{_c(C.YEL, 'IMPORTANT -- For HA (Dual-Controller) Systems:')}",
|
||||
"",
|
||||
"After updating this controller's IOMs:",
|
||||
" 1. Log into TrueNAS and initiate a failover.",
|
||||
" 2. Re-run this tool to update the other controller.",
|
||||
], colour=C.YEL)
|
||||
print()
|
||||
prompt("Press Enter to return to main menu")
|
||||
Reference in New Issue
Block a user