diff --git a/es24n_conf.py b/es24n_conf.py index 7b8c2fe..8de60c4 100644 --- a/es24n_conf.py +++ b/es24n_conf.py @@ -53,7 +53,7 @@ def banner(): print(_c(C.CYN, " |") + _c(C.WHT + C.BOLD, " TrueNAS ES24N IOM Configuration Tool ") + _c(C.CYN, " |")) print(_c(C.CYN, " |") + _c(C.DIM, - " Serial Network Setup v2.0 (stdlib only) ") + _c(C.CYN, " |")) + " Serial Config & Firmware Updates (stdlib only) ") + _c(C.CYN, " |")) print(_c(C.CYN, " +" + "-" * w + "+")) print() @@ -418,12 +418,14 @@ def open_serial_connection(device: str) -> Optional[SerialPort]: # ── Redfish helpers (GET and PATCH) ────────────────────────────────────────── def _redfish_request(password: str, method: str, path: str, - payload: Optional[dict] = None) -> tuple: + payload: Optional[dict] = None, + host: str = "127.0.0.1") -> tuple: """ - Issue a Redfish HTTP request over the loopback (127.0.0.1). + Issue a Redfish HTTP request. host defaults to the serial loopback (127.0.0.1) + but can be set to an IOM's network IP for firmware update operations. Returns (success: bool, data: dict|str). """ - url = f"https://127.0.0.1{path}" + url = f"https://{host}{path}" credentials = b64encode(f"Admin:{password}".encode()).decode() ctx = ssl.create_default_context() ctx.check_hostname = False @@ -456,6 +458,353 @@ def _redfish_request(password: str, method: str, path: str, return False, f"Connection error: {e}" +# ── Redfish helpers (firmware upload & update) ──────────────────────────────── +def _redfish_upload_firmware(password: str, host: str, fw_path: str) -> tuple: + """ + Upload a firmware file to /redfish/v1/UpdateService using multipart/form-data. + Equivalent to: curl -k -u Admin: https:///redfish/v1/UpdateService + -X POST -F "software=@" + """ + try: + with open(fw_path, "rb") as f: + file_data = f.read() + except OSError as e: + return False, f"Cannot read file: {e}" + + filename = os.path.basename(fw_path) + boundary = f"FormBoundary{int(time.time() * 1000)}" + + body = ( + f"--{boundary}\r\n" + f'Content-Disposition: form-data; name="software"; filename="{filename}"\r\n' + "Content-Type: application/octet-stream\r\n" + "\r\n" + ).encode() + file_data + f"\r\n--{boundary}--\r\n".encode() + + url = f"https://{host}/redfish/v1/UpdateService" + credentials = b64encode(f"Admin:{password}".encode()).decode() + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + + req = urllib.request.Request( + url, data=body, method="POST", + headers={ + "Authorization": f"Basic {credentials}", + "Content-Type": f"multipart/form-data; boundary={boundary}", + }, + ) + try: + with urllib.request.urlopen(req, context=ctx, timeout=120) as resp: + raw = resp.read().decode("utf-8", errors="replace") + try: + return True, json.loads(raw) if raw.strip() else {} + except json.JSONDecodeError: + return True, {} + except urllib.error.HTTPError as e: + raw = e.read().decode("utf-8", errors="replace") + try: + msg = json.loads(raw).get("error", {}).get("message", raw) + except json.JSONDecodeError: + msg = raw + return False, f"HTTP {e.code}: {msg[:120]}" + except OSError as e: + return False, f"Connection error: {e}" + + +def _redfish_trigger_update(password: str, host: str, target: str) -> tuple: + """ + Trigger a Redfish SimpleUpdate for the given target resource path. + target: e.g. "/redfish/v1/Managers/IOM1" + or "/redfish/v1/Chassis/IOM1/NetworkAdapters/1" + """ + return _redfish_request( + password, "POST", + "/redfish/v1/UpdateService/Actions/SimpleUpdate", + payload={ + "ImageURI": "/redfish/v1/UpdateService/software", + "Targets": [target], + }, + host=host, + ) + + +def _redfish_poll_tasks(password: str, host: str, timeout: int = 600) -> tuple: + """ + Poll /redfish/v1/TaskService/Tasks/ until all tasks reach a terminal state + or timeout is exceeded. Returns (success: bool, message: str). + """ + TERMINAL = {"Completed", "Killed", "Exception"} + deadline = time.monotonic() + timeout + elapsed = 0 + + while time.monotonic() < deadline: + ok_flag, data = _redfish_request( + password, "GET", "/redfish/v1/TaskService/Tasks/", host=host, + ) + if not ok_flag: + return False, f"Task service error: {data}" + + members = data.get("Members", []) + if not members: + return True, "No pending tasks." + + running = [] + for member in members: + state = member.get("TaskState") + if state is None: + # Resolve individual task link + task_path = member.get("@odata.id", "") + if task_path: + t_ok, t_data = _redfish_request( + password, "GET", task_path, host=host, + ) + state = (t_data.get("TaskState") + if t_ok and isinstance(t_data, dict) else "Running") + else: + state = "Running" + if state not in TERMINAL: + running.append(state) + + if not running: + return True, "All tasks completed." + + info(f" Tasks running ({', '.join(running)})... [{elapsed}s elapsed]") + time.sleep(10) + elapsed += 10 + + return False, f"Timeout after {timeout}s waiting for tasks." + + +def _redfish_restart_iom(password: str, host: str, iom: str) -> tuple: + return _redfish_request( + password, "POST", + f"/redfish/v1/Managers/{iom}/Actions/Manager.Reset", + payload={"ResetType": "GracefulRestart"}, + host=host, + ) + + +def _redfish_reset_fabric(password: str, host: str, iom: str) -> tuple: + return _redfish_request( + password, "POST", + f"/redfish/v1/Chassis/{iom}/NetworkAdapters/1/Actions/NetworkAdapter.Reset", + payload={"ResetType": "GracefulRestart"}, + host=host, + ) + + +def _get_iom_fw_version(password: str, host: str, iom: str) -> str: + ok_flag, data = _redfish_request( + password, "GET", f"/redfish/v1/Managers/{iom}", host=host, + ) + if ok_flag and isinstance(data, dict): + return data.get("FirmwareVersion", "Unknown") + return _c(C.RED, "Unreachable") + + +def _get_fabric_fw_version(password: str, host: str, iom: str) -> str: + ok_flag, data = _redfish_request( + password, "GET", + f"/redfish/v1/Chassis/{iom}/NetworkAdapters/1", + host=host, + ) + if ok_flag and isinstance(data, dict): + version = (data.get("Oem", {}) + .get("Version", {}) + .get("ActiveFirmwareVersion")) + return version or "Unknown" + return _c(C.RED, "Unreachable") + + +def _show_fw_versions(password: str, iom1_ip: str, iom2_ip: str): + info("Querying firmware versions...") + rows = [] + for iom, ip in [("IOM1", iom1_ip), ("IOM2", iom2_ip)]: + iom_ver = _get_iom_fw_version(password, ip, iom) + fabric_ver = _get_fabric_fw_version(password, ip, iom) + rows.append([iom, ip, iom_ver, fabric_ver]) + print() + draw_table( + ["IOM", "IP Address", "IOM Firmware", "Fabric Firmware"], + rows, + [5, 16, 32, 20], + ) + print() + + +def _update_iom_fw(password: str, ip: str, iom: str, fw_path: str) -> bool: + """Upload and apply IOM firmware for one IOM, then restart it.""" + sz = os.path.getsize(fw_path) + info(f"Uploading IOM firmware ({sz // 1024} KB) to {iom} at {ip}...") + ok_flag, data = _redfish_upload_firmware(password, ip, fw_path) + if not ok_flag: + error(f"Upload failed: {data}") + return False + ok("Firmware file uploaded.") + + info(f"Triggering {iom} firmware update...") + ok_flag, data = _redfish_trigger_update( + password, ip, f"/redfish/v1/Managers/{iom}", + ) + if not ok_flag: + error(f"Update trigger failed: {data}") + return False + ok("Update triggered.") + + info("Monitoring update progress (this may take several minutes)...") + ok_flag, msg = _redfish_poll_tasks(password, ip) + if not ok_flag: + warn(f"Task monitoring ended: {msg}") + else: + ok(msg) + + info(f"Restarting {iom}...") + _redfish_restart_iom(password, ip, iom) # connection drop on restart is normal + ok(f"{iom} restart initiated. Waiting 30s for reboot...") + time.sleep(30) + return True + + +def _update_fabric_fw(password: str, ip: str, iom: str, fw_path: str) -> bool: + """ + Upload and apply Fabric Card firmware for one IOM. + Per the service guide, the firmware file must be re-uploaded even if it was + already uploaded during the IOM firmware step. + After the update: restart fabric card, then restart IOM. + """ + sz = os.path.getsize(fw_path) + info(f"Uploading Fabric Card firmware ({sz // 1024} KB) to {iom} at {ip}...") + ok_flag, data = _redfish_upload_firmware(password, ip, fw_path) + if not ok_flag: + error(f"Upload failed: {data}") + return False + ok("Firmware file uploaded.") + + info(f"Triggering {iom} Fabric Card firmware update...") + ok_flag, data = _redfish_trigger_update( + password, ip, f"/redfish/v1/Chassis/{iom}/NetworkAdapters/1", + ) + if not ok_flag: + error(f"Update trigger failed: {data}") + return False + ok("Update triggered.") + + info("Monitoring update progress...") + ok_flag, msg = _redfish_poll_tasks(password, ip) + if not ok_flag: + warn(f"Task monitoring ended: {msg}") + else: + ok(msg) + + info(f"Restarting {iom} Fabric Card...") + _redfish_reset_fabric(password, ip, iom) + ok("Fabric Card restart initiated. Waiting 15s...") + time.sleep(15) + + info(f"Restarting {iom} after Fabric Card update...") + _redfish_restart_iom(password, ip, iom) + ok(f"{iom} restart initiated. Waiting 30s for reboot...") + time.sleep(30) + return True + + +# ── Firmware Update Workflow ────────────────────────────────────────────────── +def firmware_update_workflow(): + """ + Standalone firmware update for IOM and Fabric Card firmware. + Connects to each IOM via its network IP (not serial loopback) — uploading + firmware over 115200-baud serial would be impractically slow. + """ + banner() + rule("IOM & Fabric Card Firmware Update") + info("This procedure connects to each IOM via its network IP address.") + info("Ensure this system has network access to the IOM management interface.") + print() + + password = prompt_password() + print() + + info("Enter the management IP address for each IOM.") + iom1_ip = prompt_ip(" IOM1 IP address") + iom2_ip = prompt_ip(" IOM2 IP address") + print() + + rule("Current Firmware Versions") + _show_fw_versions(password, iom1_ip, iom2_ip) + + print(" What would you like to update?") + print(f" {_c(C.BOLD, '1')} IOM Firmware only") + print(f" {_c(C.BOLD, '2')} Fabric Card Firmware only") + print(f" {_c(C.BOLD, '3')} Both IOM and Fabric Card Firmware") + print(f" {_c(C.BOLD, '4')} Cancel") + print() + + while True: + choice = prompt("Select option [1-4]") + if choice in ("1", "2", "3", "4"): + break + warn("Please enter 1, 2, 3, or 4.") + + if choice == "4": + info("Firmware update cancelled.") + return + + update_iom = choice in ("1", "3") + update_fabric = choice in ("2", "3") + iom_fw_path = "" + fabric_fw_path = "" + + if update_iom: + print() + while True: + iom_fw_path = prompt("Path to IOM firmware file") + if os.path.isfile(iom_fw_path): + sz = os.path.getsize(iom_fw_path) + ok(f"File: {iom_fw_path} ({sz // 1024} KB)") + break + warn(f"File not found: {iom_fw_path}") + + if update_fabric: + print() + while True: + fabric_fw_path = prompt("Path to Fabric Card firmware file") + if os.path.isfile(fabric_fw_path): + sz = os.path.getsize(fabric_fw_path) + ok(f"File: {fabric_fw_path} ({sz // 1024} KB)") + break + warn(f"File not found: {fabric_fw_path}") + + print() + warn("For HA systems: update the passive IOM first.") + warn("IOM1 will be updated first — adjust order if IOM2 is passive.") + print() + + if not prompt_yn("Proceed with firmware update?", default=True): + info("Firmware update cancelled.") + return + + for iom, ip in [("IOM1", iom1_ip), ("IOM2", iom2_ip)]: + rule(f"{iom} ({ip})") + if update_iom: + _update_iom_fw(password, ip, iom, iom_fw_path) + if update_fabric: + _update_fabric_fw(password, ip, iom, fabric_fw_path) + + rule("Post-Update Firmware Validation") + _show_fw_versions(password, iom1_ip, iom2_ip) + + print() + draw_box([ + f"{_c(C.YEL, 'IMPORTANT -- For HA (Dual-Controller) Systems:')}", + "", + "After updating this controller's IOMs:", + " 1. Log into TrueNAS and initiate a failover.", + " 2. Re-run this tool to update the other controller.", + ], colour=C.YEL) + print() + + # ── Step 3: Fetch & display current IOM network settings ───────────────────── def fetch_current_config(cfg: ShelfConfig) -> bool: """ @@ -773,28 +1122,30 @@ def configure_shelf() -> bool: # ── Entry point ─────────────────────────────────────────────────────────────── def main(): banner() - print(f" {_c(C.DIM, 'Automates ES24N IOM network configuration over a direct serial')}") - print(f" {_c(C.DIM, 'connection using the Redfish API (loopback).')}") - print(f" {_c(C.DIM, 'No external dependencies -- Python 3 standard library only.')}") + print(f" {_c(C.DIM, 'Automates ES24N IOM network configuration and firmware updates')}") + print(f" {_c(C.DIM, 'using the Redfish API. No external dependencies.')}") print() while True: banner() draw_box([ f" {_c(C.BOLD, '1')} Configure a new ES24N shelf", - f" {_c(C.BOLD, '2')} Exit", + f" {_c(C.BOLD, '2')} Update IOM / Fabric Card Firmware", + f" {_c(C.BOLD, '3')} Exit", ]) print() - choice = prompt("Select [1/2]") + choice = prompt("Select [1/2/3]") if choice == "1": another = configure_shelf() if not another: break elif choice == "2": + firmware_update_workflow() + elif choice == "3": break else: - warn("Please enter 1 or 2.") + warn("Please enter 1, 2, or 3.") time.sleep(1) print()