es24n-conf-windows/modules/workflow_firmware.py

"""
workflow_firmware.py — IOM and Fabric Card firmware update workflow.
Connects to each IOM via its network IP address using the Redfish API.
"""

import os
import time

from redfish import (
    _redfish_upload_firmware,
    _redfish_trigger_update,
    _redfish_poll_tasks,
    _redfish_restart_iom,
    _redfish_reset_fabric,
    _wait_for_iom_online,
    _show_fw_versions,
)
from ui import (
    _c, C,
    banner, rule, draw_box,
    info, ok, warn, error,
    prompt, prompt_ip, prompt_yn, prompt_password,
)


# ── Firmware file selection helper ────────────────────────────────────────────
def _prompt_fw_file(label: str) -> str:
    """
    Scan the firmware/ directory (next to es24n_conf.py) for firmware files
    and let the user pick one by number, or enter a custom path as the last
    option.  Files are sorted most-recently-modified first.
    """
    fw_dir = os.path.normpath(
        os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "firmware")
    )
    FW_EXTS = {".bin", ".img", ".fw", ".fwc", ".hex", ".zip", ".tar", ".tgz", ".gz"}

    try:
        candidates = sorted(
            [f for f in os.listdir(fw_dir)
             if not f.startswith(".")
             and os.path.isfile(os.path.join(fw_dir, f))
             and os.path.splitext(f)[1].lower() in FW_EXTS],
            key=lambda f: os.path.getmtime(os.path.join(fw_dir, f)),
            reverse=True,
        )
    except OSError:
        candidates = []

    print()
    if candidates:
        info(f"Firmware files found in {fw_dir}:")
        for i, fname in enumerate(candidates, 1):
            sz = os.path.getsize(os.path.join(fw_dir, fname))
            print(f"    {_c(C.BOLD, str(i))}  {fname}  {_c(C.DIM, f'({sz // 1024} KB)')}")
        custom_idx = len(candidates) + 1
        print(f"    {_c(C.BOLD, str(custom_idx))}  Enter a custom file path")
        print()

        while True:
            choice = prompt(f"Select {label} [1-{custom_idx}]")
            if choice.isdigit():
                idx = int(choice)
                if 1 <= idx <= len(candidates):
                    path = os.path.join(fw_dir, candidates[idx - 1])
                    sz   = os.path.getsize(path)
                    ok(f"Selected: {candidates[idx - 1]} ({sz // 1024} KB)")
                    return path
                if idx == custom_idx:
                    break
            warn(f"Please enter a number between 1 and {custom_idx}.")
    else:
        info(f"No firmware files found in {fw_dir}.")

    # Manual path entry
    while True:
        path = prompt(f"Path to {label}")
        if os.path.isfile(path):
            sz = os.path.getsize(path)
            ok(f"File: {path} ({sz // 1024} KB)")
            return path
        warn(f"File not found: {path}")


# ── Multi-shelf IP collection ─────────────────────────────────────────────────
def _collect_shelves() -> list:
    """
    Prompt for the password and a single IOM IP address per shelf,
    offering to add more shelves after each entry.

    Either IOM's IP can be used — both IOMs share the same Redfish endpoint
    and can reach each other's resources.  Each shelf has its own password
    because the Admin password is the BMC serial number, unique per shelf.

    Returns a list of (password, ip) tuples, one per shelf.
    """
    shelves   = []
    shelf_num = 1

    while True:
        info(f"Shelf {shelf_num} — enter password and IOM IP address.")
        password = prompt_password()
        ip = prompt_ip(f"  Shelf {shelf_num} IOM IP address (IOM1 or IOM2)")
        shelves.append((password, ip))
        print()

        if not prompt_yn("Add another shelf?", default=False):
            break
        shelf_num += 1
        print()

    return shelves


def _make_targets(shelves: list, iom_choice: str) -> list:
    """
    Convert the shelves structure into a flat list of (label, iom, ip, password)
    tuples suitable for _show_fw_versions().  When there is only one shelf the
    label is just the IOM name; for multiple shelves it includes the shelf number.
    All IOMs in a shelf share the same IP connection.
    """
    ioms = []
    if iom_choice in ("1", "3"):
        ioms.append("IOM1")
    if iom_choice in ("2", "3"):
        ioms.append("IOM2")

    multi = len(shelves) > 1
    return [
        (f"S{i} / {iom}" if multi else iom, iom, ip, password)
        for i, (password, ip) in enumerate(shelves, 1)
        for iom in ioms
    ]


# ── Per-IOM update helpers ────────────────────────────────────────────────────
def _update_iom_fw(password: str, ip: str, iom: str, fw_path: str) -> bool:
    """Upload and apply IOM firmware for one IOM, then restart it."""
    sz = os.path.getsize(fw_path)
    info(f"Uploading IOM firmware ({sz // 1024} KB) to {iom} at {ip}...")
    ok_flag, data = _redfish_upload_firmware(password, ip, fw_path)
    if not ok_flag:
        error(f"Upload failed: {data}")
        return False
    ok("Firmware file uploaded.")

    info(f"Triggering {iom} firmware update...")
    ok_flag, data = _redfish_trigger_update(
        password, ip, f"/redfish/v1/Managers/{iom}",
    )
    if not ok_flag:
        error(f"Update trigger failed: {data}")
        return False
    ok("Update triggered.")

    info("Monitoring update progress (this may take several minutes)...")
    ok_flag, msg = _redfish_poll_tasks(password, ip)
    if not ok_flag:
        warn(f"Task monitoring ended: {msg}")
    else:
        ok(msg)

    info(f"Restarting {iom}...")
    _redfish_restart_iom(password, ip, iom)   # connection drop on restart is normal
    ok(f"{iom} restart initiated. Waiting for IOM to come back online...")
    time.sleep(30)  # allow time for the IOM to begin shutting down before polling
    if _wait_for_iom_online(password, ip):
        ok(f"{iom} is back online.")
        info("Allowing IOM services to fully initialize before next step...")
        time.sleep(60)
    else:
        warn(f"{iom} did not respond within 5 minutes — proceeding anyway.")
    return True


def _update_fabric_fw(password: str, ip: str, iom: str, fw_path: str) -> bool:
    """
    Upload and apply Fabric Card firmware for one IOM.
    Per the service guide, the firmware file must be re-uploaded even if it was
    already uploaded during the IOM firmware step.
    After the update: restart fabric card, then restart IOM.
    """
    sz = os.path.getsize(fw_path)

    # Retry the upload — after an IOM reboot the inter-IOM services can take
    # time to initialize, causing the first upload attempt to fail with
    # "Failed to send update package to other IOM".
    MAX_UPLOAD_ATTEMPTS = 3
    ok_flag, data = False, ""
    for attempt in range(1, MAX_UPLOAD_ATTEMPTS + 1):
        info(f"Uploading Fabric Card firmware ({sz // 1024} KB) to {iom} at {ip}"
             + (f" (attempt {attempt}/{MAX_UPLOAD_ATTEMPTS})" if attempt > 1 else "") + "...")
        ok_flag, data = _redfish_upload_firmware(password, ip, fw_path)
        if ok_flag:
            break
        if attempt < MAX_UPLOAD_ATTEMPTS:
            warn(f"Upload failed: {data}")
            info("Waiting 60s for IOM services to finish initializing...")
            time.sleep(60)

    if not ok_flag:
        error(f"Upload failed after {MAX_UPLOAD_ATTEMPTS} attempts: {data}")
        return False
    ok("Firmware file uploaded.")

    info(f"Triggering {iom} Fabric Card firmware update...")
    ok_flag, data = _redfish_trigger_update(
        password, ip, f"/redfish/v1/Chassis/{iom}/NetworkAdapters/1",
    )
    if not ok_flag:
        error(f"Update trigger failed: {data}")
        return False
    ok("Update triggered.")

    info("Monitoring update progress...")
    ok_flag, msg = _redfish_poll_tasks(password, ip)
    if not ok_flag:
        warn(f"Task monitoring ended: {msg}")
    else:
        ok(msg)

    info(f"Restarting {iom} Fabric Card...")
    _redfish_reset_fabric(password, ip, iom)
    ok("Fabric Card restart initiated. Waiting 15s...")
    time.sleep(15)

    info(f"Restarting {iom} after Fabric Card update...")
    _redfish_restart_iom(password, ip, iom)
    ok(f"{iom} restart initiated. Waiting for IOM to come back online...")
    time.sleep(30)  # allow time for the IOM to begin shutting down before polling
    if _wait_for_iom_online(password, ip):
        ok(f"{iom} is back online.")
    else:
        warn(f"{iom} did not respond within 5 minutes — proceeding anyway.")
    return True


# ── Firmware Update Workflow ──────────────────────────────────────────────────
def firmware_update_workflow():
    """
    Standalone firmware update for IOM and Fabric Card firmware.
    Connects to each IOM via its network IP (not serial loopback) — uploading
    firmware over 115200-baud serial would be impractically slow.
    Supports updating multiple shelves in a single run.
    """
    banner()
    rule("IOM & Fabric Card Firmware Update")
    info("This procedure connects to each IOM via its network IP address.")
    info("Ensure this system has network access to the IOM management interface.")
    print()

    print("  Which IOM(s) would you like to update?")
    print(f"    {_c(C.BOLD, '1')}  IOM1 only")
    print(f"    {_c(C.BOLD, '2')}  IOM2 only")
    print(f"    {_c(C.BOLD, '3')}  Both IOM1 and IOM2  {_c(C.YEL, '(Do not use on production systems)')}")
    print()

    while True:
        iom_choice = prompt("Select option [1-3]")
        if iom_choice in ("1", "2", "3"):
            break
        warn("Please enter 1, 2, or 3.")
    print()

    # ── Collect IPs for one or more shelves ───────────────────────────────────
    shelves = _collect_shelves()
    targets = _make_targets(shelves, iom_choice)

    rule("Current Firmware Versions")
    _show_fw_versions(targets)

    print("  What would you like to update?")
    print(f"    {_c(C.BOLD, '1')}  IOM Firmware only")
    print(f"    {_c(C.BOLD, '2')}  Fabric Card Firmware only")
    print(f"    {_c(C.BOLD, '3')}  Both IOM and Fabric Card Firmware")
    print(f"    {_c(C.BOLD, '4')}  Cancel")
    print()

    while True:
        choice = prompt("Select option [1-4]")
        if choice in ("1", "2", "3", "4"):
            break
        warn("Please enter 1, 2, 3, or 4.")

    if choice == "4":
        info("Firmware update cancelled.")
        return

    update_iom    = choice in ("1", "3")
    update_fabric = choice in ("2", "3")

    fw_path = _prompt_fw_file("firmware file")

    print()
    warn("For HA systems: update the passive IOM first.")
    if len(shelves) > 1:
        warn(f"Updating {len(shelves)} shelves sequentially — Shelf 1 first.")
    elif iom_choice == "3":
        warn("IOM1 will be updated first — adjust order if IOM2 is passive.")
    print()

    if not prompt_yn("Proceed with firmware update?", default=True):
        info("Firmware update cancelled.")
        return

    # ── Run updates per target (shelf × IOM) ─────────────────────────────────
    for label, iom, ip, password in targets:
        rule(f"{label} ({ip})")
        if update_iom:
            _update_iom_fw(password, ip, iom, fw_path)
        if update_fabric:
            _update_fabric_fw(password, ip, iom, fw_path)

    rule("Post-Update Firmware Validation")
    _show_fw_versions(targets)

    ok("Firmware update complete.")
    print()
    draw_box([
        f"{_c(C.YEL, 'IMPORTANT -- For HA (Dual-Controller) Systems:')}",
        "",
        "After updating this controller's IOMs:",
        "  1. Log into TrueNAS and initiate a failover.",
        "  2. Re-run this tool to update the other controller.",
    ], colour=C.YEL)
    print()
    prompt("Press Enter to return to main menu")