From 1f527476e689ad8707a89bc9455faac1ea87861a Mon Sep 17 00:00:00 2001 From: scott Date: Thu, 5 Mar 2026 11:27:16 -0500 Subject: [PATCH] Add CSV import support for non-TrueNAS sources - Add truenas_migrate/csv_source.py: parses SMB and NFS share definitions from customer-supplied CSV files; returns same dict shape as parse_archive() so migrate.py is untouched - Add smb_shares_template.csv and nfs_shares_template.csv with annotated headers and example rows (# comment rows are skipped by the parser) - Update cli.py: interactive wizard gains a source-type step (archive vs CSV); run() resolves CSV source via --smb-csv / --nfs-csv args; --debug-tar is now optional; argparse validates mutual exclusion of archive and CSV flags Co-Authored-By: Claude Sonnet 4.6 --- nfs_shares_template.csv | 8 ++ smb_shares_template.csv | 7 ++ truenas_migrate/cli.py | 223 +++++++++++++++++++++++----------- truenas_migrate/csv_source.py | 160 ++++++++++++++++++++++++ 4 files changed, 326 insertions(+), 72 deletions(-) create mode 100644 nfs_shares_template.csv create mode 100644 smb_shares_template.csv create mode 100644 truenas_migrate/csv_source.py diff --git a/nfs_shares_template.csv b/nfs_shares_template.csv new file mode 100644 index 0000000..6284d9a --- /dev/null +++ b/nfs_shares_template.csv @@ -0,0 +1,8 @@ +path,comment,ro,maproot_user,maproot_group,mapall_user,mapall_group,security,hosts,networks,enabled +# Required columns : path +# security values : SYS KRB5 KRB5I KRB5P (space-separated for multiple; e.g. "SYS KRB5") +# hosts : space-separated hostnames or IPs allowed to mount (empty = any host) +# networks : space-separated CIDR networks (e.g. "192.168.1.0/24 10.0.0.0/8") +# Boolean columns : ro enabled (true or false) +# Lines starting with # are ignored. Delete the example row below and add your shares. +/mnt/pool/export,Example NFS export,false,root,wheel,,,SYS,,,true diff --git a/smb_shares_template.csv b/smb_shares_template.csv new file mode 100644 index 0000000..18eaf7c --- /dev/null +++ b/smb_shares_template.csv @@ -0,0 +1,7 @@ +name,path,comment,purpose,ro,browsable,guestok,abe,hostsallow,hostsdeny,timemachine,enabled +# Required columns : name path +# purpose values : NO_PRESET DEFAULT_SHARE ENHANCED_TIMEMACHINE MULTI_PROTOCOL_NFS PRIVATE_DATASETS WORM_DROPBOX +# List columns : hostsallow hostsdeny (space-separated; e.g. "192.168.1.10 192.168.1.11") +# Boolean columns : ro browsable guestok abe timemachine enabled (true or false) +# Lines starting with # are ignored. Delete the example row below and add your shares. +example-share,/mnt/pool/share,Example share description,NO_PRESET,false,true,false,false,,,false,true diff --git a/truenas_migrate/cli.py b/truenas_migrate/cli.py index d9581b3..61949ee 100644 --- a/truenas_migrate/cli.py +++ b/truenas_migrate/cli.py @@ -1,9 +1,9 @@ """ truenas_migrate – TrueNAS Share Migration Tool ================================================= -Reads SMB shares and NFS shares from a TrueNAS debug archive (.tar / .tgz) -produced by the built-in "Save Debug" feature, then re-creates them on a -destination TrueNAS system via the JSON-RPC 2.0 WebSocket API (TrueNAS 25.04+). +Reads SMB shares and NFS shares from either a TrueNAS debug archive (.tar / .tgz) +or customer-supplied CSV files, then re-creates them on a destination TrueNAS +system via the JSON-RPC 2.0 WebSocket API (TrueNAS 25.04+). SAFE BY DEFAULT • Existing shares are never overwritten or deleted. @@ -12,7 +12,7 @@ SAFE BY DEFAULT REQUIREMENTS Python 3.9+ (stdlib only – no external packages needed) -QUICK START +QUICK START — Archive source # 1. Inspect your debug archive to confirm it contains the data you need: python -m truenas_migrate --debug-tar debug.tgz --list-archive @@ -29,12 +29,14 @@ QUICK START --dest 192.168.1.50 \\ --api-key "1-xxxxxxxxxxxx" - # 4. Migrate only SMB shares (skip NFS): +QUICK START — CSV source + # Fill in smb_shares_template.csv / nfs_shares_template.csv, then: python -m truenas_migrate \\ - --debug-tar debug.tgz \\ + --smb-csv smb_shares.csv \\ + --nfs-csv nfs_shares.csv \\ --dest 192.168.1.50 \\ --api-key "1-xxxxxxxxxxxx" \\ - --migrate smb + --dry-run CONFLICT POLICY Shares that already exist on the destination are silently skipped: @@ -53,7 +55,8 @@ from typing import Optional from .archive import parse_archive, list_archive_and_exit from .client import TrueNASClient, check_dataset_paths, create_missing_datasets -from .colors import log, _bold, _bold_cyan, _bold_yellow, _cyan, _dim, _green, _yellow +from .colors import log, _bold, _bold_cyan, _bold_red, _bold_yellow, _cyan, _dim, _green, _yellow +from .csv_source import parse_csv_sources from .migrate import migrate_smb_shares, migrate_nfs_shares from .summary import Summary @@ -67,7 +70,13 @@ async def run( archive: Optional[dict] = None, ) -> Summary: if archive is None: - archive = parse_archive(args.debug_tar) + smb_csv = getattr(args, "smb_csv", None) + nfs_csv = getattr(args, "nfs_csv", None) + if smb_csv or nfs_csv: + archive = parse_csv_sources(smb_csv, nfs_csv) + else: + archive = parse_archive(args.debug_tar) + migrate_set = set(args.migrate) if args.dry_run: @@ -133,6 +142,20 @@ def _confirm(label: str) -> bool: return False +def _prompt_csv_path(share_type: str) -> Optional[str]: + """Prompt for a CSV file path. Returns resolved path string or None if skipped.""" + template = f"{share_type.lower()}_shares_template.csv" + print(f" {_dim('(template: ' + template + ')')}") + while True: + raw = _prompt(f" {share_type} shares CSV path (Enter to skip)") + if not raw: + return None + p = Path(raw) + if p.is_file(): + return str(p) + print(f" {_bold_red('File not found:')} {raw}") + + def _select_shares(shares: list[dict], share_type: str) -> list[dict]: """ Display a numbered list of *shares* and return only those the user selects. @@ -141,7 +164,7 @@ def _select_shares(shares: list[dict], share_type: str) -> list[dict]: if not shares: return shares - print(f"\n {_bold(f'{share_type} shares in archive ({len(shares)}):')} \n") + print(f"\n {_bold(f'{share_type} shares ({len(shares)}):')} \n") for i, share in enumerate(shares, 1): if share_type == "SMB": name = share.get("name", "") @@ -189,40 +212,21 @@ def _select_shares(shares: list[dict], share_type: str) -> list[dict]: # ───────────────────────────────────────────────────────────────────────────── def interactive_mode() -> None: - """Interactive wizard: pick archive → configure → dry run → confirm → apply.""" + """Interactive wizard: pick source → configure → dry run → confirm → apply.""" print( f"\n{_bold_cyan(' TrueNAS Share Migration Tool')}\n" - f" {_dim('Migrate SMB/NFS shares from a debug archive to a live system.')}\n" + f" {_dim('Migrate SMB/NFS shares to a live TrueNAS system.')}\n" ) - # 1 ── Locate debug archive ──────────────────────────────────────────────── - archives = _find_debug_archives() - if not archives: - sys.exit( - "No debug archives (.tgz / .tar.gz / .tar / .txz) found in the " - "current directory.\n" - "Copy your TrueNAS debug file here, or use --debug-tar to specify a path." - ) - - if len(archives) == 1: - chosen = archives[0] - print(f" {_dim('Archive:')} {_bold(chosen.name)} " - f"{_dim('(' + f'{chosen.stat().st_size / 1_048_576:.1f} MB' + ')')}\n") - else: - print(f" {_bold('Debug archives found:')}\n") - for i, p in enumerate(archives, 1): - print(f" {_cyan(str(i) + '.')} {p.name} " - f"{_dim('(' + f'{p.stat().st_size / 1_048_576:.1f} MB' + ')')}") - print() - while True: - raw = _prompt(f"Select archive [1-{len(archives)}]") - if raw.isdigit() and 1 <= int(raw) <= len(archives): - chosen = archives[int(raw) - 1] - break - print(f" Enter a number from 1 to {len(archives)}.") - - # 2 ── Destination ───────────────────────────────────────────────────────── + # 1 ── Source type ────────────────────────────────────────────────────────── + print(f" {_bold('Source type:')}") + print(f" {_cyan('1.')} TrueNAS debug archive (.tgz / .tar)") + print(f" {_cyan('2.')} CSV import (non-TrueNAS source)") + src_raw = _prompt(" Select source [1/2]", default="1") + use_csv = src_raw.strip() == "2" print() + + # 2 ── Destination ────────────────────────────────────────────────────────── host = "" while not host: host = _prompt("Destination TrueNAS host or IP") @@ -232,7 +236,7 @@ def interactive_mode() -> None: port_raw = _prompt("WebSocket port", default="443") port = int(port_raw) if port_raw.isdigit() else 443 - # 3 ── API key ───────────────────────────────────────────────────────────── + # 3 ── API key ────────────────────────────────────────────────────────────── api_key = "" while not api_key: try: @@ -243,26 +247,74 @@ def interactive_mode() -> None: if not api_key: print(" API key is required.") - # 4 ── Migration scope ───────────────────────────────────────────────────── - print(f"\n {_bold('What to migrate?')}") - print(f" {_cyan('1.')} SMB shares") - print(f" {_cyan('2.')} NFS shares") - sel_raw = _prompt( - "Selection (space-separated numbers, Enter for all)", default="1 2" - ) - _sel_map = {"1": "smb", "2": "nfs"} - migrate: list[str] = [] - for tok in sel_raw.split(): - if tok in _sel_map and _sel_map[tok] not in migrate: - migrate.append(_sel_map[tok]) - if not migrate: - migrate = ["smb", "nfs"] + if use_csv: + # ── CSV source ────────────────────────────────────────────────────────── + print(f"\n {_bold('CSV file paths:')}") + print(f" {_dim('Press Enter to skip a share type.')}\n") + smb_csv_path = _prompt_csv_path("SMB") + print() + nfs_csv_path = _prompt_csv_path("NFS") - # 5 ── Parse archive once (reused for dry + live runs) ──────────────────── - print() - archive_data = parse_archive(str(chosen)) + migrate: list[str] = [] + if smb_csv_path: + migrate.append("smb") + if nfs_csv_path: + migrate.append("nfs") + if not migrate: + sys.exit("No CSV files provided – nothing to migrate.") - # 5b ── Select individual shares ─────────────────────────────────────────── + print() + archive_data = parse_csv_sources(smb_csv_path, nfs_csv_path) + extra_ns: dict = {"smb_csv": smb_csv_path, "nfs_csv": nfs_csv_path} + + else: + # ── Archive source ────────────────────────────────────────────────────── + archives = _find_debug_archives() + if not archives: + sys.exit( + "No debug archives (.tgz / .tar.gz / .tar / .txz) found in the " + "current directory.\n" + "Copy your TrueNAS debug file here, or use --debug-tar to specify a path." + ) + + if len(archives) == 1: + chosen = archives[0] + print(f" {_dim('Archive:')} {_bold(chosen.name)} " + f"{_dim('(' + f'{chosen.stat().st_size / 1_048_576:.1f} MB' + ')')}\n") + else: + print(f" {_bold('Debug archives found:')}\n") + for i, p in enumerate(archives, 1): + print(f" {_cyan(str(i) + '.')} {p.name} " + f"{_dim('(' + f'{p.stat().st_size / 1_048_576:.1f} MB' + ')')}") + print() + while True: + raw = _prompt(f"Select archive [1-{len(archives)}]") + if raw.isdigit() and 1 <= int(raw) <= len(archives): + chosen = archives[int(raw) - 1] + break + print(f" Enter a number from 1 to {len(archives)}.") + + # ── Migration scope ───────────────────────────────────────────────────── + print(f"\n {_bold('What to migrate?')}") + print(f" {_cyan('1.')} SMB shares") + print(f" {_cyan('2.')} NFS shares") + sel_raw = _prompt( + "Selection (space-separated numbers, Enter for all)", default="1 2" + ) + _sel_map = {"1": "smb", "2": "nfs"} + migrate = [] + for tok in sel_raw.split(): + if tok in _sel_map and _sel_map[tok] not in migrate: + migrate.append(_sel_map[tok]) + if not migrate: + migrate = ["smb", "nfs"] + + # ── Parse archive ─────────────────────────────────────────────────────── + print() + archive_data = parse_archive(str(chosen)) + extra_ns = {"debug_tar": str(chosen)} + + # ── Select individual shares (common) ────────────────────────────────────── if "smb" in migrate and archive_data["smb_shares"]: archive_data["smb_shares"] = _select_shares(archive_data["smb_shares"], "SMB") if "nfs" in migrate and archive_data["nfs_shares"]: @@ -270,15 +322,15 @@ def interactive_mode() -> None: print() base_ns = dict( - debug_tar=str(chosen), dest=host, port=port, api_key=api_key, verify_ssl=False, migrate=migrate, + **extra_ns, ) - # 6 ── Dry run ───────────────────────────────────────────────────────────── + # 6 ── Dry run ────────────────────────────────────────────────────────────── dry_summary = asyncio.run( run(argparse.Namespace(**base_ns, dry_run=True), archive_data) ) @@ -314,7 +366,7 @@ def interactive_mode() -> None: print("Aborted – no changes made.") sys.exit(0) - # 7 ── Live run ──────────────────────────────────────────────────────────── + # 7 ── Live run ───────────────────────────────────────────────────────────── print() live_summary = asyncio.run( run(argparse.Namespace(**base_ns, dry_run=False), archive_data) @@ -336,23 +388,32 @@ def main() -> None: p = argparse.ArgumentParser( prog="truenas_migrate", description=( - "Migrate SMB and NFS shares from a TrueNAS debug archive " - "to a live destination system." + "Migrate SMB and NFS shares to a live TrueNAS destination system. " + "Source can be a TrueNAS debug archive or customer-supplied CSV files." ), formatter_class=argparse.RawDescriptionHelpFormatter, epilog=__doc__, ) # ── Source ──────────────────────────────────────────────────────────────── - p.add_argument( - "--debug-tar", required=True, metavar="FILE", + src = p.add_argument_group("source (choose one)") + src.add_argument( + "--debug-tar", metavar="FILE", help="Path to the TrueNAS debug .tar / .tgz from the SOURCE system.", ) + src.add_argument( + "--smb-csv", metavar="FILE", + help="Path to a CSV file containing SMB share definitions (non-TrueNAS source).", + ) + src.add_argument( + "--nfs-csv", metavar="FILE", + help="Path to a CSV file containing NFS share definitions (non-TrueNAS source).", + ) p.add_argument( "--list-archive", action="store_true", help=( "List all JSON files found in the archive and exit. " - "Run this first to verify the archive contains share data." + "Requires --debug-tar." ), ) @@ -396,7 +457,7 @@ def main() -> None: ) p.add_argument( "--dry-run", action="store_true", - help="Parse archive and connect to destination, but make no changes.", + help="Parse source and connect to destination, but make no changes.", ) p.add_argument( "--verbose", "-v", action="store_true", @@ -408,14 +469,32 @@ def main() -> None: if args.verbose: log.setLevel(logging.DEBUG) - if not Path(args.debug_tar).is_file(): - p.error(f"Archive not found: {args.debug_tar}") + has_archive = bool(args.debug_tar) + has_csv = bool(args.smb_csv or args.nfs_csv) - if args.list_archive: - list_archive_and_exit(args.debug_tar) # does not return + if has_archive and has_csv: + p.error("Cannot combine --debug-tar with --smb-csv / --nfs-csv.") + + if not has_archive and not has_csv: + p.error( + "Specify a source: --debug-tar FILE or --smb-csv / --nfs-csv FILE(s)." + ) + + if has_archive: + if not Path(args.debug_tar).is_file(): + p.error(f"Archive not found: {args.debug_tar}") + if args.list_archive: + list_archive_and_exit(args.debug_tar) # does not return + else: + if args.list_archive: + p.error("--list-archive requires --debug-tar.") + if args.smb_csv and not Path(args.smb_csv).is_file(): + p.error(f"SMB CSV not found: {args.smb_csv}") + if args.nfs_csv and not Path(args.nfs_csv).is_file(): + p.error(f"NFS CSV not found: {args.nfs_csv}") if not args.dest: - p.error("--dest is required (or use --list-archive to inspect the archive).") + p.error("--dest is required.") if not args.api_key: p.error("--api-key is required.") diff --git a/truenas_migrate/csv_source.py b/truenas_migrate/csv_source.py new file mode 100644 index 0000000..6a4943a --- /dev/null +++ b/truenas_migrate/csv_source.py @@ -0,0 +1,160 @@ +"""CSV source parser – reads SMB/NFS share definitions from customer-supplied CSV files.""" +from __future__ import annotations + +import csv +import sys +from pathlib import Path +from typing import Any + +from .colors import log + + +# ───────────────────────────────────────────────────────────────────────────── +# Column type metadata +# ───────────────────────────────────────────────────────────────────────────── + +# Columns coerced to bool +_SMB_BOOL_COLS = frozenset({"ro", "browsable", "guestok", "abe", "timemachine", "enabled"}) +# Columns coerced to list[str] (space-or-comma-separated in CSV) +_SMB_LIST_COLS = frozenset({"hostsallow", "hostsdeny"}) +_SMB_REQUIRED = frozenset({"name", "path"}) + +_NFS_BOOL_COLS = frozenset({"ro", "enabled"}) +_NFS_LIST_COLS = frozenset({"security", "hosts", "networks"}) +_NFS_REQUIRED = frozenset({"path"}) + + +# ───────────────────────────────────────────────────────────────────────────── +# Internal helpers +# ───────────────────────────────────────────────────────────────────────────── + +def _parse_bool(value: str, col: str, row_num: int) -> bool: + v = value.strip().lower() + if v in ("true", "yes", "1"): + return True + if v in ("false", "no", "0", ""): + return False + log.warning(" row %d: unrecognised boolean %r for column %r – treating as False", + row_num, value, col) + return False + + +def _parse_list(value: str) -> list[str]: + """Split space-or-comma-separated value into a list, dropping blanks.""" + return [p for p in value.replace(",", " ").split() if p] + + +def _coerce_row( + row: dict[str, str], + bool_cols: frozenset[str], + list_cols: frozenset[str], + required: frozenset[str], + row_num: int, +) -> dict[str, Any] | None: + """Validate and type-coerce one CSV row. Returns None to skip the row.""" + if not any((v or "").strip() for v in row.values()): + return None # blank row + + first_val = next(iter(row.values()), "") or "" + if first_val.strip().startswith("#"): + return None # comment row + + result: dict[str, Any] = {} + for col, raw in row.items(): + if col is None: + continue + col = col.strip() + val = (raw or "").strip() + + if not val: + continue # omit empty optional fields; API uses its defaults + + if col in bool_cols: + result[col] = _parse_bool(val, col, row_num) + elif col in list_cols: + result[col] = _parse_list(val) + else: + result[col] = val + + for req in required: + if req not in result: + log.warning(" row %d: missing required field %r – skipping row", row_num, req) + return None + + return result + + +def _parse_csv( + csv_path: str, + bool_cols: frozenset[str], + list_cols: frozenset[str], + required: frozenset[str], + label: str, +) -> list[dict]: + path = Path(csv_path) + if not path.is_file(): + log.error("%s CSV file not found: %s", label, csv_path) + sys.exit(1) + + shares: list[dict] = [] + try: + with path.open(newline="", encoding="utf-8-sig") as fh: + reader = csv.DictReader(fh) + if reader.fieldnames is None: + log.error("%s CSV has no header row: %s", label, csv_path) + sys.exit(1) + + header = {c.strip() for c in reader.fieldnames if c is not None} + missing_req = required - header + if missing_req: + log.error( + "%s CSV is missing required column(s): %s", + label, ", ".join(sorted(missing_req)), + ) + sys.exit(1) + + for row_num, row in enumerate(reader, start=2): + normalised = {(k or "").strip(): v for k, v in row.items()} + share = _coerce_row(normalised, bool_cols, list_cols, required, row_num) + if share is not None: + shares.append(share) + + except OSError as exc: + log.error("Cannot read %s CSV: %s", label, exc) + sys.exit(1) + + log.info(" %-14s → %s (%d share(s))", label.lower() + "_shares", csv_path, len(shares)) + return shares + + +# ───────────────────────────────────────────────────────────────────────────── +# Public API +# ───────────────────────────────────────────────────────────────────────────── + +def parse_smb_csv(csv_path: str) -> list[dict]: + """Parse an SMB shares CSV. Returns share dicts compatible with migrate.py.""" + return _parse_csv(csv_path, _SMB_BOOL_COLS, _SMB_LIST_COLS, _SMB_REQUIRED, "SMB") + + +def parse_nfs_csv(csv_path: str) -> list[dict]: + """Parse an NFS shares CSV. Returns share dicts compatible with migrate.py.""" + return _parse_csv(csv_path, _NFS_BOOL_COLS, _NFS_LIST_COLS, _NFS_REQUIRED, "NFS") + + +def parse_csv_sources(smb_csv: str | None, nfs_csv: str | None) -> dict[str, Any]: + """ + Parse one or both CSV files. + Returns {"smb_shares": list, "nfs_shares": list} — same shape as parse_archive(). + """ + log.info("Loading shares from CSV source(s).") + result: dict[str, Any] = {"smb_shares": [], "nfs_shares": []} + if smb_csv: + result["smb_shares"] = parse_smb_csv(smb_csv) + if nfs_csv: + result["nfs_shares"] = parse_nfs_csv(nfs_csv) + log.info( + "Loaded: %d SMB share(s), %d NFS share(s)", + len(result["smb_shares"]), + len(result["nfs_shares"]), + ) + return result