scott/TrueMigration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add CSV import support for non-TrueNAS sources

Browse Source 
- Add truenas_migrate/csv_source.py: parses SMB and NFS share definitions
  from customer-supplied CSV files; returns same dict shape as parse_archive()
  so migrate.py is untouched
- Add smb_shares_template.csv and nfs_shares_template.csv with annotated
  headers and example rows (# comment rows are skipped by the parser)
- Update cli.py: interactive wizard gains a source-type step (archive vs CSV);
  run() resolves CSV source via --smb-csv / --nfs-csv args; --debug-tar is now
  optional; argparse validates mutual exclusion of archive and CSV flags

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
scott
2026-03-05 11:27:16 -05:00
parent 880fa42f5b
commit 1f527476e6
4 changed files with 326 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

223
truenas_migrate/cli.py
View File

@@ -1,9 +1,9 @@
"""
truenas_migrate – TrueNAS Share Migration Tool
=================================================
Reads SMB shares and NFS shares from a TrueNAS debug archive (.tar / .tgz)
produced by the built-in "Save Debug" feature, then re-creates them on a
destination TrueNAS system via the JSON-RPC 2.0 WebSocket API (TrueNAS 25.04+).
Reads SMB shares and NFS shares from either a TrueNAS debug archive (.tar / .tgz)
or customer-supplied CSV files, then re-creates them on a destination TrueNAS
system via the JSON-RPC 2.0 WebSocket API (TrueNAS 25.04+).
SAFE BY DEFAULT
• Existing shares are never overwritten or deleted.
@@ -12,7 +12,7 @@ SAFE BY DEFAULT
REQUIREMENTS
Python 3.9+ (stdlib only – no external packages needed)
QUICK START
QUICK START — Archive source
# 1. Inspect your debug archive to confirm it contains the data you need:
python -m truenas_migrate --debug-tar debug.tgz --list-archive
@@ -29,12 +29,14 @@ QUICK START
--dest 192.168.1.50 \\
--api-key "1-xxxxxxxxxxxx"
# 4. Migrate only SMB shares (skip NFS):
QUICK START — CSV source
# Fill in smb_shares_template.csv / nfs_shares_template.csv, then:
python -m truenas_migrate \\
--debug-tar debug.tgz \\
--smb-csv smb_shares.csv \\
--nfs-csv nfs_shares.csv \\
--dest 192.168.1.50 \\
--api-key "1-xxxxxxxxxxxx" \\
--migrate smb
--dry-run
CONFLICT POLICY
Shares that already exist on the destination are silently skipped:
@@ -53,7 +55,8 @@ from typing import Optional
from .archive import parse_archive, list_archive_and_exit
from .client import TrueNASClient, check_dataset_paths, create_missing_datasets
from .colors import log, _bold, _bold_cyan, _bold_yellow, _cyan, _dim, _green, _yellow
from .colors import log, _bold, _bold_cyan, _bold_red, _bold_yellow, _cyan, _dim, _green, _yellow
from .csv_source import parse_csv_sources
from .migrate import migrate_smb_shares, migrate_nfs_shares
from .summary import Summary
@@ -67,7 +70,13 @@ async def run(
archive: Optional[dict] = None,
) -> Summary:
if archive is None:
archive = parse_archive(args.debug_tar)
smb_csv = getattr(args, "smb_csv", None)
nfs_csv = getattr(args, "nfs_csv", None)
if smb_csv or nfs_csv:
archive = parse_csv_sources(smb_csv, nfs_csv)
else:
archive = parse_archive(args.debug_tar)
migrate_set = set(args.migrate)
if args.dry_run:
@@ -133,6 +142,20 @@ def _confirm(label: str) -> bool:
return False
def _prompt_csv_path(share_type: str) -> Optional[str]:
"""Prompt for a CSV file path. Returns resolved path string or None if skipped."""
template = f"{share_type.lower()}_shares_template.csv"
print(f" {_dim('(template: ' + template + ')')}")
while True:
raw = _prompt(f" {share_type} shares CSV path (Enter to skip)")
if not raw:
return None
p = Path(raw)
if p.is_file():
return str(p)
print(f" {_bold_red('File not found:')} {raw}")
def _select_shares(shares: list[dict], share_type: str) -> list[dict]:
"""
Display a numbered list of *shares* and return only those the user selects.
@@ -141,7 +164,7 @@ def _select_shares(shares: list[dict], share_type: str) -> list[dict]:
if not shares:
return shares
print(f"\n {_bold(f'{share_type} shares in archive ({len(shares)}):')} \n")
print(f"\n {_bold(f'{share_type} shares ({len(shares)}):')} \n")
for i, share in enumerate(shares, 1):
if share_type == "SMB":
name = share.get("name", "<unnamed>")
@@ -189,40 +212,21 @@ def _select_shares(shares: list[dict], share_type: str) -> list[dict]:
# ─────────────────────────────────────────────────────────────────────────────
def interactive_mode() -> None:
"""Interactive wizard: pick archive → configure → dry run → confirm → apply."""
"""Interactive wizard: pick source → configure → dry run → confirm → apply."""
print(
f"\n{_bold_cyan(' TrueNAS Share Migration Tool')}\n"
f" {_dim('Migrate SMB/NFS shares from a debug archive to a live system.')}\n"
f" {_dim('Migrate SMB/NFS shares to a live TrueNAS system.')}\n"
)
# 1 ── Locate debug archive ────────────────────────────────────────────────
archives = _find_debug_archives()
if not archives:
sys.exit(
"No debug archives (.tgz / .tar.gz / .tar / .txz) found in the "
"current directory.\n"
"Copy your TrueNAS debug file here, or use --debug-tar to specify a path."
)
if len(archives) == 1:
chosen = archives[0]
print(f" {_dim('Archive:')} {_bold(chosen.name)} "
f"{_dim('(' + f'{chosen.stat().st_size / 1_048_576:.1f} MB' + ')')}\n")
else:
print(f" {_bold('Debug archives found:')}\n")
for i, p in enumerate(archives, 1):
print(f" {_cyan(str(i) + '.')} {p.name} "
f"{_dim('(' + f'{p.stat().st_size / 1_048_576:.1f} MB' + ')')}")
print()
while True:
raw = _prompt(f"Select archive [1-{len(archives)}]")
if raw.isdigit() and 1 <= int(raw) <= len(archives):
chosen = archives[int(raw) - 1]
break
print(f" Enter a number from 1 to {len(archives)}.")
# 2 ── Destination ─────────────────────────────────────────────────────────
# 1 ── Source type ──────────────────────────────────────────────────────────
print(f" {_bold('Source type:')}")
print(f" {_cyan('1.')} TrueNAS debug archive (.tgz / .tar)")
print(f" {_cyan('2.')} CSV import (non-TrueNAS source)")
src_raw = _prompt(" Select source [1/2]", default="1")
use_csv = src_raw.strip() == "2"
print()
# 2 ── Destination ──────────────────────────────────────────────────────────
host = ""
while not host:
host = _prompt("Destination TrueNAS host or IP")
@@ -232,7 +236,7 @@ def interactive_mode() -> None:
port_raw = _prompt("WebSocket port", default="443")
port = int(port_raw) if port_raw.isdigit() else 443
# 3 ── API key ─────────────────────────────────────────────────────────────
# 3 ── API key ──────────────────────────────────────────────────────────────
api_key = ""
while not api_key:
try:
@@ -243,26 +247,74 @@ def interactive_mode() -> None:
if not api_key:
print(" API key is required.")
# 4 ── Migration scope ─────────────────────────────────────────────────────
print(f"\n {_bold('What to migrate?')}")
print(f" {_cyan('1.')} SMB shares")
print(f" {_cyan('2.')} NFS shares")
sel_raw = _prompt(
"Selection (space-separated numbers, Enter for all)", default="1 2"
)
_sel_map = {"1": "smb", "2": "nfs"}
migrate: list[str] = []
for tok in sel_raw.split():
if tok in _sel_map and _sel_map[tok] not in migrate:
migrate.append(_sel_map[tok])
if not migrate:
migrate = ["smb", "nfs"]
if use_csv:
# ── CSV source ──────────────────────────────────────────────────────────
print(f"\n {_bold('CSV file paths:')}")
print(f" {_dim('Press Enter to skip a share type.')}\n")
smb_csv_path = _prompt_csv_path("SMB")
print()
nfs_csv_path = _prompt_csv_path("NFS")
# 5 ── Parse archive once (reused for dry + live runs) ────────────────────
print()
archive_data = parse_archive(str(chosen))
migrate: list[str] = []
if smb_csv_path:
migrate.append("smb")
if nfs_csv_path:
migrate.append("nfs")
if not migrate:
sys.exit("No CSV files provided – nothing to migrate.")
# 5b ── Select individual shares ───────────────────────────────────────────
print()
archive_data = parse_csv_sources(smb_csv_path, nfs_csv_path)
extra_ns: dict = {"smb_csv": smb_csv_path, "nfs_csv": nfs_csv_path}
else:
# ── Archive source ──────────────────────────────────────────────────────
archives = _find_debug_archives()
if not archives:
sys.exit(
"No debug archives (.tgz / .tar.gz / .tar / .txz) found in the "
"current directory.\n"
"Copy your TrueNAS debug file here, or use --debug-tar to specify a path."
)
if len(archives) == 1:
chosen = archives[0]
print(f" {_dim('Archive:')} {_bold(chosen.name)} "
f"{_dim('(' + f'{chosen.stat().st_size / 1_048_576:.1f} MB' + ')')}\n")
else:
print(f" {_bold('Debug archives found:')}\n")
for i, p in enumerate(archives, 1):
print(f" {_cyan(str(i) + '.')} {p.name} "
f"{_dim('(' + f'{p.stat().st_size / 1_048_576:.1f} MB' + ')')}")
print()
while True:
raw = _prompt(f"Select archive [1-{len(archives)}]")
if raw.isdigit() and 1 <= int(raw) <= len(archives):
chosen = archives[int(raw) - 1]
break
print(f" Enter a number from 1 to {len(archives)}.")
# ── Migration scope ─────────────────────────────────────────────────────
print(f"\n {_bold('What to migrate?')}")
print(f" {_cyan('1.')} SMB shares")
print(f" {_cyan('2.')} NFS shares")
sel_raw = _prompt(
"Selection (space-separated numbers, Enter for all)", default="1 2"
)
_sel_map = {"1": "smb", "2": "nfs"}
migrate = []
for tok in sel_raw.split():
if tok in _sel_map and _sel_map[tok] not in migrate:
migrate.append(_sel_map[tok])
if not migrate:
migrate = ["smb", "nfs"]
# ── Parse archive ───────────────────────────────────────────────────────
print()
archive_data = parse_archive(str(chosen))
extra_ns = {"debug_tar": str(chosen)}
# ── Select individual shares (common) ──────────────────────────────────────
if "smb" in migrate and archive_data["smb_shares"]:
archive_data["smb_shares"] = _select_shares(archive_data["smb_shares"], "SMB")
if "nfs" in migrate and archive_data["nfs_shares"]:
@@ -270,15 +322,15 @@ def interactive_mode() -> None:
print()
base_ns = dict(
debug_tar=str(chosen),
dest=host,
port=port,
api_key=api_key,
verify_ssl=False,
migrate=migrate,
**extra_ns,
)
# 6 ── Dry run ─────────────────────────────────────────────────────────────
# 6 ── Dry run ──────────────────────────────────────────────────────────────
dry_summary = asyncio.run(
run(argparse.Namespace(**base_ns, dry_run=True), archive_data)
)
@@ -314,7 +366,7 @@ def interactive_mode() -> None:
print("Aborted – no changes made.")
sys.exit(0)
# 7 ── Live run ────────────────────────────────────────────────────────────
# 7 ── Live run ─────────────────────────────────────────────────────────────
print()
live_summary = asyncio.run(
run(argparse.Namespace(**base_ns, dry_run=False), archive_data)
@@ -336,23 +388,32 @@ def main() -> None:
p = argparse.ArgumentParser(
prog="truenas_migrate",
description=(
"Migrate SMB and NFS shares from a TrueNAS debug archive "
"to a live destination system."
"Migrate SMB and NFS shares to a live TrueNAS destination system. "
"Source can be a TrueNAS debug archive or customer-supplied CSV files."
),
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=__doc__,
)
# ── Source ────────────────────────────────────────────────────────────────
p.add_argument(
"--debug-tar", required=True, metavar="FILE",
src = p.add_argument_group("source (choose one)")
src.add_argument(
"--debug-tar", metavar="FILE",
help="Path to the TrueNAS debug .tar / .tgz from the SOURCE system.",
)
src.add_argument(
"--smb-csv", metavar="FILE",
help="Path to a CSV file containing SMB share definitions (non-TrueNAS source).",
)
src.add_argument(
"--nfs-csv", metavar="FILE",
help="Path to a CSV file containing NFS share definitions (non-TrueNAS source).",
)
p.add_argument(
"--list-archive", action="store_true",
help=(
"List all JSON files found in the archive and exit. "
"Run this first to verify the archive contains share data."
"Requires --debug-tar."
),
)
@@ -396,7 +457,7 @@ def main() -> None:
)
p.add_argument(
"--dry-run", action="store_true",
help="Parse archive and connect to destination, but make no changes.",
help="Parse source and connect to destination, but make no changes.",
)
p.add_argument(
"--verbose", "-v", action="store_true",
@@ -408,14 +469,32 @@ def main() -> None:
if args.verbose:
log.setLevel(logging.DEBUG)
if not Path(args.debug_tar).is_file():
p.error(f"Archive not found: {args.debug_tar}")
has_archive = bool(args.debug_tar)
has_csv = bool(args.smb_csv or args.nfs_csv)
if args.list_archive:
list_archive_and_exit(args.debug_tar) # does not return
if has_archive and has_csv:
p.error("Cannot combine --debug-tar with --smb-csv / --nfs-csv.")
if not has_archive and not has_csv:
p.error(
"Specify a source: --debug-tar FILE or --smb-csv / --nfs-csv FILE(s)."
)
if has_archive:
if not Path(args.debug_tar).is_file():
p.error(f"Archive not found: {args.debug_tar}")
if args.list_archive:
list_archive_and_exit(args.debug_tar) # does not return
else:
if args.list_archive:
p.error("--list-archive requires --debug-tar.")
if args.smb_csv and not Path(args.smb_csv).is_file():
p.error(f"SMB CSV not found: {args.smb_csv}")
if args.nfs_csv and not Path(args.nfs_csv).is_file():
p.error(f"NFS CSV not found: {args.nfs_csv}")
if not args.dest:
p.error("--dest is required (or use --list-archive to inspect the archive).")
p.error("--dest is required.")
if not args.api_key:
p.error("--api-key is required.")