Validate RTX 3080 LeRobot environment

2026-09-09 10:43:51 +08:00
parent d71fcc7650
commit 2e33355a9c
12 changed files with 2773 additions and 14 deletions
@@ -1,13 +1,13 @@
 # GPU 环境与机器记录

-基线：LeRobot 0.6.1 / Python 3.12。脚本适用 Ubuntu x86_64 GPU 主机；不用于 Windows 或 RPi。当前 GPU 环境与模型训练均未验收。
+基线：LeRobot 0.6.1 / Python 3.12。脚本适用 Ubuntu x86_64 GPU 主机；不用于 Windows 或 RPi。3080 的 CUDA、视频和合成 ACT 短训练已验收，详见 [记录](../docs/p0-3080-20260908.md)；3090 与真实任务未验收。

 ## 分工与路径

 | 机器 | 用途 | 环境记录 |
 |---|---|---|
 | Windows | 编辑、静态检查、git 同步 | 本仓库 `D:\robot\lerobot` |
-| 3080 10GB | USB 遥操、采集、推理，首次解析依赖 | OS/驱动/Python/uv/锁 SHA256：待填 |
+| 3080 10GB | USB 遥操、采集、推理，首次解析依赖 | Ubuntu 24.04.3 / 550.163.01 / Python 3.12.3 / uv 0.12.10；锁见实测记录 |
 | 3090 24GB | 训练，同局域网 | OS/驱动/Python/uv/锁 SHA256：待填 |
 | RPi 5 | P3 起硬件驱动、采图、执行和故障处理 | ARM64 环境另行验证，不复用 x86_64 锁 |
 | DRIVE Orin | 可选实验 | 不阻塞主线 |
@@ -16,9 +16,9 @@ GPU 机统一路径：项目 `~/robot/proj`；专用环境 `~/robot/.venv-lerobo

 ## 首次安装与依赖锁

-前置：安装 NVIDIA 驱动、git、uv、FFmpeg。脚本用 uv 创建 Python 3.12 环境（必要时 uv 下载 Python），不自动 sudo 安装系统组件。缺少前置即退出。
+前置：安装 NVIDIA 驱动、git、uv、FFmpeg，以及构建遥操依赖 evdev 所需的 `build-essential`、`python3-dev`。Ubuntu 可执行 `sudo apt-get install -y ffmpeg build-essential python3-dev`。脚本用 uv 创建 Python 3.12 环境（必要时 uv 下载 Python），不自动 sudo 安装系统组件。缺少前置即退出。

-先在 3080：
+首次解析流程（本机已完成；新机器有仓库附带锁时直接使用下方默认安装命令）：

 ```bash
 mkdir -p ~/robot
@@ -35,7 +35,7 @@ bash ~/robot/proj/env/setup_lerobot.sh
 source ~/robot/.venv-lerobot-0.6.1/bin/activate
 ```

-默认安装严格使用锁与包哈希，不重新解析。两台比较报告中的锁 SHA256；验收通过后把锁、输入指纹与精简环境信息提交 git。**当前仓库尚未包含经过 Ubuntu/GPU 验证的传递依赖锁，默认安装会明确拒绝继续。**
+默认安装严格使用锁与包哈希，不重新解析。两台比较报告中的锁 SHA256；验收通过后把锁、输入指纹与精简环境信息提交 git。**当前仓库已包含 3080 验证的完整依赖锁和指纹，默认安装可直接使用。不要对现有锁执行 `--resolve`。**

 两台尽量统一 Ubuntu 版本，记录 Python 补丁版本、uv 版本与 NVIDIA 驱动；同一 Python 包锁不能锁定系统库或驱动。脚本 `uv pip sync` 仅作用于新命名的项目专用环境，会使其中包与锁一致，请勿把它用于其他项目。

@@ -53,7 +53,7 @@ rsync -avP ~/robot/data/ USER@TRAIN_HOST:~/robot/data/
 rsync -avP USER@TRAIN_HOST:~/robot/outputs/ ~/robot/outputs/
 ```

-每个实验保留数据版本/校验值、模型与处理器、配置、校准备份和评测记录。模型快照记录来源 revision，不自动随 Hub main 更新。同步代码使用 git；当前未配置 GPU 主机远程访问，因此 GPU 执行结果需从目标机器回传。
+每个实验保留数据版本/校验值、模型与处理器、配置、校准备份和评测记录。模型快照记录来源 revision，不自动随 Hub main 更新。同步代码使用 git；3080 为当前本机，3090 的远程访问尚未验证。

 ## 验收报告

@@ -0,0 +1 @@
+e236f8a3986eecd6bafb490108f52dd8adc1c9f76fd85d30c49a67ea00e3fd80  requirements.in
@@ -2,3 +2,8 @@
 # Resolve once on 3080; share the lock and its input fingerprint with 3090.
 # ACT uses the base dependencies; v0.6.1 has no 'act' extra.
 lerobot[core_scripts,training,feetech,smolvla]==0.6.1
+# Keep CUDA 12 wheels for the 3080 host's 550 driver; unconstrained resolution
+# selects torch 2.11 / CUDA 13. TorchCodec must match PyTorch's ABI.
+torch==2.7.1
+torchvision==0.22.1
+torchcodec==0.4.0
@@ -18,7 +18,7 @@ esac
 [[ "$(uname -s)" == Linux && "$(uname -m)" == x86_64 ]] || {
  echo "Only Linux x86_64 GPU hosts are supported; RPi needs a separate environment." >&2; exit 1;
 }
-for cmd in uv nvidia-smi ffmpeg sha256sum; do
+for cmd in uv nvidia-smi ffmpeg sha256sum gcc; do
  command -v "$cmd" >/dev/null || { echo "Missing prerequisite: $cmd" >&2; exit 1; }
 done
 if ! $RESOLVE && [[ ! -s "$LOCK" ]]; then
@@ -34,12 +34,19 @@ exec > >(tee "$REPORT/setup.log") 2>&1
 echo "[1/5] Prerequisites"
 nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv,noheader
 uv --version
+cat /etc/os-release > "$REPORT/os-release.txt"
+uname -srmo > "$REPORT/kernel.txt"
+nvidia-smi > "$REPORT/nvidia-smi.txt"
 ffmpeg -version > "$REPORT/ffmpeg.txt"
 # Set UV_DEFAULT_INDEX explicitly if a mirror is needed. No automatic Hub/token routing.
 echo "[2/5] Dedicated Python 3.12 environment"
 if [[ ! -d "$VENV" ]]; then uv venv --python 3.12 "$VENV"; fi
 PYTHON="${VENV}/bin/python"
 "$PYTHON" -c 'import sys; assert sys.version_info[:2] == (3, 12), sys.version'
+PYTHON_INCLUDE="$("$PYTHON" -c 'import sysconfig; print(sysconfig.get_path("include"))')"
+[[ -f "$PYTHON_INCLUDE/Python.h" ]] || {
+  echo "Missing Python development headers: $PYTHON_INCLUDE/Python.h" >&2; exit 1;
+}
 if $RESOLVE; then
  TEMP_LOCK="$(mktemp "${LOCK}.tmp.XXXXXX")"
  trap 'rm -f -- "$TEMP_LOCK"' EXIT
@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+"""Offline synthetic-data acceptance. No robot access; no task-quality claim."""
+import argparse
+import json
+import os
+import re
+from pathlib import Path
+import subprocess
+import sys
+import time
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("--height", type=int, default=96)
+    parser.add_argument("--width", type=int, default=128)
+    parser.add_argument("--episodes", type=int, default=3)
+    parser.add_argument("--frames-per-episode", type=int, default=24)
+    parser.add_argument("--batch-size", type=int, default=2)
+    parser.add_argument("--steps", type=int, default=10)
+    parser.add_argument("--chunk-size", type=int, default=8)
+    parser.add_argument("--pretrained-backbone", action="store_true")
+    args = parser.parse_args()
+    root = args.output.resolve()
+    root.mkdir(parents=True, exist_ok=False)
+    os.environ["HF_HUB_OFFLINE"] = "1"
+    os.environ["WANDB_MODE"] = "disabled"
+    import numpy as np
+    import torch
+    from lerobot.datasets.lerobot_dataset import LeRobotDataset
+    from lerobot.configs.video import RGBEncoderConfig
+
+    torch.set_num_threads(4)
+    assert torch.cuda.is_available()
+    features = {
+        "observation.state": {"dtype": "float32", "shape": (6,), "names": [f"joint_{i}" for i in range(6)]},
+        "action": {"dtype": "float32", "shape": (6,), "names": [f"joint_{i}" for i in range(6)]},
+        "observation.images.front": {"dtype": "video", "shape": (args.height, args.width, 3), "names": ["height", "width", "channels"]},
+    }
+    dataset = LeRobotDataset.create(
+        repo_id="local/p0_synthetic", root=root / "dataset", fps=10,
+        features=features, robot_type="synthetic", use_videos=True,
+        rgb_encoder=RGBEncoderConfig(vcodec="h264"), encoder_threads=2,
+    )
+    for episode in range(args.episodes):
+        for frame in range(args.frames_per_episode):
+            phase = frame / 10 + episode
+            state = np.sin(phase + np.arange(6)).astype(np.float32)
+            image = np.zeros((args.height, args.width, 3), dtype=np.uint8)
+            image[:, :, 1] = 40 + episode * 50
+            box = max(8, min(args.height, args.width) // 6)
+            x = frame * max(1, (args.width - box) // max(1, args.frames_per_episode - 1))
+            y = (args.height - box) // 2
+            image[y:y+box, x:x+box, 0] = 220
+            dataset.add_frame({"observation.state": state,
+                "action": np.sin(phase + 0.1 + np.arange(6)).astype(np.float32),
+                "observation.images.front": image, "task": "synthetic pipeline check"})
+        dataset.save_episode()
+    dataset.finalize()
+    from torchcodec.decoders import VideoDecoder
+    videos = list((root / "dataset").rglob("*.mp4"))
+    assert videos
+    for video in videos:
+        decoded = VideoDecoder(str(video))
+        assert len(decoded) > 0 and decoded[0].shape == (3, args.height, args.width)
+    frame_count = args.episodes * args.frames_per_episode
+    for backend in ("pyav", "torchcodec"):
+        loaded = LeRobotDataset("local/p0_synthetic", root=root / "dataset", video_backend=backend)
+        assert len(loaded) == frame_count
+        boundary_indices = sorted({0, frame_count - 1} | {
+            i for episode in range(1, args.episodes)
+            for i in (episode * args.frames_per_episode - 1, episode * args.frames_per_episode)
+        })
+        for index in boundary_indices:
+            item = loaded[index]
+            assert item["observation.images.front"].shape == (3, args.height, args.width)
+            assert torch.isfinite(item["action"]).all()
+
+    train_wrapper = (
+        "import json,torch; from pathlib import Path; "
+        "from lerobot.scripts.lerobot_train import main; "
+        "torch.cuda.reset_peak_memory_stats(); main(); "
+        f"Path({str(root / 'training_memory.json')!r}).write_text(json.dumps("
+        "{'allocated_mib':torch.cuda.max_memory_allocated()/2**20,"
+        "'reserved_mib':torch.cuda.max_memory_reserved()/2**20}))"
+    )
+    command = [sys.executable, "-c", train_wrapper,
+        "--policy.type=act", "--policy.device=cuda", "--policy.push_to_hub=false",
+        f"--policy.chunk_size={args.chunk_size}", f"--policy.n_action_steps={args.chunk_size}",
+        "--dataset.repo_id=local/p0_synthetic",
+        f"--dataset.root={root / 'dataset'}", "--dataset.video_backend=torchcodec",
+        f"--batch_size={args.batch_size}", f"--steps={args.steps}", "--num_workers=0", "--seed=42",
+        "--log_freq=1", "--save_freq=10", "--wandb.enable=false",
+        f"--output_dir={root / 'training'}", "--job_name=p0_synthetic"]
+    if not args.pretrained_backbone:
+        command.append("--policy.pretrained_backbone_weights=null")
+    env = dict(os.environ, OMP_NUM_THREADS="4", MKL_NUM_THREADS="4")
+    start = time.perf_counter()
+    with (root / "train.log").open("w") as log:
+        subprocess.run(command, env=env, stdout=log, stderr=subprocess.STDOUT, check=True)
+    elapsed = time.perf_counter() - start
+    checkpoints = sorted((root / "training" / "checkpoints").glob("[0-9]*/pretrained_model"))
+    assert checkpoints, "No checkpoint saved"
+    from lerobot.policies.act.modeling_act import ACTPolicy
+    from lerobot.policies.factory import make_pre_post_processors
+    checkpoint = checkpoints[-1]
+    policy = ACTPolicy.from_pretrained(checkpoint).to("cuda").eval()
+    pre, post = make_pre_post_processors(policy.config, pretrained_path=checkpoint,
+        preprocessor_overrides={"device_processor": {"device": "cuda"}})
+    observation = {k: v.unsqueeze(0) for k, v in loaded[0].items()
+        if k.startswith("observation.")}
+    timings = []
+    torch.cuda.reset_peak_memory_stats()
+    with torch.inference_mode():
+        for i in range(25):
+            policy.reset()
+            torch.cuda.synchronize()
+            begin = time.perf_counter()
+            action = post(policy.select_action(pre(observation)))
+            torch.cuda.synchronize()
+            assert action.shape == (1, 6) and torch.isfinite(action).all()
+            if i >= 5:
+                timings.append((time.perf_counter() - begin) * 1000)
+    report = {"status": "PASS", "scope": "synthetic software pipeline only; not robot/task validation",
+        "frames": frame_count, "episodes": args.episodes, "train_steps": args.steps,
+        "batch_size": args.batch_size, "chunk_size": args.chunk_size,
+        "image_shape": [args.height, args.width, 3],
+        "pretrained_backbone": args.pretrained_backbone,
+        "training_wall_seconds": elapsed,
+        # Upstream resets CUDA peak counters each step. Its per-step log is rounded to 0.01 GiB.
+        "training_peak_allocated_gib_from_step_logs": max(float(v) for v in re.findall(
+            r"mem_gb:([0-9.]+)", (root / "train.log").read_text())),
+        "training_last_step_memory": json.loads((root / "training_memory.json").read_text()),
+        "checkpoint": str(checkpoint),
+        "inference_ms_p50_p95_p99": np.percentile(timings, [50, 95, 99]).tolist(),
+        "inference_peak_allocated_mib": torch.cuda.max_memory_allocated() / 2**20,
+        "torch": torch.__version__, "cuda": torch.version.cuda,
+        "gpu": torch.cuda.get_device_name(0)}
+    (root / "result.json").write_text(json.dumps(report, indent=2) + "\n")
+    print(json.dumps(report, indent=2))
+
+
+if __name__ == "__main__":
+    main()
				`@@ -0,0 +1 @@`
				`e236f8a3986eecd6bafb490108f52dd8adc1c9f76fd85d30c49a67ea00e3fd80 requirements.in`