Сортировочная ячейка Isaac Sim: CV-пайплайн и меши товаров
Замкнутый контур "поток -> CV -> механика": товары идут по конвейеру с шагом 700 мм, класс определяется стереопайплайном во время движения, пушер и плуг реагируют физически. Состав: * control_test/ - ячейка и CV. run_sorting_cv.py + cv_worker.py (два процесса, потому что torch внутри Isaac роняет сцену), cell.py (физика лент, плуга, пушера), measure_plane.py (замер габаритов), README.md и .memory.md с замерами, проблемами и ловушками * robozon_sorter/ - модули симуляции, scripts/ - утилиты, scene/ - сцены * assets/ - меши товаров, плуг, объекты Objaverse Бейзлайн CV: DEFOM-Stereo vitl, вход 480, iters 24, кроп зоны осмотра, без сегментации. На потоке 700 мм - классы 8/9, габариты MAE 32.8 мм, 469 мс на товар при такте 700 мс. Веса моделей (4.5 ГБ) и пропсы конвейера NVIDIA (274 МБ) не включены - источники и команды скачивания в MODELS.md. Выход прогонов (captures/, runtime/) не включён: воспроизводится. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
dasha_f
@@ -0,0 +1,207 @@
|
||||
"""STAGE 1 (inside Isaac, no torch): render rectified L/R pairs + an EXACT object mask.
|
||||
|
||||
Two fixes over capture_cfg.py:
|
||||
* the reference used to be composed onto the same prim whose xformOpOrder we then cleared,
|
||||
which destroyed the mesh's own placement and dropped every item 0.6 m under the belt.
|
||||
The reference now lives on a child, so only our holder carries the placement.
|
||||
* ground truth is the mesh's real bbox in the scene, not the product catalogue - the two
|
||||
disagree by 2.0-2.8x per item, so catalogue dims cannot score a size prediction.
|
||||
|
||||
The mask is analytic (mesh points projected through the camera), the same trick the earlier
|
||||
ROI pipeline used for its GT channel - no segmentation error contaminates a geometry study.
|
||||
"""
|
||||
import json, os, sys
|
||||
REPO = "/home/dasha/robozon-sorter"
|
||||
for e in (REPO, f"{REPO}/control_test"):
|
||||
if e not in sys.path:
|
||||
sys.path.insert(0, e)
|
||||
for _m in [k for k in list(sys.modules) if k.startswith(("cam_configs", "classify", "cell"))]:
|
||||
del sys.modules[_m]
|
||||
import importlib; importlib.invalidate_caches()
|
||||
|
||||
import asyncio, numpy as np, cv2
|
||||
import omni.usd, omni.timeline
|
||||
import omni.kit.viewport.utility as vp
|
||||
import isaacsim.core.experimental.utils.app as app_utils
|
||||
from pxr import UsdPhysics, Gf, Usd, UsdGeom, UsdLux
|
||||
|
||||
import cam_configs as CC
|
||||
import cell
|
||||
import classify as CL
|
||||
|
||||
try: # --file runs isolated, so an injected arg lands in
|
||||
CFG = cfg # LOCALS, not globals() - the bare name catches both
|
||||
except NameError:
|
||||
CFG = CC.DEFAULT
|
||||
ITEMS = globals().get("items", ["bag", "backpack", "lunchbox", "helmet", "pillow",
|
||||
"detergent", "bucket", "box_400x400x300", "box_300x200x200"])
|
||||
OUT = f"{REPO}/control_test/captures/{CFG}"
|
||||
os.makedirs(OUT, exist_ok=True)
|
||||
|
||||
stage = omni.usd.get_context().get_stage()
|
||||
tl = omni.timeline.get_timeline_interface()
|
||||
if tl.is_playing():
|
||||
tl.stop(); await app_utils.update_app_async(steps=10)
|
||||
|
||||
calib = CC.apply_config(stage, CFG)
|
||||
print(f"config {CFG}: {len(calib)} камер | " + ", ".join(
|
||||
f"{n.replace('_Left','')} h={c['height_mm']:.0f} d={c['standoff_mm']:.0f}"
|
||||
for n, c in calib.items() if n.endswith("_Left")))
|
||||
|
||||
# without this the cell has no floor and no dome: everything off the belt renders as
|
||||
# void, which both looks wrong over WebRTC and starves the side views of bounce light
|
||||
_gl = cell.add_ground_and_light(stage)
|
||||
print(f"пол {_gl['ground']}, купол {_gl['light']}")
|
||||
|
||||
for nm, pos in (("K0", (-0.75, 1.6, 2.6)), ("K1", (-0.75, -1.6, 2.6)),
|
||||
("K2", (0.6, 0.0, 2.6)), ("K3", (-2.1, 0.0, 2.6))):
|
||||
p = f"/World/_CapLight_{nm}"
|
||||
if not stage.GetPrimAtPath(p).IsValid():
|
||||
sl = UsdLux.SphereLight.Define(stage, p)
|
||||
sl.CreateRadiusAttr().Set(0.25); sl.CreateIntensityAttr().Set(90000.0)
|
||||
UsdGeom.Xformable(sl.GetPrim()).AddTranslateOp().Set(Gf.Vec3d(*pos))
|
||||
dome = stage.GetPrimAtPath("/Environment/_BrightFill")
|
||||
if dome.IsValid():
|
||||
dome.GetAttribute("inputs:intensity").Set(3500.0)
|
||||
|
||||
lib = {r["name"]: r for r in CL.load_library(f"{REPO}/control_test/items") if "error" not in r}
|
||||
ROOT = "/World/CapItems2"
|
||||
UsdGeom.Xform.Define(stage, ROOT)
|
||||
BB = lambda: UsdGeom.BBoxCache(Usd.TimeCode.Default(),
|
||||
[UsdGeom.Tokens.default_, UsdGeom.Tokens.render])
|
||||
|
||||
async def spawn(name):
|
||||
"""holder Xform carries the placement; the reference sits on a child so its own
|
||||
transform survives."""
|
||||
path = f"{ROOT}/{name}"
|
||||
if stage.GetPrimAtPath(path).IsValid():
|
||||
stage.RemovePrim(path)
|
||||
holder = UsdGeom.Xform.Define(stage, path).GetPrim()
|
||||
inner = UsdGeom.Xform.Define(stage, f"{path}/mesh").GetPrim()
|
||||
inner.GetReferences().AddReference(lib[name]["path"])
|
||||
r = None
|
||||
for _ in range(12): # a reference does not compose within the tick
|
||||
await app_utils.update_app_async(steps=2)
|
||||
r = BB().ComputeWorldBound(inner).ComputeAlignedRange()
|
||||
if not r.IsEmpty():
|
||||
break
|
||||
if r is None or r.IsEmpty():
|
||||
raise RuntimeError(f"{name}: пустой bbox - ссылка не разрешилась")
|
||||
mn, mx = r.GetMin(), r.GetMax()
|
||||
UsdGeom.Xformable(holder).AddTranslateOp(precision=UsdGeom.XformOp.PrecisionDouble).Set(
|
||||
Gf.Vec3d(CC.TARGET[0] - (mn[0] + mx[0]) / 2.0,
|
||||
CC.TARGET[1] - (mn[1] + mx[1]) / 2.0,
|
||||
CC.TARGET[2] - mn[2] + 0.001))
|
||||
# the exported item layers author visibility=invisible on their own root, and
|
||||
# MakeVisible on an ancestor does NOT clear a descendant's authored value - that is
|
||||
# why every earlier capture showed bare belt
|
||||
for d in Usd.PrimRange(holder):
|
||||
if d.IsA(UsdGeom.Imageable):
|
||||
UsdGeom.Imageable(d).GetVisibilityAttr().Set(UsdGeom.Tokens.inherited)
|
||||
UsdGeom.Imageable(holder).MakeVisible()
|
||||
r2 = BB().ComputeWorldBound(inner).ComputeAlignedRange()
|
||||
ext = [(r2.GetMax()[i] - r2.GetMin()[i]) * 1000.0 for i in range(3)]
|
||||
return holder, ext, [r2.GetMin()[i] for i in range(3)], [r2.GetMax()[i] for i in range(3)]
|
||||
|
||||
def world_geom(prim):
|
||||
"""every mesh of the item in world space, as vertices + triangles. Vertices alone are
|
||||
not enough: a box has eight of them, so a point-splat mask covers ~60 px and the ROI
|
||||
collapses. Filling the projected triangles gives the true silhouette."""
|
||||
xc = UsdGeom.XformCache(Usd.TimeCode.Default()); V = []; T = []; base = 0
|
||||
for d in Usd.PrimRange(prim):
|
||||
if not d.IsA(UsdGeom.Mesh):
|
||||
continue
|
||||
m = UsdGeom.Mesh(d)
|
||||
pts = m.GetPointsAttr().Get()
|
||||
if not pts:
|
||||
continue
|
||||
M = np.array(xc.GetLocalToWorldTransform(d), dtype=np.float64)
|
||||
P = np.asarray(pts, dtype=np.float64)
|
||||
V.append((np.c_[P, np.ones(len(P))] @ M)[:, :3])
|
||||
cnt = m.GetFaceVertexCountsAttr().Get() or []
|
||||
idx = m.GetFaceVertexIndicesAttr().Get() or []
|
||||
o = 0
|
||||
for c in cnt: # fan-triangulate each polygon
|
||||
for k in range(1, c - 1):
|
||||
T.append((base + idx[o], base + idx[o + k], base + idx[o + k + 1]))
|
||||
o += c
|
||||
base += len(P)
|
||||
if not V:
|
||||
return np.zeros((0, 3)), np.zeros((0, 3), int)
|
||||
return np.concatenate(V, 0), np.asarray(T, dtype=np.int64).reshape(-1, 3)
|
||||
|
||||
|
||||
def gt_mask(V, T, cam):
|
||||
W, H = cam["width"], cam["height"]
|
||||
Minv = np.linalg.inv(np.array(cam["M"]))
|
||||
c = (np.c_[V, np.ones(len(V))] @ Minv)[:, :3]
|
||||
z = -c[:, 2]
|
||||
u = c[:, 0] / np.maximum(z, 1e-9) * cam["fx"] + cam["cx"]
|
||||
v = -c[:, 1] / np.maximum(z, 1e-9) * cam["fy"] + cam["cy"]
|
||||
uv = np.c_[u, v]
|
||||
m = np.zeros((H, W), np.uint8)
|
||||
if len(T):
|
||||
good = (z[T] > 1e-3).all(1)
|
||||
tri = uv[T[good]].astype(np.int32)
|
||||
tri = np.clip(tri, [-4 * W, -4 * H], [4 * W, 4 * H])
|
||||
cv2.fillPoly(m, list(tri), 1)
|
||||
else:
|
||||
ok = (z > 1e-3) & (u >= 0) & (u < W) & (v >= 0) & (v < H)
|
||||
m[v[ok].astype(int), u[ok].astype(int)] = 1
|
||||
m = cv2.morphologyEx(m, cv2.MORPH_CLOSE, np.ones((5, 5), np.uint8))
|
||||
n, lab, st, _ = cv2.connectedComponentsWithStats(m)
|
||||
if n > 1:
|
||||
m = (lab == (1 + np.argmax(st[1:, cv2.CC_STAT_AREA]))).astype(np.uint8)
|
||||
return m.astype(bool)
|
||||
|
||||
|
||||
w = vp.get_active_viewport(); orig_cam = w.camera_path
|
||||
manifest = {"config": CFG, "target": [float(v) for v in CC.TARGET],
|
||||
"standoff_m": CC.STANDOFF, "calib": calib, "items": {}}
|
||||
|
||||
for name in ITEMS:
|
||||
if name not in lib:
|
||||
print(f" пропуск {name} (нет в библиотеке)"); continue
|
||||
prim, ext, bmin, bmax = await spawn(name)
|
||||
await app_utils.update_app_async(steps=20)
|
||||
VV, TT = world_geom(prim)
|
||||
files, masks, cover, seen = {}, {}, {}, {}
|
||||
for cam_name, cam in calib.items():
|
||||
w.camera_path = f"/RigRS/{cam_name}" if stage.GetPrimAtPath(f"/RigRS/{cam_name}").IsValid() \
|
||||
else CC._cam(stage, cam_name).GetPath()
|
||||
await app_utils.update_app_async(steps=22); await asyncio.sleep(0)
|
||||
f = f"{OUT}/{name}__{cam_name}.png"
|
||||
vp.capture_viewport_to_file(w, file_path=f)
|
||||
await app_utils.update_app_async(steps=12); await asyncio.sleep(0)
|
||||
files[cam_name] = f
|
||||
mk = gt_mask(VV, TT, cam)
|
||||
_img = cv2.imread(f)
|
||||
if _img is not None and _img.shape[:2] == mk.shape:
|
||||
_g = cv2.cvtColor(_img, cv2.COLOR_BGR2GRAY).astype(float)
|
||||
_ring = cv2.dilate(mk.astype(np.uint8), np.ones((41, 41), np.uint8)).astype(bool) & ~mk
|
||||
seen[cam_name] = round(float(abs(_g[mk].mean() - _g[_ring].mean())), 1)
|
||||
np.savez_compressed(f"{OUT}/{name}__{cam_name}_mask.npz", m=mk)
|
||||
masks[cam_name] = f"{OUT}/{name}__{cam_name}_mask.npz"
|
||||
cover[cam_name] = int(mk.sum())
|
||||
manifest["items"][name] = dict(
|
||||
files=files, masks=masks, mask_px=cover, nverts=int(len(VV)), ntris=int(len(TT)),
|
||||
contrast=seen, gt_catalogue=lib[name]["dims_mm"], cls=lib[name]["cls"],
|
||||
gt_scene_mm=[round(e, 1) for e in ext], bmin=bmin, bmax=bmax)
|
||||
print(f" {name}: в сцене {[round(e) for e in sorted(ext, reverse=True)]} мм "
|
||||
f"маска {min(cover.values())}-{max(cover.values())} px, контраст {min(seen.values()):.0f}-{max(seen.values()):.0f}"
|
||||
+ (" <-- НЕ ВИДЕН" if min(seen.values()) < 3 else ""))
|
||||
# Спрятать МАЛО: невидимость не убирает коллайдер, и снятый товар остаётся твёрдой
|
||||
# стеной ровно в точке осмотра, посреди рабочей линии. После двух прогонов там стояло
|
||||
# 18 невидимых предметов, и поток вставал на них, не доезжая до плуга.
|
||||
UsdGeom.Imageable(prim).MakeInvisible()
|
||||
for _d in Usd.PrimRange(prim):
|
||||
_a = _d.GetAttribute("physics:collisionEnabled")
|
||||
if _a and _a.IsValid():
|
||||
_a.Set(False)
|
||||
elif _d.HasAPI(UsdPhysics.CollisionAPI):
|
||||
UsdPhysics.CollisionAPI(_d).CreateCollisionEnabledAttr().Set(False)
|
||||
|
||||
w.camera_path = orig_cam
|
||||
await app_utils.update_app_async(steps=10)
|
||||
json.dump(manifest, open(f"{OUT}/manifest.json", "w"), indent=1)
|
||||
print(f"\nmanifest -> {OUT}/manifest.json")
|
||||
Reference in New Issue
Block a user