Celebrating Modern: Voxel Gauntlet

In celebration of the updates to Modern in the latest Beta, I had Claude do this AR project:

It creates a voxel gauntlet around your hand, and it has a couple other nifty tricks to boot. :blush:

All in one tab too:


-- Voxel Glove v0.9 (Modern): live rear camera, Apple Vision hand pose,
-- the hand drawn as a 3D glove of lit cubes that turns with the hand.
-- Depth from foreshortening: palm width (index to little knuckle) sets the
-- hand's scale; each part's true length = its proportion x that width, and
-- how much shorter it looks on screen gives how far it leans in or out.
-- Which hand is decided once, over the first frames after the hand appears
-- (arm entering from the right = right hand, else Vision's guess), and kept
-- until the hand leaves view. Palm facing then follows the thumb's side.
-- Short dropouts hold the glove still; gone for GONE_AFTER s clears it.
-- Joints are steadied with a One Euro filter (heavy smoothing when still,
-- light when moving), and small tilts near flat are ignored.
-- The sword is a latched state: a fist with the thumb wrapped (hilt grip)
-- draws it, opening the hand puts it away, and losing sight of the fingers
-- does not drop it. Its grip sits between the knuckles and the curled
-- fingers' middle joints, inside the fist.
-- Flick: a fingertip that starts against the thumb tip and snaps away from
-- it within FLICK_TIME throws a burst of sparkles off that fingertip.
-- Opens on a title card that fades by itself, or at once on a tap.
-- Parameters (sidebar, hide it with the viewer's button): Cubes (cubes
-- across the palm), Status (prints frame rate, hand, tool etc. to the
-- console), Calibrate, Cycle orientation. A tap on the screen also
-- calibrates: hold your hand flat, fingers straight.
-- Camera/Vision code is unchanged from v0.1, plus chirality.
SKELETON={{"VNHLKWRI","VNHLKTCMC","VNHLKTMP","VNHLKTIP","VNHLKTTIP"},
  {"VNHLKWRI","VNHLKIMCP","VNHLKIPIP","VNHLKIDIP","VNHLKITIP"},
  {"VNHLKWRI","VNHLKMMCP","VNHLKMPIP","VNHLKMDIP","VNHLKMTIP"},
  {"VNHLKWRI","VNHLKRMCP","VNHLKRPIP","VNHLKRDIP","VNHLKRTIP"},
  {"VNHLKWRI","VNHLKPMCP","VNHLKPPIP","VNHLKPDIP","VNHLKPTIP"}}
CHAINS={thumb={"VNHLKTCMC","VNHLKTMP","VNHLKTIP","VNHLKTTIP"},
  index={"VNHLKIMCP","VNHLKIPIP","VNHLKIDIP","VNHLKITIP"},
  middle={"VNHLKMMCP","VNHLKMPIP","VNHLKMDIP","VNHLKMTIP"},
  ring={"VNHLKRMCP","VNHLKRPIP","VNHLKRDIP","VNHLKRTIP"},
  little={"VNHLKPMCP","VNHLKPPIP","VNHLKPDIP","VNHLKPTIP"}}
ORDER={"index","middle","ring","little","thumb"}
-- segment length / palm width; typical adult proportions until you calibrate
REST={palm=1.05,thumb={0.47,0.35,0.29},index={0.47,0.28,0.22},
  middle={0.53,0.33,0.24},ring={0.48,0.32,0.23},little={0.35,0.22,0.20}}
-- Vision orientation code, name, picture rotation in degrees
ORIENTS={{1,"up",0},{6,"right",-90},{3,"down",180},{8,"left",90}}
PREVIEW_EVERY=1    -- camera picture refresh, in draw frames (0 = off)
PREVIEW_SCALE=0.03 -- picture size before conversion; smaller = bigger blocks
SHOW_SKELETON=false
INTRO_HOLD,INTRO_FADE=4,2 -- title card: seconds shown, then seconds fading out
ACROSS=6           -- cubes across the palm; sets the cube size (Cubes parameter)
ACROSS_MIN,ACROSS_MAX=4,16
HAND="auto"        -- "auto", "right" or "left"
LOCK_FRAMES=8      -- frames of evidence before the hand is locked as left or right
GONE_AFTER=0.5     -- seconds without a hand before the glove is cleared
-- sizes in palm widths, kept the same whatever the cube size: Cubes only
-- changes how many cubes fill each part (fingers: 1 cube across at 4, 5 at 16)
PALM_W=1.15        -- palm plate width (1 = knuckle to knuckle)
PALM_T=0.5         -- palm thickness
FINGER_W=0.32      -- finger width and thickness
THUMB_W=0.45       -- thumb base segment width and thickness
TIP_EXT=0.15       -- how far each fingertip reaches past Vision's tip point
CUFF_LEN=0.5       -- cuff length up the arm
-- block textures from Codea's built-in Blocks pack, first name found wins;
-- if none is found the colours below are used and the pack's names printed
USE_TEXTURES=true
TEX_BACK={"Brick_Red","Brick","Red_Brick"}
TEX_PALM={"Stone","Cobblestone","Rock"}
TEX_CUFF={"Sand","Sandstone"}
BACK=color(120,220,255) PALM=color(55,105,150) CUFF=color(240,240,250)
-- steadying: One Euro filter. Lower MIN_CUTOFF = steadier when still but
-- floatier; higher BETA = less lag on fast moves
MIN_CUTOFF=1.0     -- Hz
BETA=0.01          -- per screen point per second
D_CUTOFF=1.0       -- Hz
LEAN_DEAD=12       -- degrees of tilt ignored near flat
-- tools
GRIP_ON=0.15       -- s a hilt grip must hold before the sword appears
RELEASE_T=0.25     -- s the open hand must hold before the sword is put away
CURL_K=1.0         -- finger curled: tip nearer the wrist than this x its knuckle
OPEN_K=1.3         -- finger open: tip further from the wrist than this x its knuckle
THUMB_WRAP=0.6     -- grip: thumb tip within this many palm widths of the index middle joint
FLICK_IN=0.35      -- flick: fingertip starts within this many palm widths of the thumb tip
FLICK_OUT=0.75     -- flick: and ends further than this
FLICK_TIME=0.2     -- flick: within this many seconds
FLICK_COOL=0.3     -- s before the same finger can flick again
FLICK_SPARKS=16    -- sparkles per flick
FLICK_SPEED=7      -- sparkle launch speed, palm widths per second
SWORD_PX=0.14      -- sword model pixel size in palm widths
CUBE_GAP=0.9       -- cube size inside its cell (1 = solid)
Z_SIGN=1           -- +1: larger z is further from the camera; flip if lit wrong
PALM_TILT=1        -- +1: knuckles lean away when the palm looks short; -1 toward
ROLL_TILT=-1       -- same for the index side when the palm looks narrow
-- sword model, an original pixel-art longsword, pommel at the bottom.
-- Rows are drawn along the blade, columns across its width.
SWORD={
  "....D....",
  "...DgD...",
  "..DgRgD..",
  "...DGD...",
  "....D....",
  "...lLl...",
  "...LlL...",
  "...lLl...",
  "...LlL...",
  "...lLl...",
  "...LlL...",
  ".DD...DD.",
  "DgGGRGGgD",
  ".DGGGGGD.",
  "...SsS...",
  "...WsS...",
  "...WsS...",
  "...WsS...",
  "...WsS...",
  "...WsS...",
  "...WsS...",
  "...WsS...",
  "...WsS...",
  "...WsS...",
  "...WsS...",
  "...WsS...",
  "...WsS...",
  "...WsS...",
  "...WsS...",
  "...WsS...",
  "...WsS...",
  "...WsS...",
  "...WsS...",
  "...WsS...",
  "...WsS...",
  "...WsS...",
  "...WsS...",
  "...WsS...",
  "...WsS...",
  "...WSS...",
  "....WS...",
  "....W....",
}
SWORD_GRIP={6,11}  -- rows held in the fist (1 = bottom row)
SWORD_COLS={D=color(150,100,30),G=color(225,175,55),g=color(255,230,140),
  R=color(200,30,50),L=color(95,55,30),l=color(140,90,50),
S=color(185,195,210),s=color(130,140,160),W=color(245,250,255)}
SWORD_THICK={S=0.6,s=0.6,W=0.6} -- depth in pixels; anything else 1.4
SPARK_COLS={color(255,255,255),color(255,215,90),color(120,230,255),color(255,140,230)}
function setup()
  viewer.mode=FULLSCREEN
  joints,status,orient,chir={},"starting camera",1,0
  frames,runs,visionMs,rate,t0=0,0,0,0,now()
  camImg,previewOk,tick,camW,camH=nil,true,0,1920,1440
  calibrated,sparks=false,{}
  clearHand()
  backBox=boxOf(BACK,TEX_BACK)
  palmBox=boxOf(PALM,TEX_PALM)
  cuffBox=boxOf(CUFF,TEX_CUFF)
  swordBox={}
  for k,c in pairs(SWORD_COLS) do swordBox[k]=boxOf(c) end
  sparkBox={}
  for i,c in ipairs(SPARK_COLS) do sparkBox[i]=glowOf(c) end
  sparkQ=quat.angleAxis(45,vec3(1,1,0))
  sun=light.directional(vec3(0.35,-0.6,1)) sun.intensity=2.5
  cuffCells=ringCells()
  lastT=now()
  introT,introGone=now(),false
  parameter.integer("Cubes",ACROSS_MIN,ACROSS_MAX,ACROSS,function(n)
    if n~=ACROSS then ACROSS=n cuffCells=ringCells() end
  end)
  parameter.action("Status",printStatus)
  parameter.action("Calibrate",calibrate)
  parameter.action("Cycle orientation",function()
    orient=orient%#ORIENTS+1 joints={} clearHand()
    print("orientation "..ORIENTS[orient][2])
  end)
  local ok,err=pcall(startCamera)
  if not ok then status="camera error: "..tostring(err) end
  print(status)
end
-- everything the old on-screen status line showed, to the console
function printStatus()
  print(string.format("%s | vision %.0f ms | %.1f runs/s | orient %s | %s %s | %s | %d cubes | %s",
  status,visionMs,rate,ORIENTS[orient][2],
  rig and rig.hand or "no hand",rig and (rig.toward and "palm" or "back") or "",
  tool or "no tool",cubeCount or 0,calibrated and "calibrated" or "not calibrated"))
end
-- forget the hand: glove off screen, hand side unlocked, tools put away
function clearHand()
  rig,held,flt,sFlt=nil,{},{},{}
  lockedHand,votes,nvotes,toward,flipN=nil,{},0,true,0
  tool,timers,barN,lastSeen,flicks=nil,{},{},nil,{}
end
function draw()
  background(20,22,30)
  tick=tick+1
  autoOrient()
  local f=arSession and arSession.currentFrame
  if f then
    local ts=f.timestamp
    if ts~=lastTs then
      lastTs,frames=ts,frames+1
      local ok,err=pcall(detect,f)
      if not ok then status="vision error: "..tostring(err) print(status) end
      if previewOk and PREVIEW_EVERY>0 and tick%PREVIEW_EVERY==0 then grabPreview(f) end
    end
    f=nil
    collectgarbage() -- hand the camera buffer back to ARKit right away
  end
  local tn=now()
  DT=math.max(1/120,math.min(0.1,tn-lastT)) lastT=tn
  if tn-t0>=1 then rate,runs,t0=runs/(tn-t0),0,tn end
  local o=ORIENTS[orient]
  local x0,y0,dw,dh=viewRect()
  style.push()
  if camImg then
    local sw,sh=dw,dh
    if o[3]==90 or o[3]==-90 then sw,sh=dh,dw end
    matrix.push()
    matrix.translate(WIDTH/2,HEIGHT/2)
    matrix.rotate(o[3],0,0,1)
    style.shapeMode(CENTER)
    sprite(camImg,0,0,sw,sh)
    matrix.pop()
  end
  local ok,err=pcall(function()
    updateRig(x0,y0,dw,dh)
    if rig and lastSeen and tn-lastSeen>GONE_AFTER then clearHand() end
    drawGlove()
  end)
  if not ok then status="glove error: "..tostring(err) print(status) end
  if SHOW_SKELETON then
    style.stroke(255,220,0) style.strokeWidth(5)
    for _,b in ipairs(SKELETON) do
      for i=2,#b do
        local p,q=joints[b[i-1]],joints[b[i]]
        if p and q then line(x0+p.x*dw,y0+p.y*dh,x0+q.x*dw,y0+q.y*dh) end
      end
    end
  end
  style.pop()
  drawIntro()
end
-- title card: fades by itself after INTRO_HOLD, or is gone at once on a tap.
-- Each line wraps at 85% of the screen width (style.textWrapWidth), and the
-- lines are stacked by their measured, wrapped heights (textSize).
function drawIntro()
  if introGone then return end
  local t=now()-introT
  local a=1
  if t>INTRO_HOLD then a=1-(t-INTRO_HOLD)/INTRO_FADE end
  if a<=0 then introGone=true return end
  local u=math.min(WIDTH,HEIGHT)
  local big,small=u*0.13,math.max(16,u*0.035)
  local lines={{"Voxel",big,BACK,0},{"Gauntlet",big,BACK,small*1.2},
    {"Close your hand as if holding a sword, and a sword appears",small,color(255),small*0.6},
    {"Flick your fingers to make magic dust",small,color(255),0}}
  style.push() style.noStroke() style.shapeMode(CENTER)
  style.fill(0,0,0,150*a) rect(WIDTH/2,HEIGHT/2,WIDTH,HEIGHT)
  style.textAlign(CENTER) style.textWrapWidth(WIDTH*0.85) style.textMode(CENTER)
  local total=0
  for _,l in ipairs(lines) do
    style.fontSize(l[2])
    local _,h=textSize(l[1])
    l.h=h total=total+h+l[4]
  end
  local y=HEIGHT/2+total/2
  for _,l in ipairs(lines) do
    style.fontSize(l[2])
    local cy=y-l.h/2
    local o=l[2]*0.06
    style.fill(0,0,0,220*a) text(l[1],WIDTH/2+o,cy-o)
    style.fill(l[3].r,l[3].g,l[3].b,255*a) text(l[1],WIDTH/2,cy)
    y=y-l.h-l[4]
  end
  style.pop()
end
function touched(t)
  if t.state~=ENDED then return end
  if not introGone then introGone=true return end
  calibrate()
end
-- a unit cube with a lit material: a block texture if one of the names is
-- in the Blocks pack, else the plain colour
function boxOf(col,names)
  local b=mesh.box(vec3(0.5,0.5,0.5))
  b.material=material.lit()
  b.material.color=col
  if USE_TEXTURES and names then
    for _,n in ipairs(names) do
      local ok,img=pcall(function() return image.read(asset.builtin.Blocks[n]) end)
      if ok and img then
        pcall(function() img.filter=image.filter.point end)
        b.material.map=img b.material.color=color(255)
        return b
      end
    end
    if not listedBlocks then
      listedBlocks=true
      local t={}
      pcall(function() for name in pairs(asset.builtin.Blocks) do t[#t+1]=tostring(name) end end)
      table.sort(t)
      print("no block texture found for "..names[1]..". Blocks pack has: "..table.concat(t,", "))
    end
  end
  return b
end
-- an unlit cube that glows in its own colour (sparkles)
function glowOf(col)
  local b=mesh.box(vec3(0.5,0.5,0.5))
  b.material=material.unlit()
  b.material.color=col
  return b
end
-- screen position of a joint, as a vec2
function sp(n)
  local j=joints[n]
  if j then return vec2(VX0+j.x*VDW,VY0+j.y*VDH) end
end
-- how far a part of true length L leans in or out when it looks l long.
-- The first LEAN_DEAD degrees of tilt are ignored: near flat, a tiny change
-- in apparent length means a big change in angle, which is where jitter
-- came from.
function lean(L,l)
  if L<=0 then return 0 end
  local a=math.acos(math.max(-1,math.min(1,l/L)))-math.rad(LEAN_DEAD)
  if a<=0 then return 0 end
  return L*math.sin(a)
end
-- record this hand's proportions; assumes it is flat and fingers straight
function calibrate()
  local w,im,mm,pm=sp("VNHLKWRI"),sp("VNHLKIMCP"),sp("VNHLKMMCP"),sp("VNHLKPMCP")
  if not (w and im and mm and pm) then print("calibrate: no full hand in view") return end
  local W=im:distance(pm)
  if W<10 then return end
  REST.palm=w:distance(mm)/W
  for name,ch in pairs(CHAINS) do
    for i=2,#ch do
      local a,b=sp(ch[i-1]),sp(ch[i])
      if a and b then REST[name][i-1]=a:distance(b)/W end
    end
  end
  calibrated=true
  local s=string.format("calibrated palm %.2f",REST.palm)
  for _,n in ipairs(ORDER) do s=s..string.format(" %s %.2f %.2f %.2f",n,REST[n][1],REST[n][2],REST[n][3]) end
  print(s)
end
-- one vote for which hand this is: from the side the arm enters, else
-- Vision's guess; nil when neither says anything
function handVote(w,km)
  local a=w-km -- points back up the arm
  if math.abs(a.x)>math.abs(a.y)*0.6 then return (a.x>0) and "right" or "left" end
  if chir==1 then return "right" elseif chir==-1 then return "left" end
end
-- which hand: voted on over the first LOCK_FRAMES frames, then fixed until
-- the hand leaves view
function whichHand(w,km)
  if HAND~="auto" then return HAND end
  if lockedHand then return lockedHand end
  local v=handVote(w,km)
  if v then votes[v]=(votes[v] or 0)+1 nvotes=nvotes+1 end
  local h=((votes.left or 0)>(votes.right or 0)) and "left" or "right"
  if nvotes>=LOCK_FRAMES then lockedHand=h print("hand locked: "..h) end
  return h
end
-- palm toward the camera? For a right hand seen from behind (the camera is
-- roughly where your eyes are), fingers up with the thumb to the right means
-- palm toward. A change has to show for 3 frames in a row before it's taken.
function palmToward(w,km,thumb,S,hand)
  local f=km-w
  local t=thumb-w
  local c=f.x*t.y-f.y*t.x -- negative: thumb clockwise of the fingers
  if math.abs(c)>S*S*0.05 then
    local want=(hand=="right")==(c<0)
    if want~=toward then
      flipN=flipN+1
      if flipN>=3 then toward,flipN=want,0 end
    else flipN=0 end
  end
  return toward
end
-- One Euro filter, for a vec3 (st=state table) or a number (num=true)
local function alpha(cut,dt) local r=2*math.pi*cut*dt return r/(r+1) end
function euro(st,x,num)
  if st.x==nil then st.x,st.d=x,num and 0 or vec3(0,0,0) return x end
  local dx=(x-st.x)*(1/DT)
  st.d=st.d+(dx-st.d)*alpha(D_CUTOFF,DT)
  local speed=num and math.abs(st.d) or st.d.length
  st.x=st.x+(x-st.x)*alpha(MIN_CUTOFF+BETA*speed,DT)
  return st.x
end
-- seconds a condition has held, as a test: true once it has held for secs
function hold(key,cond,secs)
  if not cond then timers[key]=nil return false end
  timers[key]=timers[key] or now()
  return now()-timers[key]>=secs
end
-- finger tests from the raw 2D joints
function fingerCount(w,test)
  local n=0
  for _,name in ipairs({"index","middle","ring","little"}) do
    local ch=CHAINS[name]
    local mcp,tip=sp(ch[1]),sp(ch[4])
    if mcp and tip and test(tip:distance(w),mcp:distance(w)) then n=n+1 end
  end
  return n
end
function isGrip(w,S)
  local tt,ip=sp("VNHLKTTIP"),sp("VNHLKIPIP") or sp("VNHLKIMCP")
  return fingerCount(w,function(t,m) return t<m*CURL_K end)>=3
  and tt and ip and tt:distance(ip)<THUMB_WRAP*S
end
function isOpen(w) return fingerCount(w,function(t,m) return t>m*OPEN_K end)>=3 end
-- the sword is latched: only a clearly open hand puts it away, so it stays
-- in the hand when the fingers tilt out of sight
function updateTools(w,S)
  local g=hold("grip",isGrip(w,S),GRIP_ON)
  local o=hold("open",isOpen(w),RELEASE_T)
  if tool=="sword" then
    if o then tool=nil end
  elseif g then tool="sword" end
end
-- flick: per finger, remember when its tip was last against the thumb tip;
-- if it is now well away from it and that was under FLICK_TIME ago, it was
-- flicked. Checked once per new Vision frame, from the raw joints (the
-- smoothed ones lag behind a flick). Off while the sword is out.
function checkFlicks(S)
  if frames==flickFrame then return end
  flickFrame=frames
  local tt=sp("VNHLKTTIP")
  if not tt or tool=="sword" then return end
  local t=now()
  for _,name in ipairs({"index","middle","ring","little"}) do
    local ch=CHAINS[name]
    local tip,dip=sp(ch[4]),sp(ch[3])
    local f=flicks[name] or {}
    flicks[name]=f
    if tip then
      local d=tip:distance(tt)/S
      if d<FLICK_IN then f.closed=t
      elseif d>FLICK_OUT and f.closed and t-f.closed<FLICK_TIME and t>(f.cool or 0) then
        f.closed,f.cool=nil,t+FLICK_COOL
        local z=rig.P[ch[4]] and rig.P[ch[4]].z or 0
        local dir=dip and unit(vec3(tip.x-dip.x,tip.y-dip.y,0)) or unit(vec3(tip.x-tt.x,tip.y-tt.y,0))
        burst(vec3(tip.x,tip.y,z),dir,S)
      end
    end
  end
end
-- a spray of sparkles from p, mostly along dir
function burst(p,dir,S)
  for i=1,FLICK_SPARKS do
    local j=vec3(math.random()-0.5,math.random()-0.5,math.random()-0.5)*0.9
    local v=unit(dir+j)*(S*FLICK_SPEED*(0.5+math.random()*0.6))
    sparks[#sparks+1]={p=p,v=v,life=0,max=0.4+math.random()*0.5,drag=0.95,
      m=sparkBox[math.random(#sparkBox)],size=S*(0.05+math.random()*0.06)}
  end
end
-- 3D joint positions (screen points, z = lean) plus the hand's frame.
-- Missing joints hold their last position; a missing palm holds the rig.
function updateRig(x0,y0,dw,dh)
  VX0,VY0,VDW,VDH=x0,y0,dw,dh
  local w,im,mm,pm,rm=sp("VNHLKWRI"),sp("VNHLKIMCP"),sp("VNHLKMMCP"),sp("VNHLKPMCP"),sp("VNHLKRMCP")
  if not (w and im and mm and pm) then return end
  lastSeen=now()
  local W,lp=im:distance(pm),w:distance(mm)
  -- both palm measures can only shrink, so the larger one is nearer the truth
  local S=euro(sFlt,math.max(W,lp/REST.palm),true)
  local km=(im+pm)*0.5
  local hand=whichHand(w,km)
  -- thumb side: thumb MP if seen, else the index knuckle (it sits thumb-side)
  local tside=sp("VNHLKTMP") or sp("VNHLKTCMC") or im
  local tw=palmToward(w,km,tside,S,hand)
  local fs=(tw and -1 or 1)*Z_SIGN
  local zp=lean(REST.palm*S,lp)*PALM_TILT*Z_SIGN
  local zr=lean(S,W)*ROLL_TILT*Z_SIGN
  local P={}
  local function put(n,p2,z) if p2 then P[n]=vec3(p2.x,p2.y,z) end end
  put("VNHLKWRI",w,0)
  put("VNHLKMMCP",mm,zp)
  put("VNHLKIMCP",im,zp+zr*0.5)
  put("VNHLKPMCP",pm,zp-zr*0.5)
  put("VNHLKRMCP",rm,zp-zr*0.17)
  put("VNHLKTCMC",sp("VNHLKTCMC"),zp*0.25+zr*0.4)
  for name,ch in pairs(CHAINS) do
    for i=2,#ch do
      local a,b=P[ch[i-1]] or held[ch[i-1]],sp(ch[i])
      if a and b then
        local l=vec2(a.x,a.y):distance(b)
        put(ch[i],b,a.z+fs*lean(REST[name][i-1]*S,l))
      end
    end
  end
  for n,p in pairs(P) do
    flt[n]=flt[n] or {}
    P[n]=euro(flt[n],p) held[n]=P[n]
  end
  for n,p in pairs(held) do if not P[n] then P[n]=p end end -- unseen: hold
  local nrm=(P.VNHLKIMCP-P.VNHLKWRI):cross(P.VNHLKPMCP-P.VNHLKWRI)
  if nrm.length<1e-6 then nrm=vec3(0,0,1) end
  nrm=unit(nrm)
  -- palm side of every bar: -normal for a right hand, +normal for a left
  local low=(hand=="right")
  rig={P=P,S=S,v=S/ACROSS,toward=tw,hand=hand,n=nrm,palmLow=low,ps=low and nrm*-1 or nrm}
  updateTools(w,S)
  checkFlicks(S)
end
-- one cube: centre c, rotation q, size sx,sy,sz
function cube(m,c,q,sx,sy,sz)
  cubeCount=cubeCount+1
  matrix.push()
  matrix.translate(c.x,c.y,c.z)
  matrix.rotate(q)
  matrix.scale(sx*CUBE_GAP,sy*CUBE_GAP,sz*CUBE_GAP)
  m:draw()
  matrix.pop()
end
-- unit vector and dot product, written out so they don't depend on whether
-- this Codea build exposes normalized/dot as methods or properties
function unit(v)
  local l=math.sqrt(v.x*v.x+v.y*v.y+v.z*v.z)
  if l<1e-9 then return vec3(0,0,1) end
  return vec3(v.x/l,v.y/l,v.z/l)
end
function dot3(a,b) return a.x*b.x+a.y*b.y+a.z*b.z end
-- orthonormal frame: Z along dir, Y as close to up as possible
function frame(dir,up)
  local Z=unit(dir)
  local Y=up-Z*dot3(up,Z)
  if Y.length<1e-4 then Y=vec3(0,0,1)-Z*Z.z end
  if Y.length<1e-4 then Y=vec3(0,1,0)-Z*Z.y end
  Y=unit(Y)
  return Y:cross(Z),Y,Z
end
-- a bar of cubes from a to b, wf by tf palm widths in cross-section. The
-- cube count across is that size x ACROSS (at least 1), and the cubes are
-- sized to fill it exactly, so the bar keeps its size at any Cubes setting.
-- Along the bar, cube length stretches to span joint to joint; the count
-- along it only changes once the length has moved well past the boundary,
-- so cubes don't pop in and out as the hand breathes. key names the bar.
-- mp: material for the palm-side layer (nil = same as m). up: the bar's
-- thickness direction (default the palm normal).
function bar(key,a,b,wf,tf,m,mp,up)
  local d=b-a
  local len=d.length
  if len<1 then return end
  local X,Y,Z=frame(d,up or rig.n)
  local q=quat.lookRotation(Z,Y)
  local S,v=rig.S,rig.v
  local w=math.max(1,math.floor(wf*ACROSS+0.5))
  local t=math.max(1,math.floor(tf*ACROSS+0.5))
  local cw,ct=wf*S/w,tf*S/t
  local r=len/v
  local n=barN[key]
  if not n or math.abs(r-n)>0.8 then n=math.max(1,math.floor(r+0.5)) barN[key]=n end
  local step=len/n
  local pj=rig.palmLow and 0 or t-1
  -- hollow: only the outer shell, plus solid end slices so no bar shows
  -- its inside; cube count grows with the square of ACROSS, not the cube
  for k=0,n-1 do
    local base,cap=a+Z*((k+0.5)*step),(k==0 or k==n-1)
    for i=0,w-1 do
      for j=0,t-1 do
        if cap or i==0 or i==w-1 or j==0 or j==t-1 then
          local mm=(mp and t>1 and j==pj) and mp or m
          cube(mm,base+X*((i-(w-1)/2)*cw)+Y*((j-(t-1)/2)*ct),q,cw,ct,step)
        end
      end
    end
  end
end
-- cuff cross-section, in cubes: an oval ring one cube thick around the arm,
-- a little wider and thicker than the palm
function ringCells()
  local a,b=ACROSS*PALM_W/2+0.6,ACROSS*PALM_T/2+1.2
  local cells={}
  for i=-math.ceil(a),math.ceil(a)-1 do
    for j=-math.ceil(b),math.ceil(b)-1 do
      local x,y=i+0.5,j+0.5
      local o=(x/a)^2+(y/b)^2
      local inn=(x/(a-1))^2+(y/(b-1))^2
      if o<=1 and inn>1 then cells[#cells+1]={x,y} end
    end
  end
  return cells
end
function drawGlove()
  cubeCount=0
  matrix.push()
  matrix.ortho(0,WIDTH,0,HEIGHT,-5000,5000)
  light.push(sun)
  if rig then
    local P,v=rig.P,rig.v
    local w,km=P.VNHLKWRI,(P.VNHLKIMCP+P.VNHLKPMCP)*0.5
    bar("palm",w,km,PALM_W,PALM_T,backBox,palmBox)
    for _,name in ipairs(ORDER) do
      local ch=CHAINS[name]
      for i=2,#ch do
        local a,b=P[ch[i-1]],P[ch[i]]
        if a and b then
          local s=(name=="thumb" and i==2) and THUMB_W or FINGER_W
          -- Vision's tip point sits short of the real fingertip, so the last
          -- segment carries on past it
          if i==#ch then b=b+unit(b-a)*(TIP_EXT*rig.S) end
          bar(name..i,a,b,s,s,backBox,palmBox)
        end
      end
    end
    -- cuff: rings stepping back up the arm; skip cubes behind the arm's axis
    local X,Y,Z=frame(w-km,rig.n)
    local q=quat.lookRotation(Z,Y)
    for k=0,math.max(1,math.floor(CUFF_LEN*ACROSS+0.5))-1 do
      local axis=w+Z*((k+0.5)*v)
      for _,c in ipairs(cuffCells) do
        local p=axis+X*(c[1]*v)+Y*(c[2]*v)
        if (p.z-axis.z)*Z_SIGN<=0 then cube(cuffBox,p,q,v,v,v) end
      end
    end
    if tool=="sword" then drawSword(P,w,km) end
  end
  drawSparks()
  light.pop()
  matrix.pop()
end
-- sword in the fist. The grip's centre is halfway between the knuckles and
-- the curled fingers' middle joints, which puts it inside the fist (this is
-- where v0.5 put it). Those joints hold their last position when unseen, so
-- the sword stays put when the fingers tilt away; with none known yet it
-- falls back to just inside the knuckles. The grip runs along the knuckle
-- line, blade out of the thumb side; the edge faces the way the knuckles point.
function drawSword(P,w,km)
  local S=rig.S
  local A=unit(P.VNHLKIMCP-P.VNHLKPMCP)
  local F=unit(km-w)
  local pips,np=vec3(0,0,0),0
  for _,n in ipairs({"VNHLKIPIP","VNHLKMPIP","VNHLKRPIP","VNHLKPPIP"}) do
    if P[n] then pips=pips+P[n] np=np+1 end
  end
  local c=(np>0) and (km+pips*(1/np))*0.5 or km+rig.ps*(0.3*S)
  local X,Y,Z=frame(A:cross(F),A) -- Y: along the blade, Z: through its flat
  local q=quat.lookRotation(Z,Y)
  local px=SWORD_PX*S
  local rows,cols=#SWORD,#SWORD[1]
  local mid=(SWORD_GRIP[1]+SWORD_GRIP[2])/2
  for r=1,rows do
    local row=SWORD[r]
    for k=1,cols do
      local ch=row:sub(k,k)
      local m=swordBox[ch]
      if m then
        local p=c+X*((k-(cols+1)/2)*px)+Y*((r-mid)*px)
        cube(m,p,q,px,px,px*(SWORD_THICK[ch] or 1.4))
      end
    end
  end
end
-- sparkles fly on from where they were thrown, slow down, fall and shrink away
function drawSparks()
  local keep={}
  for _,s in ipairs(sparks) do
    s.life=s.life+DT
    if s.life<s.max then
      s.p=s.p+s.v*DT
      s.v=s.v*(s.drag or 0.92)+vec3(0,-(rig and rig.S or 100)*0.6*DT,0)
      local k=1-s.life/s.max
      cube(s.m,s.p,sparkQ,s.size*k,s.size*k,s.size*k)
      keep[#keep+1]=s
    end
  end
  sparks=keep
end
-- default orientation from the device's screen shape (not the viewer's,
-- which changes when the sidebar opens), rechecked when it changes
function autoOrient()
  local w,h=WIDTH,HEIGHT
  pcall(function()
    local b=objc.UIScreen.mainScreen.bounds
    if b and b.size then w,h=b.size.width,b.size.height end
  end)
  if w==lastW and h==lastH then return end
  lastW,lastH=w,h
  orient=(h>w) and 2 or 1
  joints={}
end
-- screen rectangle the oriented camera image fills (aspect-fill, cropped)
function viewRect()
  local w,h=camW,camH
  local r=ORIENTS[orient][3]
  if r==90 or r==-90 then w,h=h,w end
  local s=math.max(WIDTH/w,HEIGHT/h)
  local dw,dh=w*s,h*s
  return (WIDTH-dw)/2,(HEIGHT-dh)/2,dw,dh
end
-- wall-clock seconds (os.clock is CPU time, wrong for Vision)
function now()
  local ok,t=pcall(function() return objc.NSProcessInfo.processInfo.systemUptime end)
  if ok and type(t)=="number" then return t end
  return os.clock()
end
-- ARKit session on the rear camera; 3-DOF tracking is the cheapest config.
-- The delegate only reports errors; it stores a message and does nothing else.
function startCamera()
  local cfg
  local ok=pcall(function() cfg=objc.AROrientationTrackingConfiguration() end)
  if not ok or not cfg then cfg=objc.ARWorldTrackingConfiguration() end
  local D=objc.delegate("ARSessionDelegate")
  function D:session_didFailWithError_(objSession,objError)
    camFailed=tostring(objError)
  end
  arDelegate=D()
  arSession=objc.ARSession()
  arSession.delegate=arDelegate
  arSession:runWithConfiguration_(cfg)
  status="camera running"
end
function detect(frame)
  if camFailed then status="ARKit failed: "..camFailed return end
  local pb=frame.capturedImage
  if not pb then status="no pixel buffer from frame" return end
  local c=now()
  local req=objc.VNDetectHumanHandPoseRequest()
  req.maximumHandCount=1
  -- string-keyed table so the bridge sends a dictionary, not an array
  local hnd=objc.VNImageRequestHandler:alloc():initWithCVPixelBuffer_orientation_options_(pb,ORIENTS[orient][1],{codeaDummy=1})
  if not hnd then status="request handler not created" return end
  hnd:performRequests_error_({req},nil)
  visionMs=(now()-c)*1000
  runs=runs+1
  local res=req.results
  if type(res)~="table" or #res==0 then joints={} status="no hand" return end
  local obs,js=res[1],{}
  chir=tonumber(obs.chirality) or 0
  local names=obs.availableJointNames or {}
  for i=1,#names do
    local p=obs:recognizedPointForJointName_error_(names[i],nil)
    if p and (p.confidence or 0)>0.2 then
      local x,y=p.x,p.y
      if not x and p.location then x,y=p.location.x,p.location.y end
      if x then js[tostring(names[i])]={x=x,y=y,c=p.confidence} end
    end
  end
  joints=js
  status="hand"
end
-- camera picture, shrunk with Core Image's scale filter (plain numbers, no
-- struct), cropped to whole pixels, drawn as hard-edged blocks
function grabPreview(frame)
  local ok,img=pcall(function()
    local ci=objc.CIImage:imageWithCVPixelBuffer_(frame.capturedImage)
    local ok2,e=pcall(function() return ci.extent end)
    if ok2 and e and e.size then camW,camH=e.size.width,e.size.height end
    local k=PREVIEW_SCALE
    if k and k<1 then
      local f=objc.CIFilter:filterWithName_("CILanczosScaleTransform")
      f:setValue_forKey_(ci,"inputImage")
      f:setValue_forKey_(k,"inputScale")
      f:setValue_forKey_(1,"inputAspectRatio")
      ci=f.outputImage:imageByCroppingToRect_(objc.rect(0,0,math.floor(camW*k),math.floor(camH*k)))
    end
    return objc.UIImage:imageWithCIImage_(ci)
  end)
  if ok and img and img.width then
    pcall(function() img.filter=image.filter.point end)
    camImg=img
  else previewOk=false print("preview off: "..tostring(img)) end
end