In celebration of the updates to Modern in the latest Beta, I had Claude do this AR project:
It creates a voxel gauntlet around your hand, and it has a couple other nifty tricks to boot. ![]()
All in one tab too:
-- Voxel Glove v0.9 (Modern): live rear camera, Apple Vision hand pose,
-- the hand drawn as a 3D glove of lit cubes that turns with the hand.
-- Depth from foreshortening: palm width (index to little knuckle) sets the
-- hand's scale; each part's true length = its proportion x that width, and
-- how much shorter it looks on screen gives how far it leans in or out.
-- Which hand is decided once, over the first frames after the hand appears
-- (arm entering from the right = right hand, else Vision's guess), and kept
-- until the hand leaves view. Palm facing then follows the thumb's side.
-- Short dropouts hold the glove still; gone for GONE_AFTER s clears it.
-- Joints are steadied with a One Euro filter (heavy smoothing when still,
-- light when moving), and small tilts near flat are ignored.
-- The sword is a latched state: a fist with the thumb wrapped (hilt grip)
-- draws it, opening the hand puts it away, and losing sight of the fingers
-- does not drop it. Its grip sits between the knuckles and the curled
-- fingers' middle joints, inside the fist.
-- Flick: a fingertip that starts against the thumb tip and snaps away from
-- it within FLICK_TIME throws a burst of sparkles off that fingertip.
-- Opens on a title card that fades by itself, or at once on a tap.
-- Parameters (sidebar, hide it with the viewer's button): Cubes (cubes
-- across the palm), Status (prints frame rate, hand, tool etc. to the
-- console), Calibrate, Cycle orientation. A tap on the screen also
-- calibrates: hold your hand flat, fingers straight.
-- Camera/Vision code is unchanged from v0.1, plus chirality.
SKELETON={{"VNHLKWRI","VNHLKTCMC","VNHLKTMP","VNHLKTIP","VNHLKTTIP"},
{"VNHLKWRI","VNHLKIMCP","VNHLKIPIP","VNHLKIDIP","VNHLKITIP"},
{"VNHLKWRI","VNHLKMMCP","VNHLKMPIP","VNHLKMDIP","VNHLKMTIP"},
{"VNHLKWRI","VNHLKRMCP","VNHLKRPIP","VNHLKRDIP","VNHLKRTIP"},
{"VNHLKWRI","VNHLKPMCP","VNHLKPPIP","VNHLKPDIP","VNHLKPTIP"}}
CHAINS={thumb={"VNHLKTCMC","VNHLKTMP","VNHLKTIP","VNHLKTTIP"},
index={"VNHLKIMCP","VNHLKIPIP","VNHLKIDIP","VNHLKITIP"},
middle={"VNHLKMMCP","VNHLKMPIP","VNHLKMDIP","VNHLKMTIP"},
ring={"VNHLKRMCP","VNHLKRPIP","VNHLKRDIP","VNHLKRTIP"},
little={"VNHLKPMCP","VNHLKPPIP","VNHLKPDIP","VNHLKPTIP"}}
ORDER={"index","middle","ring","little","thumb"}
-- segment length / palm width; typical adult proportions until you calibrate
REST={palm=1.05,thumb={0.47,0.35,0.29},index={0.47,0.28,0.22},
middle={0.53,0.33,0.24},ring={0.48,0.32,0.23},little={0.35,0.22,0.20}}
-- Vision orientation code, name, picture rotation in degrees
ORIENTS={{1,"up",0},{6,"right",-90},{3,"down",180},{8,"left",90}}
PREVIEW_EVERY=1 -- camera picture refresh, in draw frames (0 = off)
PREVIEW_SCALE=0.03 -- picture size before conversion; smaller = bigger blocks
SHOW_SKELETON=false
INTRO_HOLD,INTRO_FADE=4,2 -- title card: seconds shown, then seconds fading out
ACROSS=6 -- cubes across the palm; sets the cube size (Cubes parameter)
ACROSS_MIN,ACROSS_MAX=4,16
HAND="auto" -- "auto", "right" or "left"
LOCK_FRAMES=8 -- frames of evidence before the hand is locked as left or right
GONE_AFTER=0.5 -- seconds without a hand before the glove is cleared
-- sizes in palm widths, kept the same whatever the cube size: Cubes only
-- changes how many cubes fill each part (fingers: 1 cube across at 4, 5 at 16)
PALM_W=1.15 -- palm plate width (1 = knuckle to knuckle)
PALM_T=0.5 -- palm thickness
FINGER_W=0.32 -- finger width and thickness
THUMB_W=0.45 -- thumb base segment width and thickness
TIP_EXT=0.15 -- how far each fingertip reaches past Vision's tip point
CUFF_LEN=0.5 -- cuff length up the arm
-- block textures from Codea's built-in Blocks pack, first name found wins;
-- if none is found the colours below are used and the pack's names printed
USE_TEXTURES=true
TEX_BACK={"Brick_Red","Brick","Red_Brick"}
TEX_PALM={"Stone","Cobblestone","Rock"}
TEX_CUFF={"Sand","Sandstone"}
BACK=color(120,220,255) PALM=color(55,105,150) CUFF=color(240,240,250)
-- steadying: One Euro filter. Lower MIN_CUTOFF = steadier when still but
-- floatier; higher BETA = less lag on fast moves
MIN_CUTOFF=1.0 -- Hz
BETA=0.01 -- per screen point per second
D_CUTOFF=1.0 -- Hz
LEAN_DEAD=12 -- degrees of tilt ignored near flat
-- tools
GRIP_ON=0.15 -- s a hilt grip must hold before the sword appears
RELEASE_T=0.25 -- s the open hand must hold before the sword is put away
CURL_K=1.0 -- finger curled: tip nearer the wrist than this x its knuckle
OPEN_K=1.3 -- finger open: tip further from the wrist than this x its knuckle
THUMB_WRAP=0.6 -- grip: thumb tip within this many palm widths of the index middle joint
FLICK_IN=0.35 -- flick: fingertip starts within this many palm widths of the thumb tip
FLICK_OUT=0.75 -- flick: and ends further than this
FLICK_TIME=0.2 -- flick: within this many seconds
FLICK_COOL=0.3 -- s before the same finger can flick again
FLICK_SPARKS=16 -- sparkles per flick
FLICK_SPEED=7 -- sparkle launch speed, palm widths per second
SWORD_PX=0.14 -- sword model pixel size in palm widths
CUBE_GAP=0.9 -- cube size inside its cell (1 = solid)
Z_SIGN=1 -- +1: larger z is further from the camera; flip if lit wrong
PALM_TILT=1 -- +1: knuckles lean away when the palm looks short; -1 toward
ROLL_TILT=-1 -- same for the index side when the palm looks narrow
-- sword model, an original pixel-art longsword, pommel at the bottom.
-- Rows are drawn along the blade, columns across its width.
SWORD={
"....D....",
"...DgD...",
"..DgRgD..",
"...DGD...",
"....D....",
"...lLl...",
"...LlL...",
"...lLl...",
"...LlL...",
"...lLl...",
"...LlL...",
".DD...DD.",
"DgGGRGGgD",
".DGGGGGD.",
"...SsS...",
"...WsS...",
"...WsS...",
"...WsS...",
"...WsS...",
"...WsS...",
"...WsS...",
"...WsS...",
"...WsS...",
"...WsS...",
"...WsS...",
"...WsS...",
"...WsS...",
"...WsS...",
"...WsS...",
"...WsS...",
"...WsS...",
"...WsS...",
"...WsS...",
"...WsS...",
"...WsS...",
"...WsS...",
"...WsS...",
"...WsS...",
"...WsS...",
"...WSS...",
"....WS...",
"....W....",
}
SWORD_GRIP={6,11} -- rows held in the fist (1 = bottom row)
SWORD_COLS={D=color(150,100,30),G=color(225,175,55),g=color(255,230,140),
R=color(200,30,50),L=color(95,55,30),l=color(140,90,50),
S=color(185,195,210),s=color(130,140,160),W=color(245,250,255)}
SWORD_THICK={S=0.6,s=0.6,W=0.6} -- depth in pixels; anything else 1.4
SPARK_COLS={color(255,255,255),color(255,215,90),color(120,230,255),color(255,140,230)}
function setup()
viewer.mode=FULLSCREEN
joints,status,orient,chir={},"starting camera",1,0
frames,runs,visionMs,rate,t0=0,0,0,0,now()
camImg,previewOk,tick,camW,camH=nil,true,0,1920,1440
calibrated,sparks=false,{}
clearHand()
backBox=boxOf(BACK,TEX_BACK)
palmBox=boxOf(PALM,TEX_PALM)
cuffBox=boxOf(CUFF,TEX_CUFF)
swordBox={}
for k,c in pairs(SWORD_COLS) do swordBox[k]=boxOf(c) end
sparkBox={}
for i,c in ipairs(SPARK_COLS) do sparkBox[i]=glowOf(c) end
sparkQ=quat.angleAxis(45,vec3(1,1,0))
sun=light.directional(vec3(0.35,-0.6,1)) sun.intensity=2.5
cuffCells=ringCells()
lastT=now()
introT,introGone=now(),false
parameter.integer("Cubes",ACROSS_MIN,ACROSS_MAX,ACROSS,function(n)
if n~=ACROSS then ACROSS=n cuffCells=ringCells() end
end)
parameter.action("Status",printStatus)
parameter.action("Calibrate",calibrate)
parameter.action("Cycle orientation",function()
orient=orient%#ORIENTS+1 joints={} clearHand()
print("orientation "..ORIENTS[orient][2])
end)
local ok,err=pcall(startCamera)
if not ok then status="camera error: "..tostring(err) end
print(status)
end
-- everything the old on-screen status line showed, to the console
function printStatus()
print(string.format("%s | vision %.0f ms | %.1f runs/s | orient %s | %s %s | %s | %d cubes | %s",
status,visionMs,rate,ORIENTS[orient][2],
rig and rig.hand or "no hand",rig and (rig.toward and "palm" or "back") or "",
tool or "no tool",cubeCount or 0,calibrated and "calibrated" or "not calibrated"))
end
-- forget the hand: glove off screen, hand side unlocked, tools put away
function clearHand()
rig,held,flt,sFlt=nil,{},{},{}
lockedHand,votes,nvotes,toward,flipN=nil,{},0,true,0
tool,timers,barN,lastSeen,flicks=nil,{},{},nil,{}
end
function draw()
background(20,22,30)
tick=tick+1
autoOrient()
local f=arSession and arSession.currentFrame
if f then
local ts=f.timestamp
if ts~=lastTs then
lastTs,frames=ts,frames+1
local ok,err=pcall(detect,f)
if not ok then status="vision error: "..tostring(err) print(status) end
if previewOk and PREVIEW_EVERY>0 and tick%PREVIEW_EVERY==0 then grabPreview(f) end
end
f=nil
collectgarbage() -- hand the camera buffer back to ARKit right away
end
local tn=now()
DT=math.max(1/120,math.min(0.1,tn-lastT)) lastT=tn
if tn-t0>=1 then rate,runs,t0=runs/(tn-t0),0,tn end
local o=ORIENTS[orient]
local x0,y0,dw,dh=viewRect()
style.push()
if camImg then
local sw,sh=dw,dh
if o[3]==90 or o[3]==-90 then sw,sh=dh,dw end
matrix.push()
matrix.translate(WIDTH/2,HEIGHT/2)
matrix.rotate(o[3],0,0,1)
style.shapeMode(CENTER)
sprite(camImg,0,0,sw,sh)
matrix.pop()
end
local ok,err=pcall(function()
updateRig(x0,y0,dw,dh)
if rig and lastSeen and tn-lastSeen>GONE_AFTER then clearHand() end
drawGlove()
end)
if not ok then status="glove error: "..tostring(err) print(status) end
if SHOW_SKELETON then
style.stroke(255,220,0) style.strokeWidth(5)
for _,b in ipairs(SKELETON) do
for i=2,#b do
local p,q=joints[b[i-1]],joints[b[i]]
if p and q then line(x0+p.x*dw,y0+p.y*dh,x0+q.x*dw,y0+q.y*dh) end
end
end
end
style.pop()
drawIntro()
end
-- title card: fades by itself after INTRO_HOLD, or is gone at once on a tap.
-- Each line wraps at 85% of the screen width (style.textWrapWidth), and the
-- lines are stacked by their measured, wrapped heights (textSize).
function drawIntro()
if introGone then return end
local t=now()-introT
local a=1
if t>INTRO_HOLD then a=1-(t-INTRO_HOLD)/INTRO_FADE end
if a<=0 then introGone=true return end
local u=math.min(WIDTH,HEIGHT)
local big,small=u*0.13,math.max(16,u*0.035)
local lines={{"Voxel",big,BACK,0},{"Gauntlet",big,BACK,small*1.2},
{"Close your hand as if holding a sword, and a sword appears",small,color(255),small*0.6},
{"Flick your fingers to make magic dust",small,color(255),0}}
style.push() style.noStroke() style.shapeMode(CENTER)
style.fill(0,0,0,150*a) rect(WIDTH/2,HEIGHT/2,WIDTH,HEIGHT)
style.textAlign(CENTER) style.textWrapWidth(WIDTH*0.85) style.textMode(CENTER)
local total=0
for _,l in ipairs(lines) do
style.fontSize(l[2])
local _,h=textSize(l[1])
l.h=h total=total+h+l[4]
end
local y=HEIGHT/2+total/2
for _,l in ipairs(lines) do
style.fontSize(l[2])
local cy=y-l.h/2
local o=l[2]*0.06
style.fill(0,0,0,220*a) text(l[1],WIDTH/2+o,cy-o)
style.fill(l[3].r,l[3].g,l[3].b,255*a) text(l[1],WIDTH/2,cy)
y=y-l.h-l[4]
end
style.pop()
end
function touched(t)
if t.state~=ENDED then return end
if not introGone then introGone=true return end
calibrate()
end
-- a unit cube with a lit material: a block texture if one of the names is
-- in the Blocks pack, else the plain colour
function boxOf(col,names)
local b=mesh.box(vec3(0.5,0.5,0.5))
b.material=material.lit()
b.material.color=col
if USE_TEXTURES and names then
for _,n in ipairs(names) do
local ok,img=pcall(function() return image.read(asset.builtin.Blocks[n]) end)
if ok and img then
pcall(function() img.filter=image.filter.point end)
b.material.map=img b.material.color=color(255)
return b
end
end
if not listedBlocks then
listedBlocks=true
local t={}
pcall(function() for name in pairs(asset.builtin.Blocks) do t[#t+1]=tostring(name) end end)
table.sort(t)
print("no block texture found for "..names[1]..". Blocks pack has: "..table.concat(t,", "))
end
end
return b
end
-- an unlit cube that glows in its own colour (sparkles)
function glowOf(col)
local b=mesh.box(vec3(0.5,0.5,0.5))
b.material=material.unlit()
b.material.color=col
return b
end
-- screen position of a joint, as a vec2
function sp(n)
local j=joints[n]
if j then return vec2(VX0+j.x*VDW,VY0+j.y*VDH) end
end
-- how far a part of true length L leans in or out when it looks l long.
-- The first LEAN_DEAD degrees of tilt are ignored: near flat, a tiny change
-- in apparent length means a big change in angle, which is where jitter
-- came from.
function lean(L,l)
if L<=0 then return 0 end
local a=math.acos(math.max(-1,math.min(1,l/L)))-math.rad(LEAN_DEAD)
if a<=0 then return 0 end
return L*math.sin(a)
end
-- record this hand's proportions; assumes it is flat and fingers straight
function calibrate()
local w,im,mm,pm=sp("VNHLKWRI"),sp("VNHLKIMCP"),sp("VNHLKMMCP"),sp("VNHLKPMCP")
if not (w and im and mm and pm) then print("calibrate: no full hand in view") return end
local W=im:distance(pm)
if W<10 then return end
REST.palm=w:distance(mm)/W
for name,ch in pairs(CHAINS) do
for i=2,#ch do
local a,b=sp(ch[i-1]),sp(ch[i])
if a and b then REST[name][i-1]=a:distance(b)/W end
end
end
calibrated=true
local s=string.format("calibrated palm %.2f",REST.palm)
for _,n in ipairs(ORDER) do s=s..string.format(" %s %.2f %.2f %.2f",n,REST[n][1],REST[n][2],REST[n][3]) end
print(s)
end
-- one vote for which hand this is: from the side the arm enters, else
-- Vision's guess; nil when neither says anything
function handVote(w,km)
local a=w-km -- points back up the arm
if math.abs(a.x)>math.abs(a.y)*0.6 then return (a.x>0) and "right" or "left" end
if chir==1 then return "right" elseif chir==-1 then return "left" end
end
-- which hand: voted on over the first LOCK_FRAMES frames, then fixed until
-- the hand leaves view
function whichHand(w,km)
if HAND~="auto" then return HAND end
if lockedHand then return lockedHand end
local v=handVote(w,km)
if v then votes[v]=(votes[v] or 0)+1 nvotes=nvotes+1 end
local h=((votes.left or 0)>(votes.right or 0)) and "left" or "right"
if nvotes>=LOCK_FRAMES then lockedHand=h print("hand locked: "..h) end
return h
end
-- palm toward the camera? For a right hand seen from behind (the camera is
-- roughly where your eyes are), fingers up with the thumb to the right means
-- palm toward. A change has to show for 3 frames in a row before it's taken.
function palmToward(w,km,thumb,S,hand)
local f=km-w
local t=thumb-w
local c=f.x*t.y-f.y*t.x -- negative: thumb clockwise of the fingers
if math.abs(c)>S*S*0.05 then
local want=(hand=="right")==(c<0)
if want~=toward then
flipN=flipN+1
if flipN>=3 then toward,flipN=want,0 end
else flipN=0 end
end
return toward
end
-- One Euro filter, for a vec3 (st=state table) or a number (num=true)
local function alpha(cut,dt) local r=2*math.pi*cut*dt return r/(r+1) end
function euro(st,x,num)
if st.x==nil then st.x,st.d=x,num and 0 or vec3(0,0,0) return x end
local dx=(x-st.x)*(1/DT)
st.d=st.d+(dx-st.d)*alpha(D_CUTOFF,DT)
local speed=num and math.abs(st.d) or st.d.length
st.x=st.x+(x-st.x)*alpha(MIN_CUTOFF+BETA*speed,DT)
return st.x
end
-- seconds a condition has held, as a test: true once it has held for secs
function hold(key,cond,secs)
if not cond then timers[key]=nil return false end
timers[key]=timers[key] or now()
return now()-timers[key]>=secs
end
-- finger tests from the raw 2D joints
function fingerCount(w,test)
local n=0
for _,name in ipairs({"index","middle","ring","little"}) do
local ch=CHAINS[name]
local mcp,tip=sp(ch[1]),sp(ch[4])
if mcp and tip and test(tip:distance(w),mcp:distance(w)) then n=n+1 end
end
return n
end
function isGrip(w,S)
local tt,ip=sp("VNHLKTTIP"),sp("VNHLKIPIP") or sp("VNHLKIMCP")
return fingerCount(w,function(t,m) return t<m*CURL_K end)>=3
and tt and ip and tt:distance(ip)<THUMB_WRAP*S
end
function isOpen(w) return fingerCount(w,function(t,m) return t>m*OPEN_K end)>=3 end
-- the sword is latched: only a clearly open hand puts it away, so it stays
-- in the hand when the fingers tilt out of sight
function updateTools(w,S)
local g=hold("grip",isGrip(w,S),GRIP_ON)
local o=hold("open",isOpen(w),RELEASE_T)
if tool=="sword" then
if o then tool=nil end
elseif g then tool="sword" end
end
-- flick: per finger, remember when its tip was last against the thumb tip;
-- if it is now well away from it and that was under FLICK_TIME ago, it was
-- flicked. Checked once per new Vision frame, from the raw joints (the
-- smoothed ones lag behind a flick). Off while the sword is out.
function checkFlicks(S)
if frames==flickFrame then return end
flickFrame=frames
local tt=sp("VNHLKTTIP")
if not tt or tool=="sword" then return end
local t=now()
for _,name in ipairs({"index","middle","ring","little"}) do
local ch=CHAINS[name]
local tip,dip=sp(ch[4]),sp(ch[3])
local f=flicks[name] or {}
flicks[name]=f
if tip then
local d=tip:distance(tt)/S
if d<FLICK_IN then f.closed=t
elseif d>FLICK_OUT and f.closed and t-f.closed<FLICK_TIME and t>(f.cool or 0) then
f.closed,f.cool=nil,t+FLICK_COOL
local z=rig.P[ch[4]] and rig.P[ch[4]].z or 0
local dir=dip and unit(vec3(tip.x-dip.x,tip.y-dip.y,0)) or unit(vec3(tip.x-tt.x,tip.y-tt.y,0))
burst(vec3(tip.x,tip.y,z),dir,S)
end
end
end
end
-- a spray of sparkles from p, mostly along dir
function burst(p,dir,S)
for i=1,FLICK_SPARKS do
local j=vec3(math.random()-0.5,math.random()-0.5,math.random()-0.5)*0.9
local v=unit(dir+j)*(S*FLICK_SPEED*(0.5+math.random()*0.6))
sparks[#sparks+1]={p=p,v=v,life=0,max=0.4+math.random()*0.5,drag=0.95,
m=sparkBox[math.random(#sparkBox)],size=S*(0.05+math.random()*0.06)}
end
end
-- 3D joint positions (screen points, z = lean) plus the hand's frame.
-- Missing joints hold their last position; a missing palm holds the rig.
function updateRig(x0,y0,dw,dh)
VX0,VY0,VDW,VDH=x0,y0,dw,dh
local w,im,mm,pm,rm=sp("VNHLKWRI"),sp("VNHLKIMCP"),sp("VNHLKMMCP"),sp("VNHLKPMCP"),sp("VNHLKRMCP")
if not (w and im and mm and pm) then return end
lastSeen=now()
local W,lp=im:distance(pm),w:distance(mm)
-- both palm measures can only shrink, so the larger one is nearer the truth
local S=euro(sFlt,math.max(W,lp/REST.palm),true)
local km=(im+pm)*0.5
local hand=whichHand(w,km)
-- thumb side: thumb MP if seen, else the index knuckle (it sits thumb-side)
local tside=sp("VNHLKTMP") or sp("VNHLKTCMC") or im
local tw=palmToward(w,km,tside,S,hand)
local fs=(tw and -1 or 1)*Z_SIGN
local zp=lean(REST.palm*S,lp)*PALM_TILT*Z_SIGN
local zr=lean(S,W)*ROLL_TILT*Z_SIGN
local P={}
local function put(n,p2,z) if p2 then P[n]=vec3(p2.x,p2.y,z) end end
put("VNHLKWRI",w,0)
put("VNHLKMMCP",mm,zp)
put("VNHLKIMCP",im,zp+zr*0.5)
put("VNHLKPMCP",pm,zp-zr*0.5)
put("VNHLKRMCP",rm,zp-zr*0.17)
put("VNHLKTCMC",sp("VNHLKTCMC"),zp*0.25+zr*0.4)
for name,ch in pairs(CHAINS) do
for i=2,#ch do
local a,b=P[ch[i-1]] or held[ch[i-1]],sp(ch[i])
if a and b then
local l=vec2(a.x,a.y):distance(b)
put(ch[i],b,a.z+fs*lean(REST[name][i-1]*S,l))
end
end
end
for n,p in pairs(P) do
flt[n]=flt[n] or {}
P[n]=euro(flt[n],p) held[n]=P[n]
end
for n,p in pairs(held) do if not P[n] then P[n]=p end end -- unseen: hold
local nrm=(P.VNHLKIMCP-P.VNHLKWRI):cross(P.VNHLKPMCP-P.VNHLKWRI)
if nrm.length<1e-6 then nrm=vec3(0,0,1) end
nrm=unit(nrm)
-- palm side of every bar: -normal for a right hand, +normal for a left
local low=(hand=="right")
rig={P=P,S=S,v=S/ACROSS,toward=tw,hand=hand,n=nrm,palmLow=low,ps=low and nrm*-1 or nrm}
updateTools(w,S)
checkFlicks(S)
end
-- one cube: centre c, rotation q, size sx,sy,sz
function cube(m,c,q,sx,sy,sz)
cubeCount=cubeCount+1
matrix.push()
matrix.translate(c.x,c.y,c.z)
matrix.rotate(q)
matrix.scale(sx*CUBE_GAP,sy*CUBE_GAP,sz*CUBE_GAP)
m:draw()
matrix.pop()
end
-- unit vector and dot product, written out so they don't depend on whether
-- this Codea build exposes normalized/dot as methods or properties
function unit(v)
local l=math.sqrt(v.x*v.x+v.y*v.y+v.z*v.z)
if l<1e-9 then return vec3(0,0,1) end
return vec3(v.x/l,v.y/l,v.z/l)
end
function dot3(a,b) return a.x*b.x+a.y*b.y+a.z*b.z end
-- orthonormal frame: Z along dir, Y as close to up as possible
function frame(dir,up)
local Z=unit(dir)
local Y=up-Z*dot3(up,Z)
if Y.length<1e-4 then Y=vec3(0,0,1)-Z*Z.z end
if Y.length<1e-4 then Y=vec3(0,1,0)-Z*Z.y end
Y=unit(Y)
return Y:cross(Z),Y,Z
end
-- a bar of cubes from a to b, wf by tf palm widths in cross-section. The
-- cube count across is that size x ACROSS (at least 1), and the cubes are
-- sized to fill it exactly, so the bar keeps its size at any Cubes setting.
-- Along the bar, cube length stretches to span joint to joint; the count
-- along it only changes once the length has moved well past the boundary,
-- so cubes don't pop in and out as the hand breathes. key names the bar.
-- mp: material for the palm-side layer (nil = same as m). up: the bar's
-- thickness direction (default the palm normal).
function bar(key,a,b,wf,tf,m,mp,up)
local d=b-a
local len=d.length
if len<1 then return end
local X,Y,Z=frame(d,up or rig.n)
local q=quat.lookRotation(Z,Y)
local S,v=rig.S,rig.v
local w=math.max(1,math.floor(wf*ACROSS+0.5))
local t=math.max(1,math.floor(tf*ACROSS+0.5))
local cw,ct=wf*S/w,tf*S/t
local r=len/v
local n=barN[key]
if not n or math.abs(r-n)>0.8 then n=math.max(1,math.floor(r+0.5)) barN[key]=n end
local step=len/n
local pj=rig.palmLow and 0 or t-1
-- hollow: only the outer shell, plus solid end slices so no bar shows
-- its inside; cube count grows with the square of ACROSS, not the cube
for k=0,n-1 do
local base,cap=a+Z*((k+0.5)*step),(k==0 or k==n-1)
for i=0,w-1 do
for j=0,t-1 do
if cap or i==0 or i==w-1 or j==0 or j==t-1 then
local mm=(mp and t>1 and j==pj) and mp or m
cube(mm,base+X*((i-(w-1)/2)*cw)+Y*((j-(t-1)/2)*ct),q,cw,ct,step)
end
end
end
end
end
-- cuff cross-section, in cubes: an oval ring one cube thick around the arm,
-- a little wider and thicker than the palm
function ringCells()
local a,b=ACROSS*PALM_W/2+0.6,ACROSS*PALM_T/2+1.2
local cells={}
for i=-math.ceil(a),math.ceil(a)-1 do
for j=-math.ceil(b),math.ceil(b)-1 do
local x,y=i+0.5,j+0.5
local o=(x/a)^2+(y/b)^2
local inn=(x/(a-1))^2+(y/(b-1))^2
if o<=1 and inn>1 then cells[#cells+1]={x,y} end
end
end
return cells
end
function drawGlove()
cubeCount=0
matrix.push()
matrix.ortho(0,WIDTH,0,HEIGHT,-5000,5000)
light.push(sun)
if rig then
local P,v=rig.P,rig.v
local w,km=P.VNHLKWRI,(P.VNHLKIMCP+P.VNHLKPMCP)*0.5
bar("palm",w,km,PALM_W,PALM_T,backBox,palmBox)
for _,name in ipairs(ORDER) do
local ch=CHAINS[name]
for i=2,#ch do
local a,b=P[ch[i-1]],P[ch[i]]
if a and b then
local s=(name=="thumb" and i==2) and THUMB_W or FINGER_W
-- Vision's tip point sits short of the real fingertip, so the last
-- segment carries on past it
if i==#ch then b=b+unit(b-a)*(TIP_EXT*rig.S) end
bar(name..i,a,b,s,s,backBox,palmBox)
end
end
end
-- cuff: rings stepping back up the arm; skip cubes behind the arm's axis
local X,Y,Z=frame(w-km,rig.n)
local q=quat.lookRotation(Z,Y)
for k=0,math.max(1,math.floor(CUFF_LEN*ACROSS+0.5))-1 do
local axis=w+Z*((k+0.5)*v)
for _,c in ipairs(cuffCells) do
local p=axis+X*(c[1]*v)+Y*(c[2]*v)
if (p.z-axis.z)*Z_SIGN<=0 then cube(cuffBox,p,q,v,v,v) end
end
end
if tool=="sword" then drawSword(P,w,km) end
end
drawSparks()
light.pop()
matrix.pop()
end
-- sword in the fist. The grip's centre is halfway between the knuckles and
-- the curled fingers' middle joints, which puts it inside the fist (this is
-- where v0.5 put it). Those joints hold their last position when unseen, so
-- the sword stays put when the fingers tilt away; with none known yet it
-- falls back to just inside the knuckles. The grip runs along the knuckle
-- line, blade out of the thumb side; the edge faces the way the knuckles point.
function drawSword(P,w,km)
local S=rig.S
local A=unit(P.VNHLKIMCP-P.VNHLKPMCP)
local F=unit(km-w)
local pips,np=vec3(0,0,0),0
for _,n in ipairs({"VNHLKIPIP","VNHLKMPIP","VNHLKRPIP","VNHLKPPIP"}) do
if P[n] then pips=pips+P[n] np=np+1 end
end
local c=(np>0) and (km+pips*(1/np))*0.5 or km+rig.ps*(0.3*S)
local X,Y,Z=frame(A:cross(F),A) -- Y: along the blade, Z: through its flat
local q=quat.lookRotation(Z,Y)
local px=SWORD_PX*S
local rows,cols=#SWORD,#SWORD[1]
local mid=(SWORD_GRIP[1]+SWORD_GRIP[2])/2
for r=1,rows do
local row=SWORD[r]
for k=1,cols do
local ch=row:sub(k,k)
local m=swordBox[ch]
if m then
local p=c+X*((k-(cols+1)/2)*px)+Y*((r-mid)*px)
cube(m,p,q,px,px,px*(SWORD_THICK[ch] or 1.4))
end
end
end
end
-- sparkles fly on from where they were thrown, slow down, fall and shrink away
function drawSparks()
local keep={}
for _,s in ipairs(sparks) do
s.life=s.life+DT
if s.life<s.max then
s.p=s.p+s.v*DT
s.v=s.v*(s.drag or 0.92)+vec3(0,-(rig and rig.S or 100)*0.6*DT,0)
local k=1-s.life/s.max
cube(s.m,s.p,sparkQ,s.size*k,s.size*k,s.size*k)
keep[#keep+1]=s
end
end
sparks=keep
end
-- default orientation from the device's screen shape (not the viewer's,
-- which changes when the sidebar opens), rechecked when it changes
function autoOrient()
local w,h=WIDTH,HEIGHT
pcall(function()
local b=objc.UIScreen.mainScreen.bounds
if b and b.size then w,h=b.size.width,b.size.height end
end)
if w==lastW and h==lastH then return end
lastW,lastH=w,h
orient=(h>w) and 2 or 1
joints={}
end
-- screen rectangle the oriented camera image fills (aspect-fill, cropped)
function viewRect()
local w,h=camW,camH
local r=ORIENTS[orient][3]
if r==90 or r==-90 then w,h=h,w end
local s=math.max(WIDTH/w,HEIGHT/h)
local dw,dh=w*s,h*s
return (WIDTH-dw)/2,(HEIGHT-dh)/2,dw,dh
end
-- wall-clock seconds (os.clock is CPU time, wrong for Vision)
function now()
local ok,t=pcall(function() return objc.NSProcessInfo.processInfo.systemUptime end)
if ok and type(t)=="number" then return t end
return os.clock()
end
-- ARKit session on the rear camera; 3-DOF tracking is the cheapest config.
-- The delegate only reports errors; it stores a message and does nothing else.
function startCamera()
local cfg
local ok=pcall(function() cfg=objc.AROrientationTrackingConfiguration() end)
if not ok or not cfg then cfg=objc.ARWorldTrackingConfiguration() end
local D=objc.delegate("ARSessionDelegate")
function D:session_didFailWithError_(objSession,objError)
camFailed=tostring(objError)
end
arDelegate=D()
arSession=objc.ARSession()
arSession.delegate=arDelegate
arSession:runWithConfiguration_(cfg)
status="camera running"
end
function detect(frame)
if camFailed then status="ARKit failed: "..camFailed return end
local pb=frame.capturedImage
if not pb then status="no pixel buffer from frame" return end
local c=now()
local req=objc.VNDetectHumanHandPoseRequest()
req.maximumHandCount=1
-- string-keyed table so the bridge sends a dictionary, not an array
local hnd=objc.VNImageRequestHandler:alloc():initWithCVPixelBuffer_orientation_options_(pb,ORIENTS[orient][1],{codeaDummy=1})
if not hnd then status="request handler not created" return end
hnd:performRequests_error_({req},nil)
visionMs=(now()-c)*1000
runs=runs+1
local res=req.results
if type(res)~="table" or #res==0 then joints={} status="no hand" return end
local obs,js=res[1],{}
chir=tonumber(obs.chirality) or 0
local names=obs.availableJointNames or {}
for i=1,#names do
local p=obs:recognizedPointForJointName_error_(names[i],nil)
if p and (p.confidence or 0)>0.2 then
local x,y=p.x,p.y
if not x and p.location then x,y=p.location.x,p.location.y end
if x then js[tostring(names[i])]={x=x,y=y,c=p.confidence} end
end
end
joints=js
status="hand"
end
-- camera picture, shrunk with Core Image's scale filter (plain numbers, no
-- struct), cropped to whole pixels, drawn as hard-edged blocks
function grabPreview(frame)
local ok,img=pcall(function()
local ci=objc.CIImage:imageWithCVPixelBuffer_(frame.capturedImage)
local ok2,e=pcall(function() return ci.extent end)
if ok2 and e and e.size then camW,camH=e.size.width,e.size.height end
local k=PREVIEW_SCALE
if k and k<1 then
local f=objc.CIFilter:filterWithName_("CILanczosScaleTransform")
f:setValue_forKey_(ci,"inputImage")
f:setValue_forKey_(k,"inputScale")
f:setValue_forKey_(1,"inputAspectRatio")
ci=f.outputImage:imageByCroppingToRect_(objc.rect(0,0,math.floor(camW*k),math.floor(camH*k)))
end
return objc.UIImage:imageWithCIImage_(ci)
end)
if ok and img and img.width then
pcall(function() img.filter=image.filter.point end)
camImg=img
else previewOk=false print("preview off: "..tostring(img)) end
end
