Motion Detection V1.00 with Codea

Code Sharing
1

These days I was interesting in motion detection on Codea, we all know that iPad have very powerful GPU, so I want to try to use it to do some interesting thing, for examples, to do the real-time motion detection woth shader, and after a few tests, the baseline is OK.

Here is the demo(a video 37MB):

http://v.youku.com/v_show/id_XMjY0ODEwMDYzNg==.html

Here is the code:


-- MotionDetect V1.00 20170318 Baseline
-- GrabMotion
-- ??????

function setup()
    mo = Motion
    -- ????????????????
    mo.init()
end

function draw()

    background(40, 40, 50)
    
    -- ???? starting detect
    mo.detect()
    
    -- ??????????????????? add rect on the motion area
    mo.addRect()
end


-- ???????
Motion = {}

local mo = Motion

function Motion.init()
    displayMode(OVERLAY)
    spriteMode(CORNER)
    memory = 0
    -- ?????????? adjust the sub of two colors 
    parameter.number("deltaColor",0,0.5,0.2)
    -- parameter.number("dScaler",2,20,10)
    parameter.watch("memory")
    parameter.watch("1/DeltaTime")
    parameter.watch("ElapsedTime")
    parameter.watch("os.clock()")

    -- parameter.watch("period1")
    cameraSource(CAMERA_FRONT)
    -- cameraSource(CAMERA_BACK)

    img = image(CAMERA)
    print(img)
    
    -- ??????????????? set some parameter in the rect function
    scaler = 10
    threshold = 20
    
    -- ?????????? keep the coordinate of the motion area
    cdx,cdy = {},{}
    
    -- ?? mesh????? GPU create mesh, prepare using GPU to do the detect job
    m = mesh()
    mp = mesh()
    ma = mesh()
    
    tex = image(WIDTH,HEIGHT)
    period1 = ElapsedTime
    -- period1 = os.clock()
    
    -- ????????????
    delayNextFrame = 10/60
    
    -- ????????? delayNextFrame
    period2 = period1 + delayNextFrame
    
    img0 = image(WIDTH/1,HEIGHT/1)
    img1 = image(WIDTH/1,HEIGHT/1)
    img2 = image(WIDTH/1,HEIGHT/1)
    img3 = image(WIDTH/1,HEIGHT/1)
    img4 = image(WIDTH/scaler,HEIGHT/scaler)
    
    local w,h = img1.width, img1.height

    -- ?????? detect motion
    m:addRect(WIDTH/2,HEIGHT/2,WIDTH/1,HEIGHT/1)
    m.shader = shader(myShader.vs,myShader.fs)
    -- m.texture = img1
    m.shader.tex0 = img0
    m.shader.tex1 = img1
    m.shader.tex2 = img2
    
    -- ???????????? add rect to show where it action
    ma:addRect(WIDTH/2,HEIGHT/2,WIDTH/1,HEIGHT/1)
    ma.shader = shader(myShader.vs,myShader.fs1)
    ma.shader.resolution = vec2(WIDTH,HEIGHT)
    ma.shader.tex0 = img3
    
    -- ??????
    mp:addRect(WIDTH/2+200,HEIGHT/2,WIDTH/3,HEIGHT/3)
    mp.texture = img2   
end

function Motion:detect()
    pushStyle()
    spriteMode(CORNER)
    
    -- ????????
    memory = string.format("%.3f Mb",collectgarbage("count")/1024)
    
    -- ????????
    m.shader.deltaColor = deltaColor
        
    -- ????????? img0
    setContext(img0)
    sprite(CAMERA,0,0,WIDTH,HEIGHT)
    setContext()
    
    -- ??? draw ?????? DeltaTime
    -- currentT = os.clock()
    currentT = ElapsedTime
    
    -- ?? draw ????????????? draw ????? 2 ?
    local dTPerDraw = DeltaTime
    
    -- ????????????????????? dTPerDraw ???????????
    if currentT - period1 > dTPerDraw  then
        period1 = currentT
        setContext(img1)
        -- sprite(CAMERA,WIDTH/2,HEIGHT/2,WIDTH,HEIGHT)
        sprite(CAMERA,0,0,WIDTH,HEIGHT)
        -- print("p1: ",period1)
        setContext()
    end
    
    -- ???????????????
    if currentT - period2  > dTPerDraw  then
        period2 = currentT + delayNextFrame
        setContext(img2)
        -- sprite(CAMERA,WIDTH/2,HEIGHT/2,WIDTH,HEIGHT)
        sprite(CAMERA,0,0,WIDTH,HEIGHT)
        -- print("p2: ",period2)
        setContext()
        -- cdx,cdy = {},{}
    end
    
    -- ???? m:draw ?? shader ????????????????? img3 
    setContext(img3)
    m:draw()
    setContext()
    
    -- ?????????????? img3
    sprite(img3,0,0,WIDTH,HEIGHT)
    
    -- ???????
    sprite(CAMERA,0,0,WIDTH,HEIGHT)
    popStyle()
end

function Motion:addRect()
    --[[ ????? ma ?? shader ???????????????????
    ma.shader.tex0 = img3
    ma:draw()
    --]]
    
    -- ??? Codea ?? CPU ???????
    -- ???? img4 ??????(?????)?????????
    local sw,sh = WIDTH/scaler,HEIGHT/scaler
    setContext(img4)
    pushMatrix()
    sprite(img3,0,0,sw,sh)
    popMatrix()
    setContext()
    
    -- ??????????
    sprite(img4,WIDTH-sw-5,HEIGHT-sh-5,sw,sh)
    
    ---[[ ???????????????????????????????
    local w,h = img4.width,img4.height
    local k= 0
    -- ???????????
    local sx,sy = 2,2
    for x= 1,w,sx do
        for y = 1,h,sy do
            local r,g,b,a =img4:get(x,y)
            if (r==255 and g == 255 and b == 0) then               
                k = k + 1
                if k > threshold then
                    table.insert(cdx,x)
                    table.insert(cdy,y)
                    -- print(cdx[1])
                    speech.say("act")
                    -- speech.say("?")
                    -- sound("Game Sounds One:Blaster")
                    speech.stop()
                    k=0
                end
            end
        end
    end
       
    -- ??????? cdx,cdy ??????????????????????????
    if #cdx ~= 0 and #cdy ~= 0 then    
        pushStyle()
        -- print("#cdx,cdx[1]: ",#cdx,cdx[1])
        -- print("#cdy,cdy[1]: ",#cdy,cdy[1])
        -- local cx,cy = 
        table.sort(cdx)
        table.sort(cdy)
        -- print(cx,#cx)
        -- ????????????
        local lb,rt= vec2(cdx[1],cdy[1]),vec2(cdx[#cdx],cdy[#cdy])
        -- ?????
        lb,rt=lb*scaler,rt*scaler
        -- print(lb,rt)
        -- fill(10, 255, 0, 255)
        noFill()
        stroke(0, 221, 255, 255)
        strokeWidth(5)
        -- rectMode(CENTER)
        -- ??????????????
        local x,y,w,h = lb.x,lb.y,rt.x-lb.x,rt.y-lb.y
        rect(x,y,w,h)
        sprite("Tyrian Remastered:Flame 1",x,y,w,h)
        -- ???????????? cdx,cdy ??????
        cdx,cdy = {},{}
        popStyle()
    end
    --]]
end

myShader = {
vs =[[
uniform mat4 modelViewProjection;

//This is the current mesh vertex position, color and tex coord
// Set automatically
attribute vec4 position;
attribute vec4 color;
attribute vec2 texCoord;

//This is an output variable that will be passed to the fragment shader
varying lowp vec4 vColor;
varying highp vec2 vTexCoord;

void main()
{
gl_Position = modelViewProjection * position;

vColor = color;
vTexCoord = texCoord;
}
]],

-- ?????????? shader
fs =[[
uniform highp sampler2D tex0;
uniform highp sampler2D tex1;
uniform highp sampler2D tex2;

uniform highp float deltaColor;

//The interpolated vertex color for this fragment
varying lowp vec4 vColor;

//The interpolated texture coordinate for this fragment
varying highp vec2 vTexCoord;

uniform highp float time;

highp float grey(lowp vec4 col)
{
highp float grey = 0.2126*col.r + 0.7152* col.g + 0.0722*col.b;
return grey;
}

void main()
{
highp vec2 uv = vTexCoord;

highp vec4 col,col0,col1,col2;

col0 = texture2D(tex0,uv);
col1 = texture2D(tex1,uv);
col2 = texture2D(tex2,uv);

// ????????????????????????
highp vec4 dCol;
highp float g;

dCol = abs(col1-col2);

g = grey(dCol);

if (g <= deltaColor) { col = col0;
    // ?????????????
    //col = vec4(0.,0.,0.,1.);
} else {
    col = vec4(1.,1.,0.0,1.);
}

gl_FragColor = col;
}
]],

-- ?????????? shader
fs1 =[[

varying highp vec2 vTexCoord;

highp vec4 col,col0,col1,col2;

uniform highp sampler2D tex0;
uniform highp sampler2D tex1;
uniform highp vec2 resolution;

void main() {
highp vec2 uv = vTexCoord;

col0 = texture2D(tex0,uv);

    // ???
    highp vec2 dl,dr,db,dt;


if (col0.r ==1. && col0.g == 1. && col0.b == 0.) {
    highp vec2 step;
    step = uv.xy/resolution.xy;
    // 
    bool p =true;
    highp float k = 1.;

    while (p && k <200.) {
    //if (k < 2000.) {

    dl = uv-vec2(step.x,0.)*k;
    dr = uv+vec2(step.x,0.)*k;
    db = uv-vec2(0.,step.y)*k;
    dt = uv+vec2(0.,step.y)*k;
    
    highp vec4 l,r,b,t;
    l = texture2D(tex0,dl);
    r = texture2D(tex0,dr);
    b = texture2D(tex0,db);
    t = texture2D(tex0,dt);
    
    if ((l.r == 1. && l.g ==1.) || (r.r == 1. && r.g ==1.) || (b.r == 1. && b.g ==1.) || (t.r == 1. && t.g ==1.)){
        // ??????
        k = k+10.;
    } else {k = 1.; p = false;}

    }
    
} else {col = col0;}

    // ???
    highp float d=10.;
    if ((uv.x>=dl.x && uv.x <= dr.x) && (uv.y>=db.y && uv.y <= dt.y)) {col = vec4(1.,0.,0.,1.);}
    if ((uv.x>=dl.x+d && uv.x <= dr.x-d) && (uv.y>=db.y+d && uv.y <= dt.y-d)) {col = col0;}

gl_FragColor = col;
}

]]
}

My goal is to let the GPU to do all the heavy job, but it is hard to add the real-time rect with shader, so I have to use the CPU, but I will go on to think how to solve it, if you are interesting of these, please join me, thanks!

The code is published on github:

https://github.com/FreeBlues/CodeaMotionDetect

Here is the screenshot:

The left hand action:


The left hand stop, the right hand action:


BTW, I only have a iPad Pro now, so the code have not tested on iPad, iPad Air, iPad Mini etc. So you can tell me the best parameters on your devices.

2

Great job. They detect motion very well.

3

That looked kind of similar to what I had posted in a recent discussion. It was detecting differences in 2 images. I didn’t do as much in mine as you do in yours. I like your flame in the moving images. See the link below.

https://codea.io/talk/discussion/8167/advance-only-when-a-new-camera-capture-frame-exists#latest

I modified my code to use 2 live images and got this code below. As I said, I didn’t do as much as in your code. Yours shows the differences better.

EDIT: Modified the code below to show 1 image with movement showing as yellow.

displayMode(FULLSCREEN)

function setup()
    parameter.watch("1/DeltaTime//1")
    parameter.number("diff",0,1,.1)
    cameraSource(CAMERA_BACK)
    xSize,ySize=WIDTH//1.1,HEIGHT//1.1
    m=mesh()
    m:addRect(WIDTH//2,HEIGHT//2,xSize,ySize)
end

function draw()
    background(0)     
    collectgarbage()
    i=image(CAMERA)
    if image1 then
        if i~=nil then
            img1=i:copy(WIDTH//2,HEIGHT//2,xSize,ySize)
            m.texture1=img1
            got1=true
        end        
    elseif i~=nil then
        img2=i:copy(WIDTH//2,HEIGHT//2,xSize,ySize)
        got2=true
    end  
    if got1 and got2 then
        sprite(img1,WIDTH//2,HEIGHT//2)
        m.shader=shader(vs,fs)
        m.shader.texture2=img2
        m.shader.dd=diff
        m:draw()
    end  
    image1=not image1  
end

vs= [[  uniform mat4 modelViewProjection;
        attribute vec4 position;
        attribute vec2 texCoord;
        varying highp vec2 vTexCoord;
        uniform float dd;
        void main()
        {   vTexCoord = texCoord;
            gl_Position = modelViewProjection * position;
        }    
    ]]

fs= [[  precision highp float;
        uniform highp sampler2D texture1;
        uniform highp sampler2D texture2;
        varying highp vec2 vTexCoord;
        uniform float dd;
        void main()
        {   lowp vec4 col1=texture2D(texture1,vTexCoord);
            lowp vec4 col2=texture2D(texture2,vTexCoord);
            if (abs(col2.r-col1.r)<dd && abs(col2.g-col1.g)<dd && 
                    abs(col2.b-col1.b)<dd)
                discard; 
            else 
                gl_FragColor=vec4(1.,1.,0.,.5);
        }
    ]]
4

Wow, @binaryblues , that’s really cool!

5

@binaryblues Are you planning to make some sort of a game where it will see where your hand is.

6

@dave1707 : Yes, our essential idea is the same – to compare the different from two frames, and let the shader to do the compare, the implementation we chose different ways.

At first, I want to use the XOR(^) operator to the colors(I think maybe it will get better performance), but it seems that the operator can not used on float, I tried it on int, but also failed, I searched for some examples but can not find any, so I have to use the SUB.

Another difficult is how to get two frames flexible, I was confused by the draw()/ElapsedTime/os.clock(), so I stop and try to draw the flow on paper:

After drawing it clear, I got the code logic(so if you get some problem, draw it or write it is a good idea :smile: ), then you can select two frames at any time.

@CamelCoder : I am glad you like it! I will enhance the code for better effect.

@dave1707 : I will enhance the code first, and develop some education APP for child, or some games after that.