Magic mirror/green screen type effect (with video)

Code Sharing
1

Messing around with the camera input - made a small program which acts a bit like the “green screen” visual effect used in (older) TV/films.

Best to use with a stationary iPad - ideally on a stand. Tap the bottom right, but without standing in the field of view of the camera to capture the reference background then walk in front of the camera.a__

-- Magic Mirror
supportedOrientations(LANDSCAPE_ANY)
displayMode(FULLSCREEN)
-- Use this function to perform your initial setup
function setup()
    counter=0
    cameraSource(CAMERA_FRONT)
    refimg = image(CAMERA)  --used as the background reference image
    curimg=image(CAMERA)  --used to store the current image from the camera
    
    xres=32/4  --the size of the horizontal blocks - should divide into 1280 to leave an integer
    yres=18/2  --the size of the vertical blocks - should divide into 720 to leave an integer
    thresh=40. -- a threshold for detecting the difference between the background pixel and current pixel
    dispimg=image(1280,720)  -- used for the image to be displayed
    bgimg=image(xres,yres) -- create a small image for use as the background blocks
    
    setContext(bgimg)
    background(0,0,200) -- currently set to blue but could be changed to anything
    setContext()
    
    
    --an alternative single background image.  make the code substitution commemted on later too
    --[[
    backimg=image(1280,720)
    setContext(backimg)
    sprite("Cargo Bot:Codea Icon",WIDTH/2,HEIGHT/2,1280,720)
    setContext()
    ]]--
end

-- This function gets called once every frame
function draw()
    
    counter = counter + 1
    text("Tap in bottom right to set the background reference image",WIDTH/2,HEIGHT/2)
    if refimg~=nil then
        --
        if counter>0 then
            counter=0
            local curimg = image(CAMERA)
            setContext(dispimg)
            local xc=0
            for i=1,1280,xres do
                local yc=0
                xc = xc + 1
                for j=1,720,yres do
                    yc=yc+1
                    local r,g,b,a= curimg:get(i,j)
                    --check each reference point in turn - if the rgb values are comparable to the reference image then treat as background, otherwise treat as new and show that portion of the camera image
                    if math.abs(ref[xc][yc].r-tonumber(r))<thresh and math.abs(ref[xc][yc].g-tonumber(g))<thresh and math.abs(ref[xc][yc].b-tonumber(b))<thresh then
                        sprite(bgimg,i,j,xres,yres)
                        --substitute above with the following for a full single image as defined earlier
                        --  sprite(backimg:copy(i,j,xres,yres),i,j,xres,yres)
                    else
                        sprite(curimg:copy(i,j,xres,yres),i,j,xres,yres)
                    end
                end
            end
            
        end
        
        setContext()
        sprite(dispimg,WIDTH/2,HEIGHT/2)
    end
    --camera button image
    sprite("Cargo Bot:Crate Goal Yellow",WIDTH*0.95,HEIGHT*0.05,0.1*WIDTH,0.1*HEIGHT)
    sprite("Cargo Bot:Record Solution Icon",WIDTH*0.95,HEIGHT*0.05,0.05*WIDTH,0.05*HEIGHT)
    collectgarbage()
end

function touched(t)
    
    if t.state==ENDED and t.x>0.9*WIDTH and t.y<0.1*HEIGHT then
        --take new reference image
        cameraSource(CAMERA_FRONT)
        refimg = image(CAMERA)
        --camera snapshot resolution from my instance is 1280,720
        --generate a series of checkpoints and store the pixel color for comparison
        ref={}
        local xc=0
        for i=1,1280,xres do
            local yc=0
            xc = xc + 1
            ref[xc]={}
            for j=1,720,yres do
                local red,green,blue,alpha= refimg:get(i,j)
                table.insert(ref[xc],{r=red,g=green,b=blue,a=alpha})
            end
        end
    end
end
2

So laggy :frowning: But nice :wink:

3

Yeah I’m looking at improving this

4

I tried it and it is super cool!

5

Here are some things that you could change in your code to make it run better:
-Try using meshes instead of sprites.
-Maybe you could make the camera image smaller (remove some pixels) so that there are less pixels to look at.

6

I’ve got a version which works a lot faster pasting the current image and pasting over the “mask” for unchanged background parts. However I’m having a problem with the positions at the moment

7

Here’s a much faster version:

-- Magic Mirror
supportedOrientations(LANDSCAPE_ANY)
displayMode(FULLSCREEN)
-- Use this function to perform your initial setup
function setup()
    
    
    counter=0
    cameraSource(CAMERA_FRONT)
    
    refimg = image(CAMERA)  --used as the background reference image
    curimg=image(CAMERA)  --used to store the current image from the camera
    
    xres=4  --the size of the horizontal blocks - should divide into 1280 to leave an integer
    yres=6  --the size of the vertical blocks - should divide into 720 to leave an integer
    thresh=60 -- a threshold for detecting the difference between the background pixel and current pixel
    dispimg=image(1280,720)  -- used for the image to be displayed
    bgimg=image(xres,yres) -- create a small image for use as the background blocks
    
    setContext(bgimg)
    background(0,0,200) -- currently set to blue but could be changed to anything
    setContext()
    
end

-- This function gets called once every frame
function draw()
    
    counter = counter + 1
    text("Tap in bottom right to set the background reference image",WIDTH/2,HEIGHT/2)
    if refimg~=nil then
        --
        if counter>0 then
            counter=0
            local curimg = image(CAMERA)
            setContext(dispimg)
            sprite(curimg,WIDTH/2+128,HEIGHT/2-24,1280,720)
            local xc=0
            for i=1,1280,xres do
                local yc=0
                xc = xc + 1
                for j=1,720,yres do
                    yc=yc+1
                    local r,g,b,a= curimg:get(i,j)
                    --check each reference point in turn - if the rgb values are comparable to the reference image then treat as background, otherwise treat as new and show that portion of the camera image
                    if math.abs(ref[xc][yc].r-tonumber(r))<thresh and math.abs(ref[xc][yc].g-tonumber(g))<thresh and math.abs(ref[xc][yc].b-tonumber(b))<thresh then
                        --matches the reference so blank it out
                        sprite(bgimg,i-xres/2,j-yres/2,xres,yres)
                        
                    else
                        
                    end
                end
            end
            
        end
        
        setContext()
        sprite(dispimg,WIDTH/2,HEIGHT/2,1280,720)
    end
    --camera button image
    sprite("Cargo Bot:Crate Goal Yellow",WIDTH*0.95,HEIGHT*0.05,0.1*WIDTH,0.1*HEIGHT)
    sprite("Cargo Bot:Record Solution Icon",WIDTH*0.95,HEIGHT*0.05,0.05*WIDTH,0.05*HEIGHT)
    collectgarbage()
    
end

function touched(t)
    
    if t.state==ENDED and t.x>0.9*WIDTH and t.y<0.1*HEIGHT then
        --take new reference image
        cameraSource(CAMERA_FRONT)
        refimg = image(CAMERA)
        --camera snapshot resolution from my instance is 1280,720
        --generate a series of checkpoints and store the pixel color for comparison
        ref={}
        local xc=0
        for i=1,1280,xres do
            local yc=0
            xc = xc + 1
            ref[xc]={}
            for j=1,720,yres do
                local red,green,blue,alpha= refimg:get(i,j)
                table.insert(ref[xc],{r=red,g=green,b=blue,a=alpha})
            end
        end
    end
end
8

Some modifications to recognise specific colours. Here’s a video

http://youtu.be/inYAra9zA_U

9

Wow! That’s really cool looking!

10

@Kolosso thanks - pretty pleased at how it’s turning out

11

That is awesome, what a great idea. A shader might help here

12

@yojimbo2000 I wondered that - I still haven’t made time to learn shaders.

Meanwhile here is the updated code which generates “fire” based on any orange object on screen. I’ve also modified to allow a background image. Could be the basis of one of these cheap trick pictures where you can superimpose yourself in a scene with the pyramids of Giza/Mount Rushmore/the moon landing/etc

-- Magic Mirror
supportedOrientations(LANDSCAPE_ANY)
displayMode(FULLSCREEN)
-- Use this function to perform your initial setup
function setup()
    ref={}
    cameraSource(CAMERA_FRONT)
    refimg = image(CAMERA)  --used as the background reference image
    curimg=image(CAMERA)  --used to store the current image from the camera
    hearts={}
    xres=4--the size of the horizontal blocks - should divide into 1280 to leave an integer
    yres=6  --the size of the vertical blocks - should divide into 720 to leave an integer
    thresh=25-- a threshold for detecting the difference between the background pixel and current pixel
    dispimg=image(1280,720)  -- used for the image to be displayed
    bgimg=image(xres,yres) -- create a small image for use as the background blocks
    setContext(bgimg)
    background(72, 50, 225, 255) -- doesn't matter what color as this will be set as the transparent mask to background
    setContext()
end

-- This function gets called once every frame
function draw()
    background(0)
    text("Tap in bottom right to set the background reference image",WIDTH/2,HEIGHT/2)
    sprite("SpaceCute:Background",WIDTH/2,HEIGHT/2,WIDTH,HEIGHT)
    if refimg~=nil then
        local curimg = image(CAMERA)
        local xc=0
        for i=1,1280,xres do
            local yc=0
            xc = xc + 1
            for j=1,720,yres do
                yc=yc+1
                local r,g,b,a= curimg:get(i,j)
                --check each reference point in turn - if the rgb values are comparable to the reference image then treat as background, otherwise treat as new and show that portion of the camera image
                if math.abs(ref[xc][yc].r-tonumber(r))<thresh and math.abs(ref[xc][yc].g-tonumber(g))<thresh and math.abs(ref[xc][yc].b-tonumber(b))<thresh then
                    -- set this block marker, m to zero (background)
                    ref[xc][yc].m=0
                else
                    -- set this block marker, m to 1 (active)
                    ref[xc][yc].m=1
                    --check if the block is a certain colour - in this case orange.  if so the mark m as 2 (fire activater)
                    if r>220 and g>30 and g<230 and b<100 then
                        ref[xc][yc].m=2
                        
                        if math.random(10)==1 then
                            table.insert(hearts,{x=i-128,y=j+24,fade=255,active=1,spd=2+math.random(10),off=math.random(10),size=10+math.random(15),wob=-5+math.random(10)})
                        end
                    end
                end
            end
        end
        
        --eliminate single pixel noise
        for i=2,#ref-1 do
            for j=2,#ref[i]-1 do
                if ref[i][j].m>0 and
                ref[i-1][j+1].m==0 and ref[i][j+1].m==0 and ref[i+1][j+1].m==0 and
                ref[i-1][j].m==0 and ref[i+1][j].m==0 and
                ref[i-1][j-1].m==0 and ref[i][j-1].m==0 and ref[i+1][j-1].m==0 then
                    
                    ref[i][j].m=0
                end
            end
        end
        
        setContext(dispimg)
        sprite(curimg,WIDTH/2+128,HEIGHT/2-24,1280,720)
        local xc=0
        for i=1,1280,xres do
            local yc=0
            xc = xc + 1
            for j=1,720,yres do
                yc=yc+1
                if ref[xc][yc].m==0 then
                    --mask away current block
                    blendMode( ZERO, ONE_MINUS_SRC_ALPHA )
                    sprite(bgimg,i+xres/2,j+yres/2,xres,yres)
                    blendMode(NORMAL)
                else
                    --leave current image portion to show through
                end
            end
        end
        setContext()
        sprite(dispimg,WIDTH/2,HEIGHT/2)
    end
    --camera button image
    sprite("Cargo Bot:Crate Goal Yellow",WIDTH*0.95,HEIGHT*0.05,0.1*WIDTH,0.1*HEIGHT)
    sprite("Cargo Bot:Record Solution Icon",WIDTH*0.95,HEIGHT*0.05,0.05*WIDTH,0.05*HEIGHT)
    
    for i,h in pairs(hearts) do
        tint(255,h.fade)
        if h.fade>200 then
            sprite("Cargo Bot:Star Filled",h.x,h.y,h.size)
        else
            sprite("Cargo Bot:Smoke Particle",h.x,h.y,h.size)
            
        end
        
        h.y = h.y + h.spd
        if h.fade<200 then h.y = h.y + h.spd end
        h.x = h.x + h.wob*math.sin(ElapsedTime+h.off)
        h.fade = h.fade - 5
        if h.fade<160 then h.fade = h.fade - 5 end
        if h.fade<0 then h.active=0 end
        
    end
    noTint()
    for i=#hearts,1,-1 do
        if hearts[i].active==0 then
            table.remove(hearts,i)
        end
    end
    collectgarbage()
end

function touched(t)
    
    if t.state==ENDED and t.x>0.9*WIDTH and t.y<0.1*HEIGHT then
        --take new reference image
        cameraSource(CAMERA_FRONT)
        refimg = image(CAMERA)
        --camera snapshot resolution from my instance is 1280,720
        --generate a series of checkpoints and store the pixel color for comparison
        
        local xc=0
        for i=1,1280,xres do
            local yc=0
            xc = xc + 1
            ref[xc]={}
            for j=1,720,yres do
                local red,green,blue,alpha= refimg:get(i,j)
                table.insert(ref[xc],{r=red,g=green,b=blue,a=alpha,m=0})
            end
        end
    end
end