• fullscreen
  • opticalflow.pde
  • // Optical Flow 2010/05/28
    // Hidetoshi Shimodaira shimo@is.titech.ac.jp 2010 GPL
    
    
    ///////////////////////////////////////////////
    // parameters for desktop pc (high performance)
    int wscreen=640;
    int hscreen=480;
    int gs=10; // grid step (pixels)
    float predsec=1.0; // prediction time (sec): larger for longer vector
    
    // parameters for laptop pc (low performance)
    //int wscreen=480;
    //int hscreen=360;
    //int gs=20; // grid step (pixels)
    //float predsec=1.0; // prediction time (sec): larger for longer vector
    
    ///////////////////////////////////////////////
    // use video
    import processing.video.*;
    Capture video;
    PFont font;
    color[] vline;
    MovieMaker movie;
    
    // capture parameters
    int fps=30;
    
    // use sound
    import ddf.minim.*;
    import ddf.minim.signals.*;
    Minim minim;
    AudioOutput audioout;
    SineWave sine;
    
    // grid parameters
    
    int as=gs*2;  // window size for averaging (-as,...,+as)
    int gw=wscreen/gs;
    int gh=hscreen/gs;
    int gs2=gs/2;
    float df=predsec*fps;
    
    // regression vectors
    float[] fx, fy, ft;
    int fm=3*9; // length of the vectors
    
    // regularization term for regression
    float fc=pow(10,8); // larger values for noisy video
    
    // smoothing parameters
    float wflow=0.1; // smaller value for longer smoothing
    
    // switch
    boolean flagseg=false; // segmentation of moving objects?
    boolean flagball=true; // playing ball game?
    boolean flagmirror=true; // mirroring image?
    boolean flagflow=true; // draw opticalflow vectors?
    boolean flagsound=true; // sound effect?
    boolean flagimage=true; // show video image ?
    boolean flagmovie=false; // saving movie?
    
    // internally used variables
    float ar,ag,ab; // used as return value of pixave
    float[] dtr, dtg, dtb; // differentiation by t (red,gree,blue)
    float[] dxr, dxg, dxb; // differentiation by x (red,gree,blue)
    float[] dyr, dyg, dyb; // differentiation by y (red,gree,blue)
    float[] par, pag, pab; // averaged grid values (red,gree,blue)
    float[] flowx, flowy; // computed optical flow
    float[] sflowx, sflowy; // slowly changing version of the flow
    int clockNow,clockPrev, clockDiff; // for timing check
    
    // playing ball parameters
    float ballpx=wscreen*0.5; // position x
    float ballpy=hscreen*0.5; // position y
    float ballvx=0.0; // velocity x
    float ballvy=0.0; // velocity y
    float ballgy=0.05; // gravitation
    float ballsz=30.0; // size
    float ballsz2=ballsz/2;
    float ballfv=0.8; // rebound factor
    float ballhv=50.0; // hit factor
    float ballvmax=10.0; // max velocity (pixel/frame)
    
    void setup(){
      // screen and video
      size(wscreen, hscreen, P2D);  
      video = new Capture(this, wscreen, hscreen, fps);
      // font
      font=createFont("Verdana",10);
      textFont(font);
      // draw
      rectMode(CENTER);
      ellipseMode(CENTER);
      // sound : not essential, only for ball movement
      minim = new Minim(this);
      audioout = minim.getLineOut(Minim.STEREO, 2048);
      sine = new SineWave(440, 0.5, audioout.sampleRate());
      sine.portamento(200);
      sine.setAmp(0.0);
      audioout.addSignal(sine);
    
      // arrays
      par = new float[gw*gh];
      pag = new float[gw*gh];
      pab = new float[gw*gh];
      dtr = new float[gw*gh];
      dtg = new float[gw*gh];
      dtb = new float[gw*gh];
      dxr = new float[gw*gh];
      dxg = new float[gw*gh];
      dxb = new float[gw*gh];
      dyr = new float[gw*gh];
      dyg = new float[gw*gh];
      dyb = new float[gw*gh];
      flowx = new float[gw*gh];
      flowy = new float[gw*gh];
      sflowx = new float[gw*gh];
      sflowy = new float[gw*gh];
      fx = new float[fm];
      fy = new float[fm];
      ft = new float[fm];
      vline = new color[wscreen];
    }
    
    
    // calculate average pixel value (r,g,b) for rectangle region
    void pixave(int x1, int y1, int x2, int y2) {
      float sumr,sumg,sumb;
      color pix;
      int r,g,b;
      int n;
    
      if(x1<0) x1=0;
      if(x2>=wscreen) x2=wscreen-1;
      if(y1<0) y1=0;
      if(y2>=hscreen) y2=hscreen-1;
    
      sumr=sumg=sumb=0.0;
      for(int y=y1; y<=y2; y++) {
        for(int i=wscreen*y+x1; i<=wscreen*y+x2; i++) {
          pix=video.pixels[i];
          b=pix & 0xFF; // blue
          pix = pix >> 8;
          g=pix & 0xFF; // green
          pix = pix >> 8;
          r=pix & 0xFF; // red
          // averaging the values
          sumr += r;
          sumg += g;
          sumb += b;
        }
      }
      n = (x2-x1+1)*(y2-y1+1); // number of pixels
      // the results are stored in static variables
      ar = sumr/n; 
      ag=sumg/n; 
      ab=sumb/n;
    }
    
    // extract values from 9 neighbour grids
    void getnext9(float x[], float y[], int i, int j) {
      y[j+0] = x[i+0];
      y[j+1] = x[i-1];
      y[j+2] = x[i+1];
      y[j+3] = x[i-gw];
      y[j+4] = x[i+gw];
      y[j+5] = x[i-gw-1];
      y[j+6] = x[i-gw+1];
      y[j+7] = x[i+gw-1];
      y[j+8] = x[i+gw+1];
    }
    
    // solve optical flow by least squares (regression analysis)
    void solveflow(int ig) {
      float xx, xy, yy, xt, yt;
      float a,u,v,w;
    
      // prepare covariances
      xx=xy=yy=xt=yt=0.0;
      for(int i=0;i<fm;i++) {
        xx += fx[i]*fx[i];
        xy += fx[i]*fy[i];
        yy += fy[i]*fy[i];
        xt += fx[i]*ft[i];
        yt += fy[i]*ft[i];
      }
    
      // least squares computation
      a = xx*yy - xy*xy + fc; // fc is for stable computation
      u = yy*xt - xy*yt; // x direction
      v = xx*yt - xy*xt; // y direction
    
      // write back
      flowx[ig] = -2*gs*u/a; // optical flow x (pixel per frame)
      flowy[ig] = -2*gs*v/a; // optical flow y (pixel per frame)
    }
    
    void draw() {
      if(video.available()){
        // video capture
        video.read();
    
        // clock in msec
        clockNow = millis();
        clockDiff = clockNow - clockPrev;
        clockPrev = clockNow;
    
        // mirror
        if(flagmirror) {
          for(int y=0;y<hscreen;y++) {
            int ig=y*wscreen;
            for(int x=0; x<wscreen; x++) 
              vline[x] = video.pixels[ig+x];
            for(int x=0; x<wscreen; x++) 
              video.pixels[ig+x]=vline[wscreen-1-x];
          }
        }
    
        // draw image
        if(flagimage) set(0,0,video);
        else background(0);
    
        // 1st sweep : differentiation by time
        for(int ix=0;ix<gw;ix++) {
          int x0=ix*gs+gs2;
          for(int iy=0;iy<gh;iy++) {
            int y0=iy*gs+gs2;
            int ig=iy*gw+ix;
            // compute average pixel at (x0,y0)
            pixave(x0-as,y0-as,x0+as,y0+as);
            // compute time difference
            dtr[ig] = ar-par[ig]; // red
            dtg[ig] = ag-pag[ig]; // green
            dtb[ig] = ab-pab[ig]; // blue
            // save the pixel
            par[ig]=ar;
            pag[ig]=ag;
            pab[ig]=ab;
          }
        }
    
        // 2nd sweep : differentiations by x and y
        for(int ix=1;ix<gw-1;ix++) {
          for(int iy=1;iy<gh-1;iy++) {
            int ig=iy*gw+ix;
            // compute x difference
            dxr[ig] = par[ig+1]-par[ig-1]; // red
            dxg[ig] = pag[ig+1]-pag[ig-1]; // green
            dxb[ig] = pab[ig+1]-pab[ig-1]; // blue
            // compute y difference
            dyr[ig] = par[ig+gw]-par[ig-gw]; // red
            dyg[ig] = pag[ig+gw]-pag[ig-gw]; // green
            dyb[ig] = pab[ig+gw]-pab[ig-gw]; // blue
          }
        }
    
        // 3rd sweep : solving optical flow
        for(int ix=1;ix<gw-1;ix++) {
          int x0=ix*gs+gs2;
          for(int iy=1;iy<gh-1;iy++) {
            int y0=iy*gs+gs2;
            int ig=iy*gw+ix;
    
            // prepare vectors fx, fy, ft
            getnext9(dxr,fx,ig,0); // dx red
            getnext9(dxg,fx,ig,9); // dx green
            getnext9(dxb,fx,ig,18);// dx blue
            getnext9(dyr,fy,ig,0); // dy red
            getnext9(dyg,fy,ig,9); // dy green
            getnext9(dyb,fy,ig,18);// dy blue
            getnext9(dtr,ft,ig,0); // dt red
            getnext9(dtg,ft,ig,9); // dt green
            getnext9(dtb,ft,ig,18);// dt blue
    
            // solve for (flowx, flowy) such that
            // fx flowx + fy flowy + ft = 0
            solveflow(ig);
    
            // smoothing
            sflowx[ig]+=(flowx[ig]-sflowx[ig])*wflow;
            sflowy[ig]+=(flowy[ig]-sflowy[ig])*wflow;
          }
        }
    
    
        // 4th sweep : draw the flow
        if(flagseg) {
          noStroke();
          fill(0);
          for(int ix=0;ix<gw;ix++) {
            int x0=ix*gs+gs2;
            for(int iy=0;iy<gh;iy++) {
              int y0=iy*gs+gs2;
              int ig=iy*gw+ix;
    
              float u=df*sflowx[ig];
              float v=df*sflowy[ig];
    
              float a=sqrt(u*u+v*v);
              if(a<2.0) rect(x0,y0,gs,gs);
            }
          }
        }
    
        // 5th sweep : draw the flow
        if(flagflow) {
          for(int ix=0;ix<gw;ix++) {
            int x0=ix*gs+gs2;
            for(int iy=0;iy<gh;iy++) {
              int y0=iy*gs+gs2;
              int ig=iy*gw+ix;
    
              float u=df*sflowx[ig];
              float v=df*sflowy[ig];
    
              // draw the line segments for optical flow
              float a=sqrt(u*u+v*v);
              if(a>=2.0) { // draw only if the length >=2.0
                float r=0.5*(1.0+u/(a+0.1));
                float g=0.5*(1.0+v/(a+0.1));
                float b=0.5*(2.0-(r+g));
                stroke(255*r,255*g,255*b);
                line(x0,y0,x0+u,y0+v);
              }
            }
          }
        }
        ///////////////////////////////////////////////////////
        // ball movement : not essential for optical flow
        if(flagball) {
          // updatating position and velocity
          ballpx += ballvx;
          ballpy += ballvy;
          ballvy += ballgy;
    
          // reflecton
          if(ballpx<ballsz2) {
            ballpx=ballsz2;
            ballvx=-ballvx*ballfv;
          } 
          else if(ballpx>wscreen-ballsz2) {
            ballpx=wscreen-ballsz2;
            ballvx=-ballvx*ballfv;
          }
          if(ballpy<ballsz2) {
            ballpy=ballsz2;
            ballvy=-ballvy*ballfv;
          } 
          else if(ballpy>hscreen-ballsz2) {
            ballpy=hscreen-ballsz2;
            ballvy=-ballvy*ballfv;
          }
    
          // draw the ball
          fill(50,200,200);
          stroke(0,100,100);
          ellipse(ballpx,ballpy,ballsz,ballsz);
    
          // find the grid 
          int ix= round((ballpx-gs2)/gs);
          int iy= round((ballpy-gs2)/gs);
          if(ix<1) ix=1;
          else if(ix>gw-2) ix=gw-2;
          if(iy<1) iy=1;
          else if(iy>gh-2) iy=gh-2;
          int ig=iy*gw+ix;
    
          // hit the ball by your movement
          float u=sflowx[ig];
          float v=sflowy[ig];
          float a=sqrt(u*u+v*v);
          u=u/a; 
          v=v/a;
          if(a>=2.0) a=2.0;
          if(a>=0.3) {
            ballvx += ballhv*a*u;
            ballvy += ballhv*a*v;
            float b=sqrt(ballvx*ballvx+ballvy*ballvy);
            if(b>ballvmax) {
              ballvx = ballvmax*ballvx/b;
              ballvy = ballvmax*ballvy/b;
            }
          }
    
          // sound
          float pan = map(ballpx, 0, wscreen, -1, 1);
          float freq = map(ballpy, 0, hscreen, 1500, 60);
          float vol = map(a,0.0,0.8,0.0,1.0);
          if(!flagsound) vol=0.0;
          sine.setPan(pan);
          sine.setFreq(freq);
          sine.setAmp(vol);
        }
      }
    
      ///////////////////////////////////////////////////
      // recording movie 
      if(flagmovie) movie.addFrame();
    
      //  print information (not shown in the movie)
      fill(255,0,0);
      text(clockDiff,10,10); // time (msec) for this frame
      if(flagmovie) text("rec", 40,10);
    
    }
    
    void stopGame(){
      minim.stop();
      super.stop();
    }
    
    void keyPressed(){
      if(key=='c') video.settings();
      else if(key=='w') flagseg=!flagseg; // segmentation on/off
      else if(key=='s') flagsound=!flagsound; //  sound on/off
      else if(key=='e') stopGame(); // quit
      else if(key=='m') flagmirror=!flagmirror; // mirror on/off
      else if(key=='i') flagimage=!flagimage; // show video on/off
      else if(key=='f') flagflow=!flagflow; // show opticalflow on/off
      else if(key=='q') {
        flagmovie=!flagmovie;
        if(flagmovie) { // start recording movie
          movie = new MovieMaker(this, width, height, "mymovie.mov",
          fps);
        } 
        else { // stop recording movie
          movie.finish();
        }
      }
      else if(key==' ') { // kick the ball
        ballvy = -3.0;
      }
      else if(key=='b') { // show the ball on/off
        flagball=!flagball;
        if(flagball) { // put the ball at the center
          ballpx=wscreen*0.5;
          ballpy=hscreen*0.5;
          ballvx=ballvy=0.0;
        }
      }
    }
    
    
    
    
    
    

    code

    tweaks (1)

    about this sketch

    This sketch is running as Java applet, exported from Processing.

    license

    advertisement

    Hidetoshi Shimodaira

    Optical Flow

    Add to Faves Me Likey@! 27
    You must login/register to add this sketch to your favorites.

    Computing motion of objects in the capture video image. The motion vector (optical flow) is drawn at each grid point (default: 10 pixels grid). You can hit a ball in the video by moving yourself.

    It uses video capture, so it does not work on the web applet. Now I am uploading the sample video to youtube account of "myhshimodaira".
    adam schmidt
    16 Jun 2010
    very nice i just tried it out
    This is amazing. Very well executed. I see references to sound. Does this have an audio driven behavior as well?

    I'd also like to invite you to join my AR group if you like. We love this stuff. http://meetup.com/augmentedreality/
    Hello Hidetoshi, WONDERFUL project! Congratulations!

    I loved the color vectors and who they seams to catch the movement!
    Really nice effect!!

    I hope you don´t mind, I used part of the code to mix it with a flow field of ants ( http://www.openprocessing.org/visuals/?visualID=10534 ). Really looking forward to used it on some other projects!
    At the beginning it was hard to manage so many variables with so complicated names. But after some time I get the concept and how it works! sweet!
    Nice job!

    Thanks!
    Hi Patricio. I have just run your code on my Windows pc. It is fun to see bugs walking around! Thank you for your using the optical flow code.
    Felix Woitzel
    23 Jun 2010
    I have finally managed to get it running on my windows machine too (finding my way through the wiki to the WinVDIG 1.0.1 hint ^^).

    I didn't look into the code so far, but I think I will reuse parts of it. It would be great if you could point me to some theoretical works on optical flow solving. I also think of a static background / moving foreground separation mask. Dunno how much is possible from what I've seen from other OpenCV demos. I've contacted you on facebook and maybe we could chat a round. ;)

    two thumbs up for this sketch!
    Thanks Felix. The theory is found at wikipedia, which I learned twenty years ago when I was an undergrad. Recently I met processing and suddenly remembered the optical flow. For implementing it, I arranged the basic theory a little bit; I used r,g,b levels independently for increasing the information instead of treating only the gray intensity. For simple implementation, I used the simple 9 neighbour grid points instead of more sophisticated spatial constraints. For stabilizing the result, I used a ridge regression technique. I made a quick coding, about three hours of try and error, and I myself was surprised to see the code worked so well. I did not elaborate further for cleaning up the code, so I apologize for the program being hard-to-read. I am looking forward to seeing programs where the optical flow effect might be used. Have fun!
    Rebecca Vickers
    24 Jan 2011
    excellent! looks great!
    I hope you don't mind if I borrow some of your code? I would like to ask you if you have any suggestions for me- I'm trying to create a project where the users motion is detected and drawn on the screen (like in your project). But if possible, I like how the vectors show the direction of movement but ideally I would really like them to be displayed in the color of the corresponding pixel in the camera feed. So that if, for example, someone has a red shirt on and moves in front of the camera the vectors would show up red where they had moved. Does that makes sense? Do you think I can do that?Any suggestions or points in the right direction would be highly appreciated. I'm very new at all of this. :)
    thank you for sharing your work! so very helpful!
    chris zeppieri
    29 Mar 2011
    looks awesome, im playing around with the code, and am trying to get it to leave the drawn vectors in their position. basically, making an image as your move your face that doenst delete itself, but i cant figure out how to get them to stick around. Any suggestions?
    I'm not sure how to keep them in place without drawing them in every draw loop. After a few seconds you'd have thousands of lines. I was able to make them persist and fade over time by using image() instead of set(), setting an opacity with tint() and adding a fourth value to the stroke() to give it transparency. That way, every draw (30 per second) the images and lines blend instead of overwrite. You can use the second value in tint() which controls the opacity of the video to adjust the speed that the lines fade. More opaque video means faster decay for the lines and less opaque means a slower decay.

    Try it:

    replace line 221 with:
    tint(255,16); if(flagimage) image(video,0,0);
    replace line 323 with:
    stroke(255*r,255*g,255*b,64);
    If you are on a fast machine you can add this to the setup anywhere after line 88:
    frameRate(fps); smooth();
    chrs
    21 Jun 2011
    thanks for this very interesting demonstration!
    i'm also playing around with your code. my goal is it to get all the motion in colored pixels instead of the lines. some kind of rainbow-motion-trail:-) it should look like this http://vimeo.com/24336775, but instead of direction-colors i thought about colors depending on the motion-speed.. fast=red, slow=green. i thought i use HSB instead of RGB to do something like hue=norm(speed, 0.1, 360) to get the faster pixels more stronger.. well, it didn't worked out till now. i guess i'm doing something wrong.. what do you think?
    thanks and greets, chris
    Audrey Green
    1 Jul 2011
    OMG this is the best project so far. I have just recently started learning processing, and this have inspired me to continue learning. I will be also playing with this code. Thank you so much for your contribution !
    Kirilly Barnett
    13 Sep 2011
    Hello Hidetoshi,
    Thank you for sharing such a wonderful code! I have been experimenting with it, and wonder if it's possible to replace the webcam input with a movie (using the processing video library)?
    If you think it's possible please let me know and I will investigate it further.
    Thanks for your time.
    Jack Kalish
    14 Oct 2012
    Great work! Thank you for sharing! I have ported your code into a class and am using it for motion detection to trigger photo-capture.
    ang
    20 Dec 2012
    have been trying most of the day to get this code working. can anyone explain why or what this error is...

    can not find class or type 'MovieMaker'

    have tried this on both windows and mac
    ang,

    You will get that error when running this code with Processing 2.0x
    The code runs in Processing 1.x to make t run in 2.0x you have to comment out the functions related to moviemaker. you also have to add video.start() to start video capture. Like this:

    video = new Capture(this, wscreen, hscreen, fps);
    video.start();

    The reason is that Processing 2.0x uses the gstreamer library instead of quicktime. This is a good move for Processing, but it means anything written to use video before 2.0 will have to be edited to include the start() and remove moviemaker. For now.

    I hope that helps,

    Sid
    ang,

    I "tweaked" the code to work in Processing 2.0x
    You can download it here:
    http://www.openprocessing.org/sketch/84287
    Brandon
    4 Jul 2013
    Running into some problems tweeking the code

    I'm trying to switch the feed from a live camera capture to a prerecorded mov file.
    This may not be the right place to be looking for help on this but I'm getting different errors through the process. Currently stuck on 'ArrayIndexOutOfBoundsException:0'

    has anyone made this kind of a switch on a sketch before?
    You need to login/register to comment.