Develop Reference

Neural Network returning NaN as output

I am trying to write my first neural network to play the game connect four.
Im using Java and deeplearning4j.
I tried to implement a genetic algorithm, but when i train the network for a while, the outputs of the network jump to NaN and I am unable to tell where I messed up so badly for this to happen..
I will post all 3 classes below, where Game is the game logic and rules, VGFrame the UI and Main all the nn stuff.
I have a pool of 35 neural networks and each iteration i let the best 5 live and breed and randomize the newly created ones a little.
To evaluate the networks I let them battle each other and give points to the winner and points for loosing later.
Since I penalize putting a stone into a column thats already full I expected the neural networks at least to be able to play the game by the rules after a while but they cant do this.
I googled the NaN problem and it seems to be an expoding gradient problem, but from my understanding this shouldn't occur in a genetic algorithm?
Any ideas where I could look for the error or whats generally wrong with my implementation?
Main
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.Random;
import org.deeplearning4j.nn.api.OptimizationAlgorithm;
import org.deeplearning4j.nn.conf.MultiLayerConfiguration;
import org.deeplearning4j.nn.conf.NeuralNetConfiguration;
import org.deeplearning4j.nn.conf.layers.DenseLayer;
import org.deeplearning4j.nn.conf.layers.OutputLayer;
import org.deeplearning4j.nn.multilayer.MultiLayerNetwork;
import org.deeplearning4j.nn.weights.WeightInit;
import org.nd4j.linalg.activations.Activation;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.learning.config.Nesterovs;
public class Main {
final int numRows = 7;
final int numColums = 6;
final int randSeed = 123;
MultiLayerNetwork[] models;
static Random random = new Random();
private static final Logger log = LoggerFactory.getLogger(Main.class);
final float learningRate = .8f;
int batchSize = 64; // Test batch size
int nEpochs = 1; // Number of training epochs
// --
public static Main current;
Game mainGame = new Game();
public static void main(String[] args) {
current = new Main();
current.frame = new VGFrame();
current.loadWeights();
}
private VGFrame frame;
private final double mutationChance = .05;
public Main() {
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER)
.activation(Activation.RELU).seed(randSeed)
.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(new Nesterovs(0.1, 0.9))
.list()
.layer(new DenseLayer.Builder().nIn(42).nOut(30).activation(Activation.RELU)
.weightInit(WeightInit.XAVIER).build())
.layer(new DenseLayer.Builder().nIn(30).nOut(15).activation(Activation.RELU)
.weightInit(WeightInit.XAVIER).build())
.layer(new OutputLayer.Builder(LossFunction.NEGATIVELOGLIKELIHOOD).nIn(15).nOut(7)
.activation(Activation.SOFTMAX).weightInit(WeightInit.XAVIER).build())
.build();
models = new MultiLayerNetwork[35];
for (int i = 0; i < models.length; i++) {
models[i] = new MultiLayerNetwork(conf);
models[i].init();
}
}
public void addChip(int i, boolean b) {
if (mainGame.gameState == 0)
mainGame.addChip(i, b);
if (mainGame.gameState == 0) {
float[] f = Main.rowsToInput(mainGame.rows);
INDArray input = Nd4j.create(f);
INDArray output = models[0].output(input);
for (int i1 = 0; i1 < 7; i1++) {
System.out.println(i1 + ": " + output.getDouble(i1));
}
System.out.println("----------------");
mainGame.addChip(Main.getHighestOutput(output), false);
}
getFrame().paint(getFrame().getGraphics());
}
public void newGame() {
mainGame = new Game();
getFrame().paint(getFrame().getGraphics());
}
public void startTraining(int iterations) {
// --------------------------
for (int gameNumber = 0; gameNumber < iterations; gameNumber++) {
System.out.println("Iteration " + gameNumber + " of " + iterations);
float[] evaluation = new float[models.length];
for (int i = 0; i < models.length; i++) {
for (int j = 0; j < models.length; j++) {
if (i != j) {
Game g = new Game();
g.playFullGame(models[i], models[j]);
if (g.gameState == 1) {
evaluation[i] += 45;
evaluation[j] += g.turnNumber;
}
if (g.gameState == 2) {
evaluation[j] += 45;
evaluation[i] += g.turnNumber;
}
}
}
}
float[] evaluationSorted = evaluation.clone();
Arrays.sort(evaluationSorted);
// keep the best 4
int n1 = 0, n2 = 0, n3 = 0, n4 = 0, n5 = 0;
for (int i = 0; i < evaluation.length; i++) {
if (evaluation[i] == evaluationSorted[evaluationSorted.length - 1])
n1 = i;
if (evaluation[i] == evaluationSorted[evaluationSorted.length - 2])
n2 = i;
if (evaluation[i] == evaluationSorted[evaluationSorted.length - 3])
n3 = i;
if (evaluation[i] == evaluationSorted[evaluationSorted.length - 4])
n4 = i;
if (evaluation[i] == evaluationSorted[evaluationSorted.length - 5])
n5 = i;
}
models[0] = models[n1];
models[1] = models[n2];
models[2] = models[n3];
models[3] = models[n4];
models[4] = models[n5];
for (int i = 3; i < evaluationSorted.length; i++) {
// random parent/keep w8ts
double r = Math.random();
if (r > .3) {
models[i] = models[random.nextInt(3)].clone();
} else if (r > .1) {
models[i].setParams(breed(models[random.nextInt(3)], models[random.nextInt(3)]));
}
// Mutate
INDArray params = models[i].params();
models[i].setParams(mutate(params));
}
}
}
private INDArray mutate(INDArray params) {
double[] d = params.toDoubleVector();
for (int i = 0; i < d.length; i++) {
if (Math.random() < mutationChance)
d[i] += (Math.random() - .5) * learningRate;
}
return Nd4j.create(d);
}
private INDArray breed(MultiLayerNetwork m1, MultiLayerNetwork m2) {
double[] d = m1.params().toDoubleVector();
double[] d2 = m2.params().toDoubleVector();
for (int i = 0; i < d.length; i++) {
if (Math.random() < .5)
d[i] += d2[i];
}
return Nd4j.create(d);
}
static int getHighestOutput(INDArray output) {
int x = 0;
for (int i = 0; i < 7; i++) {
if (output.getDouble(i) > output.getDouble(x))
x = i;
}
return x;
}
static float[] rowsToInput(byte[][] rows) {
float[] f = new float[7 * 6];
for (int i = 0; i < 6; i++) {
for (int j = 0; j < 7; j++) {
// f[j + i * 7] = rows[j][i] / 2f;
f[j + i * 7] = (rows[j][i] == 0 ? .5f : rows[j][i] == 1 ? 0f : 1f);
}
}
return f;
}
public void saveWeights() {
log.info("Saving model");
for (int i = 0; i < models.length; i++) {
File resourcesDirectory = new File("src/resources/model" + i);
try {
models[i].save(resourcesDirectory, true);
} catch (IOException e) {
e.printStackTrace();
}
}
}
public void loadWeights() {
if (new File("src/resources/model0").exists()) {
for (int i = 0; i < models.length; i++) {
File resourcesDirectory = new File("src/resources/model" + i);
try {
models[i] = MultiLayerNetwork.load(resourcesDirectory, true);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
System.out.println("col: " + models[0].params().shapeInfoToString());
}
public VGFrame getFrame() {
return frame;
}
}
VGFrame
import java.awt.Color;
import java.awt.Graphics;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import javax.swing.BorderFactory;
import javax.swing.JButton;
import javax.swing.JFrame;
import javax.swing.JPanel;
import javax.swing.JTextField;
public class VGFrame extends JFrame {
JTextField iterations;
/**
*
*/
private static final long serialVersionUID = 1L;
public VGFrame() {
super("Vier Gewinnt");
this.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
this.setSize(1300, 800);
this.setVisible(true);
JPanel panelGame = new JPanel();
panelGame.setBorder(BorderFactory.createLineBorder(Color.black, 2));
this.add(panelGame);
var handler = new Handler();
var menuHandler = new MenuHandler();
JButton b1 = new JButton("1");
JButton b2 = new JButton("2");
JButton b3 = new JButton("3");
JButton b4 = new JButton("4");
JButton b5 = new JButton("5");
JButton b6 = new JButton("6");
JButton b7 = new JButton("7");
b1.addActionListener(handler);
b2.addActionListener(handler);
b3.addActionListener(handler);
b4.addActionListener(handler);
b5.addActionListener(handler);
b6.addActionListener(handler);
b7.addActionListener(handler);
panelGame.add(b1);
panelGame.add(b2);
panelGame.add(b3);
panelGame.add(b4);
panelGame.add(b5);
panelGame.add(b6);
panelGame.add(b7);
JButton buttonTrain = new JButton("Train");
JButton buttonNewGame = new JButton("New Game");
JButton buttonSave = new JButton("Save Weights");
JButton buttonLoad = new JButton("Load Weights");
iterations = new JTextField("1000");
buttonTrain.addActionListener(menuHandler);
buttonNewGame.addActionListener(menuHandler);
buttonSave.addActionListener(menuHandler);
buttonLoad.addActionListener(menuHandler);
iterations.addActionListener(menuHandler);
panelGame.add(iterations);
panelGame.add(buttonTrain);
panelGame.add(buttonNewGame);
panelGame.add(buttonSave);
panelGame.add(buttonLoad);
this.validate();
}
#Override
public void paint(Graphics g) {
super.paint(g);
if (Main.current.mainGame.rows == null)
return;
var rows = Main.current.mainGame.rows;
for (int i = 0; i < rows.length; i++) {
for (int j = 0; j < rows[0].length; j++) {
if (rows[i][j] == 0)
break;
g.setColor((rows[i][j] == 1 ? Color.yellow : Color.red));
g.fillOval(80 + 110 * i, 650 - 110 * j, 100, 100);
}
}
}
public void update() {
}
}
class Handler implements ActionListener {
#Override
public void actionPerformed(ActionEvent event) {
if (Main.current.mainGame.playersTurn)
Main.current.addChip(Integer.parseInt(event.getActionCommand()) - 1, true);
}
}
class MenuHandler implements ActionListener {
#Override
public void actionPerformed(ActionEvent event) {
switch (event.getActionCommand()) {
case "New Game":
Main.current.newGame();
break;
case "Train":
Main.current.startTraining(Integer.parseInt(Main.current.getFrame().iterations.getText()));
break;
case "Save Weights":
Main.current.saveWeights();
break;
case "Load Weights":
Main.current.loadWeights();
break;
}
}
}
Game
import org.deeplearning4j.nn.multilayer.MultiLayerNetwork;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.factory.Nd4j;
public class Game {
int turnNumber = 0;
byte[][] rows = new byte[7][6];
boolean playersTurn = true;
int gameState = 0; // 0:running, 1:Player1, 2:Player2, 3:Draw
public boolean isRunning() {
return this.gameState == 0;
}
public void addChip(int x, boolean player1) {
turnNumber++;
byte b = nextRow(x);
if (b == 6) {
gameState = player1 ? 2 : 1;
return;
}
rows[x][b] = (byte) (player1 ? 1 : 2);
gameState = checkWinner(x, b);
}
private byte nextRow(int x) {
for (byte i = 0; i < rows[x].length; i++) {
if (rows[x][i] == 0)
return i;
}
return 6;
}
// 0 continue, 1 Player won, 2 ai won, 3 Draw
private int checkWinner(int x, int y) {
int color = rows[x][y];
// Vertikal
if (getCount(x, y, 1, 0) + getCount(x, y, -1, 0) >= 3)
return rows[x][y];
// Horizontal
if (getCount(x, y, 0, 1) + getCount(x, y, 0, -1) >= 3)
return rows[x][y];
// Diagonal1
if (getCount(x, y, 1, 1) + getCount(x, y, -1, -1) >= 3)
return rows[x][y];
// Diagonal2
if (getCount(x, y, -1, 1) + getCount(x, y, 1, -1) >= 3)
return rows[x][y];
for (byte[] bs : rows) {
for (byte s : bs) {
if (s == 0)
return 0;
}
}
return 3; // Draw
}
private int getCount(int x, int y, int dirX, int dirY) {
int color = rows[x][y];
int count = 0;
while (true) {
x += dirX;
y += dirY;
if (x < 0 | x > 6 | y < 0 | y > 5)
break;
if (color != rows[x][y])
break;
count++;
}
return count;
}
public void playFullGame(MultiLayerNetwork m1, MultiLayerNetwork m2) {
boolean player1 = true;
while (this.gameState == 0) {
float[] f = Main.rowsToInput(this.rows);
INDArray input = Nd4j.create(f);
this.addChip(Main.getHighestOutput(player1 ? m1.output(input) : m2.output(input)), player1);
player1 = !player1;
}
}
}

With a quick look, and based on the analysis of your multiplier variants, it seems like the NaN is produced by an arithmetic underflow, caused by your gradients being too small (too close to absolute 0).
This is the most suspicious part of the code:
f[j + i * 7] = (rows[j][i] == 0 ? .5f : rows[j][i] == 1 ? 0f : 1f);
If rows[j][i] == 1 then 0f is stored. I don't know how this is managed by the neural network (or even java), but mathematically speaking, a finite-sized float cannot include zero.
Even if your code would alter the 0f with some extra salt, those array values' resultants would have some risk of becoming too close to zero. Due to limited precision when representing real numbers, values very close to zero can not be represented, hence the NaN.
These values have a very friendly name: subnormal numbers.
Any non-zero number with magnitude smaller than the smallest normal
number is subnormal.
IEEE_754
As with IEEE 754-1985, The standard recommends 0 for signaling NaNs, 1 for quiet NaNs, so that a signaling NaNs can be quieted by changing only this bit to 1, while the reverse could yield the encoding of an infinity.
Above's text is important here: according to the standard, you are actually specifying a NaN with any 0f value stored.
Even if the name is misleading, Float.MIN_VALUE is a positive value,higher than 0:
The real minimum float value is, in fact: -Float.MAX_VALUE.
Is floating point math subnormal?
Normalizing the gradients
If you check the issue is only because of the 0f values, you could just alter them for other values that represent something similar; Float.MIN_VALUE, Float.MIN_NORMAL, and so on. Something like this, also in other possible parts of the code where this scenario could happen. Take these just as examples, and play with these ranges:
rows[j][i] == 1 ? Float.MIN_VALUE : 1f;
rows[j][i] == 1 ? Float.MIN_NORMAL : Float.MAX_VALUE/2;
rows[j][i] == 1 ? -Float.MAX_VALUE/2 : Float.MAX_VALUE/2;
Even so, this could also lead to a NaN, based on how these values are altered.
If so, you should normalize the values. You could try applying a GradientNormalizer for this. In your network initialization, something like this should be defined, for each layer(or for those who are problematic):
new NeuralNetConfiguration
.Builder()
.weightInit(WeightInit.XAVIER)
(...)
.layer(new DenseLayer.Builder().nIn(42).nOut(30).activation(Activation.RELU)
.weightInit(WeightInit.XAVIER)
.gradientNormalization(GradientNormalization.RenormalizeL2PerLayer) //this
.build())
(...)
There are different normalizers, so choose which one fits your schema best, and which layers should include one. The options are:
GradientNormalization
RenormalizeL2PerLayer
Rescale gradients by dividing by the L2 norm
of all gradients for the layer.
RenormalizeL2PerParamType
Rescale gradients by dividing by the L2
norm of the gradients, separately for each type of parameter within
the layer. This differs from RenormalizeL2PerLayer in that here, each
parameter type (weight, bias etc) is normalized separately. For
example, in a MLP/FeedForward network (where G is the gradient
vector), the output is as follows:
GOut_weight = G_weight / l2(G_weight) GOut_bias = G_bias / l2(G_bias)
ClipElementWiseAbsoluteValue
Clip the gradients on a per-element
basis. For each gradient g, set g <- sign(g) max(maxAllowedValue,|g|).
i.e., if a parameter gradient has absolute value greater than the
threshold, truncate it. For example, if threshold = 5, then values in
range -5<g<5 are unmodified; values <-5 are set to -5; values >5 are
set to 5.
ClipL2PerLayer
Conditional renormalization. Somewhat similar to
RenormalizeL2PerLayer, this strategy scales the gradients if and only
if the L2 norm of the gradients (for entire layer) exceeds a specified
threshold. Specifically, if G is gradient vector for the layer, then:
GOut = G if l2Norm(G) < threshold (i.e., no change) GOut =
threshold * G / l2Norm(G)
ClipL2PerParamType
Conditional renormalization. Very
similar to ClipL2PerLayer, however instead of clipping per layer, do
clipping on each parameter type separately. For example in a recurrent
neural network, input weight gradients, recurrent weight gradients and
bias gradient are all clipped separately.
Here you can find a complete example of the application of these GradientNormalizers.

I think I finally figured it out. I was trying to visualize the network using deeplearning4j-ui, but got some incompatible versions errors. After changing versions I got a new error, stating the networks input is expecting a 2d array and I found on the internet that this is expected across all versions.
So i changed
float[] f = new float[7 * 6];
Nd4j.create(f);
to
float[][] f = new float[1][7 * 6];
Nd4j.createFromArray(f);
And the NaN values finally disappeared. #aran So I guess assuming incorrect inputs was definitly the right direction. Thank you so much for your help :)

Detecting when a PNG image has been "wiped transparent" in Processing?

i'm trying to create a game using Kinect where you have to use your hand movements to wipe away an image to make it disappear revealing another image beneath it within 30 seconds. Now I have already done the code for the loosing condition where if you do not wipe away the image under 30 seconds, the loosing screen will pop up.
However, I am not sure how to code the part to detect when the entire PNG image has been "wiped away". Does this involve using get()? I am not sure how to approach this.
Imagine there are 2 Pimages moondirt.png and moonsurface.png
The Kinect controls the wiping and making Pimage moondirt.png transparent to reveal moonsurface.png
void kinect() {
//----------draw kinect------------
// Draw moon surface
image(moonSurface, 0, 0, width, height);
// Draw the moon dirt
image(moonDirt, 0, 0, width, height);
// Threshold the depth image
int[] rawDepth = kinect.getRawDepth();
for (int i=0; i < rawDepth.length; i++) {
if (rawDepth[i] >= minDepth && rawDepth[i] <= maxDepth) {
depthImg.pixels[i] = color(255);
maskingImg.pixels[i] = color(255);
} else {
depthImg.pixels[i] = color(0);
}
}
//moonDirt.resize(640, 480); //(640, 480);
moonDirt.loadPixels();
for (int i=0; i < rawDepth.length; i++) {
if ( maskingImg.pixels[i] == color(255) ) {
moonDirt.pixels[i] = color( 0, 0, 0, 0 );
}
}
moonDirt.updatePixels();
image(moonDirt, 0, 0, width, height);
color c = moonDirt.get(width, height);
updatePixels();
//--------timer-----
if (countDownTimer.complete() == true){
if (timeLeft > 1 ) {
timeLeft--;
countDownTimer.start();
} else {
state = 4;
redraw();
}
}
//show countDown TIMER
String s = "Time Left: " + timeLeft;
textAlign(CENTER);
textSize(30);
fill(255,0,0);
text(s, 380, 320);
}
//timer
class Timer {
int startTime;
int interval;
Timer(int timeInterval) {
interval = timeInterval;
}
void start() {
startTime = millis();
}
boolean complete() {
int elapsedTime = millis() - startTime;
if (elapsedTime > interval) {
return true;
}else {
return false;
}
}
}

I see the confusion in this section:
moonDirt.loadPixels();
for (int i=0; i < rawDepth.length; i++) {
if ( maskingImg.pixels[i] == color(255) ) {
moonDirt.pixels[i] = color( 0, 0, 0, 0 );
}
}
moonDirt.updatePixels();
image(moonDirt, 0, 0, width, height);
color c = moonDirt.get(width, height);
You are already using pixels[] which is more efficient than get() which is great.
Don't forget to call updatePixels() when you're done. You already do that for moonDirt, but not for maskingImg
If you want to find out if an image has been cleared (where clear means transparent black (color(0,0,0,0)) in this case).
It looks like you're already familiar with functions that take parameters and return values. The count function will need to:
take 2 arguments: the image to process and the colour to check and count
return the total count
iterate through all pixels: if any pixels match the 2nd argument, the total count increments
Something like this:
/**
* countPixels - counts pixels of of a certain colour within an image
* #param image - the PImage to loop through
* #param colorToCount - the colour to count pixels present in the image
* return int - the number of found pixels (between 0 and image.pixels.length)
*/
int countPixels(PImage image,color colorToCount){
// initial transparent black pixel count
int count = 0;
// make pixels[] available
image.loadPixels();
// for each pixel
for(int i = 0 ; i < image.pixels.length; i++){
// check if it's transparent black
if(image.pixels[i] == colorToCount){
// if so, increment the counter
count++;
}
}
// finally return the count
return count;
}
Within your code you could use it like so:
...
// Threshold the depth image
int[] rawDepth = kinect.getRawDepth();
for (int i=0; i < rawDepth.length; i++) {
if (rawDepth[i] >= minDepth && rawDepth[i] <= maxDepth) {
depthImg.pixels[i] = color(255);
maskingImg.pixels[i] = color(255);
} else {
depthImg.pixels[i] = color(0);
}
}
maskingImg.updatePixels();
//moonDirt.resize(640, 480); //(640, 480);
moonDirt.loadPixels();
for (int i=0; i < rawDepth.length; i++) {
if ( maskingImg.pixels[i] == color(255) ) {
moonDirt.pixels[i] = color( 0, 0, 0, 0 );
}
}
moonDirt.updatePixels();
image(moonDirt, 0, 0, width, height);
int leftToReveal = moonDirt.pixels.length;
int revealedPixels = countPixels(moonDirt,color(0,0,0,0));
int percentageClear = round(((float)revealedPixels / leftToReveal) * 100);
println("revealed " + revealedPixels + " of " + leftToReveal + " pixels -> ~" + percentageClear + "% cleared");
...
You have the option to set the condition for all pixels to be cleared or a ratio/percentage (e.g. if more 90% is clear, that's good enough) to then change the game state accordingly.

BreakOut game: Part of the stones constructor isn't updating

I currently work on a BreakOut game and I am almost finished. The only problem I currently have is that the color of the stones wont change, when hit by the ball.
The color of a stone is defined by its type (1-3) and whenever a stone is hit, its type is reduced by 1. I know a stone's type is succesfully reduced when hit, cause it disappears when the type turns into 0.
This is the constructor of the stones class:
public Stone(int type, Position pos) {
this.pos = pos;
this.type = type;
switch(this.type) {
case 1:
value = 5;
color = Color.LIGHT_GRAY;
break;
case 2:
value = 10;
color = Color.orange;
break;
case 3:
value = 15;
color = Color.green;
break;
}
}
This is the method, which draws the stones:
private void drawStones(Graphics2D g2) {
stones = view.getGame().getLevel().getStones();
for (int i = 0; i < stones.length; i++) {
for (int j = 0; j < stones[1].length; j++) {
int x_position = (int) stones[i][j].getPosition().getX();
int y_position = (int) stones[i][j].getPosition().getY();
if(stones[i][j].getType() >= 1) {
g2.setColor(stones[i][j].getColor());
g2.fillRoundRect(x_position, y_position,
(int) ((double)Constants.SCREEN_WIDTH/Constants.SQUARES_X)-2,
(int) ((double)Constants.SCREEN_HEIGHT/Constants.SQUARES_Y)-2 ,1,1);
}
}
}
}
And this is the method, which updates the stones type on hit:
private void updateStonesAndScore() {
int posLine = ball.getHitStonePosition().getLine();
int posColumn = ball.getHitStonePosition().getColumn();
score = score + stones[posLine][posColumn].getValue();
System.out.println(stones[posLine][posColumn].getType());
stones[posLine][posColumn].setColor(stones[posLine][posColumn].getColor());
stones[posLine][posColumn].setType(stones[posLine][posColumn].getType()-1);
}
Does anybody know why stones color doesn't change on hit, allthough its type is succesfully reduced?
Thanks in advance!

You never change the color of the stone. It is set in the constructor but changed no where else (not in code you've shown).
I wouldn't even have the stone hold a color value, but instead let the drawing code decide what to color it, perhaps something like:
private void drawStones(Graphics2D g2) {
stones = view.getGame().getLevel().getStones();
for (int i = 0; i < stones.length; i++) {
for (int j = 0; j < stones[1].length; j++) {
int x_position = (int) stones[i][j].getPosition().getX();
int y_position = (int) stones[i][j].getPosition().getY();
if(stones[i][j].getType() >= 1) {
// ********
int type = stones[i][j].getType();
Color color = .... set color based on type value here
// *******
g2.setColor(stones[i][j].getColor());
g2.fillRoundRect(x_position, y_position,
(int) ((double)Constants.SCREEN_WIDTH/Constants.SQUARES_X)-2,
(int) ((double)Constants.SCREEN_HEIGHT/Constants.SQUARES_Y)-2 ,1,1);
}
}
}
}

how to Smooth catmull rom spline curve when it is moving like amaging wire in libgdx

i have tried to move my curve but it is not moving well, when it changes its direction from left to right or right to left then the movement is quite awkward.
i want to move my curve like this video
video of curve movement what i actually want.
In this video when a it change its direction it is so graceful but in my case it change its direction and the curve gives a crazy shape at newly added point.
Experts please solve this problem.
here is my code
//create paths
private Bezier<Vector2> path1;
private CatmullRomSpline<Vector2> path2;
private ShapeRenderer sr;
int height,width;
Vector2 starting,ending,endingControl;
ArrayList<Vector2> listOfPoints;
Vector3 touchPos;
float timeDifference;
Boolean leftPos=false,rightPos=false;
Boolean isTouch=false,isTouchUp=false;
Vector2 mVector2;
private OrthographicCamera cam;
Vector2[] controlPoints;
#Override
public void create () {
width = Gdx.graphics.getWidth();
height = Gdx.graphics.getHeight();
ending=new Vector2(width/2,height/2);
endingControl=new Vector2(ending.x,ending.y+10);
starting=new Vector2(width/2,0);
controlPoints = new Vector2[]{starting,starting,ending,ending};
// set up the curves
path2 = new CatmullRomSpline<Vector2>(controlPoints, false);
listOfPoints=new ArrayList<Vector2>();
// setup ShapeRenderer
sr = new ShapeRenderer();
sr.setAutoShapeType(true);
sr.setColor(Color.BLACK);
cam=new OrthographicCamera();
cam.setToOrtho(false);
listOfPoints.add(new Vector2(width/2,0)); //starting
listOfPoints.add(new Vector2(width/2,0)); //starting
}
#Override
public void resize(int width, int height) {
// TODO Auto-generated method stub
super.resize(width, height);
cam.update();
}
#Override
public void render () {
cam.update();
Gdx.gl.glClearColor(1f, 1f, 1f, 1f);
Gdx.gl.glClear(GL20.GL_COLOR_BUFFER_BIT);
sr.begin();
sr.set(ShapeType.Filled);
if(Gdx.input.isTouched())
{
if(!isTouch){
listOfPoints.add(new Vector2(ending.x+2, ending.y-4));
int s=listOfPoints.size();
controlPoints=new Vector2[s+2];
listOfPoints.toArray(controlPoints);
controlPoints[s]=ending;
//endingControl.x=ending.y;
controlPoints[s+1]=ending;
path2 = new CatmullRomSpline<Vector2>(controlPoints, false);
}
isTouch=true;
ending.x+=3;
}
else {
if(isTouch){
listOfPoints.add(new Vector2(ending.x-2, ending.y-4));
int s=listOfPoints.size();
controlPoints=new Vector2[s+2];
listOfPoints.toArray(controlPoints);
controlPoints[s]=ending;
controlPoints[s+1]=ending;
path2 = new CatmullRomSpline<Vector2>(controlPoints, false);
}
isTouch=false;
ending.x-=3;
}
moveAndReduce();
for(int i = 0; i < 100; ++i){
float t = i /100f;
Vector2 st = new Vector2();
Vector2 end = new Vector2();
path2.valueAt(st,t);
path2.valueAt(end, t-0.01f);
sr.rectLine(st.x, st.y, end.x, end.y,3);
}
sr.end();
}
#Override
public void dispose () {
sr.dispose();
}
public void moveAndReduce()
{
for(Vector2 vector2:listOfPoints)
{
vector2.y-=3 ;
}
if(listOfPoints.size()>3 && listOfPoints.get(3).y<-1)
{
listOfPoints.remove(0);
listOfPoints.set(0, listOfPoints.get(1));
int s=listOfPoints.size();
controlPoints=new Vector2[s+2];
listOfPoints.toArray(controlPoints);
controlPoints[s]=ending;
controlPoints[s+1]=ending;
path2 = new CatmullRomSpline<Vector2>(controlPoints, false);
}
}

Going by the video the curve does not look like it is constrained by control points but just a simple trace of and accelerating point.
You create an array of floats the length in pixels matching the length of the line in the x direction. For example if screen is 200 pixels wide the line can be 100 so the array is 100 in length. Set each float in the array to the start value half the screen height. I call the array line in this answer. You call it what you like.
The you assign a head index that is the index of the rightmost point. Each frame you move the head index up by one. If it is over the array length-1 you set it to zero (beginning of array)
Rendering the Path
When you draw the line you draw all the points from head + 1
Path p = new Path();
for(int i = 0; i < 100; ++i){
p.lineTo(i, line[(i + head + 1) % 100]); // add path points
}
// draw the path;
Moving up and down
To make it move you have a movement float move that is 0 for no movement or positive and negative values the move up or down.
When you want it to move increase the move amount by a fixed value.
// moving down
if(move < maxMove){ // set a max move amount eg 10
move += moveAmount; // moveAmount 0.2 just as an example
}
Same for moving up, but subtract
When there is no input you move the move amount back to zero by a fixed rate
// assume this is code run when no input
if(move != 0){
if(Math.abs(move) < moveAmount){ // if close to zero set to zero
move = 0;
}else{
move -= Math.sign(move) * moveAmount; // else move towards zero at
// fixed rate
}
}
Moving forward
The line does not move forward, just appears to do so as we move the head position up the array each frame.
Back to moving the line's head the following move the line head position up or down (but is not complete the last line is modified to create a smoother curve)
float pos = line[head]; // get the pos of line at head
head += 1; // move the head forward 1
head %= 100; // if past end of array move to 0
line[head] = pos + move; // set the new head position
A better curve
This will move the head of the line up or down depending on move. The curve we get is not that nice so to make it a little smoother you need to change the rate the move value changes the head position.
// an sCurve for any value of move the result is from -1 to 1
// the greater or smaller move the closer to 1 or -1 the value gets
// the value -1.2 controls the rate at which the value moves to 1 or -1
// the closer to -1 the value is the slower the value moves to 1 or -1
float res = (2 / (1 + Math.pow(move,-1.2))) -1;
This in effect changes the shape of the lines curve to a almost sine wave when moving up and down
// so instead of
//line[head] = pos + move; // set the new head position
line[head] = pos + ( (2 / (1 + Math.pow(move,-1.2))) -1 ) * maxSpeed;
// max speed is the max speed the line head can move up or down
// per frame in pixels.
Example to show the curve
Below is a Javascript implementation that does it as outlined above (is not intended as answer code). Use the keyboard Arrow Up and Arrow down to move the line
If you are using a tablet or phone then the following image is what you will see as way to late for me to add and test touch for the example
const doFor = (count, callback) => {var i = 0; while (i < count) { callback(i ++) } };
const keys = {
ArrowUp : false,
ArrowDown : false,
};
function keyEvents(e){
if(keys[e.code] !== undefined){
keys[e.code] = event.type === "keydown";
e.preventDefault();
}
}
addEventListener("keyup", keyEvents);
addEventListener("keydown", keyEvents);
focus();
var gameOver = 0;
var gameOverWait = 100;
var score = 0;
var nextWallIn = 500
var nextWallCount = nextWallIn;
var wallHole = 50;
const wallWidth = 5;
const walls = [];
function addWall(){
var y = (Math.random() * (H - wallHole * 2)) + wallHole *0.5;
walls.push({
x : W,
top : y,
bottom : y + wallHole,
point : 1, // score point
});
}
function updateWalls(){
nextWallCount += 1;
if(nextWallCount >= nextWallIn){
addWall();
nextWallCount = 0;
nextWallIn -= 1;
wallHole -= 1;
}
for(var i = 0; i < walls.length; i ++){
var w = walls[i];
w.x -= 1;
if(w.x < -wallWidth){
walls.splice(i--,1);
}
if(w.x >= line.length- + wallWidth && w.x < line.length){
var pos = line[head];
if(pos < w.top || pos > w.bottom){
gameOver = gameOverWait;
}
}
if(w.point > 0 && w.x <= line.length){
score += w.point;
w.point = 0;
}
}
}
function drawWalls(){
for(var i = 0; i < walls.length; i ++){
var w = walls[i];
ctx.fillStyle = "red";
ctx.fillRect(w.x,0,wallWidth,w.top);
ctx.fillRect(w.x,w.bottom,wallWidth,H-w.bottom);
}
}
const sCurve = (x,p) => (2 / (1 + Math.pow(p,-x))) -1;
const ctx = canvas.getContext("2d");
var W,H; // canvas width and height
const line = [];
var move = 0;
var curvePower = 1.2;
var curveSpeed = 0.2;
var maxSpeed = 10;
var headMoveMultiply = 2;
var head;
function init(){
line.length = 0;
doFor(W / 2,i => line[i] = H / 2);
head = line.length - 1;
move = 0;
walls.length = 0;
score = 0;
nextWallIn = 500
nextWallCount = nextWallIn;
wallHole = 50;
ctx.font = "30px arial black";
}
function stepLine(){
var pos = line[head];
head += 1;
head %= line.length;
line[head] = pos + sCurve(move,curvePower)*headMoveMultiply ;
}
function drawLine(){
ctx.beginPath();
ctx.strokeStyle = "black";
ctx.lineWidth = 3;
ctx.lineJoin = "round";
ctx.lineCap = "round";
for(var i = 0; i <line.length; i++){
ctx.lineTo(i,line[(i + head + 1) % line.length]);
}
ctx.stroke();
}
function mainLoop(time){
if(canvas.width !== innerWidth || canvas.height !== innerHeight){
W = canvas.width = innerWidth;
H = canvas.height = innerHeight;
init();
}
if(gameOver === 1){
gameOver = 0;
init();
}
ctx.setTransform(1,0,0,1,0,0);
ctx.clearRect(0,0,W,H);
if(keys.ArrowUp){
if(move > - maxSpeed){
move -= curveSpeed;
}
}else if(keys.ArrowDown){
if(move < maxSpeed){
move += curveSpeed;
}
}else{
move -= Math.sign(move)*curveSpeed;
if(Math.abs(move) < curveSpeed){
move = 0;
}
}
if(gameOver === 0){
stepLine();
updateWalls();
}
drawLine();
drawWalls();
ctx.fillStyle = "Black";
ctx.textAlign = "left";
ctx.fillText("Score : " + score, 10,30);
if(gameOver > 0){
ctx.textAlign = "center";
ctx.fillText("Crashed !!", W / 2,H * 0.4);
gameOver -= 1;
}
requestAnimationFrame(mainLoop);
}
requestAnimationFrame(mainLoop);
canvas {
position : absolute;
top : 0px;
left : 0px;
z-index : -10;
}
<br><br><br>Up down arrow keys to move line.
<canvas id=canvas></canvas>

How to get clear mask of users in simple-openni?

I am trying to extract user silhouette and put it above my images. I was able to make a mask and cut user from rgb image. But the contour is messy.
The question is how I can make the mask more precise (to fit real user). I've tried ERODE-DILATE filters, but they don't do much. Maybe I need some Feather filter like in Photoshop. Or I don't know.
Here is my code.
import SimpleOpenNI.*;
SimpleOpenNI context;
PImage mask;
void setup()
{
size(640*2, 480);
context = new SimpleOpenNI(this);
if (context.isInit() == false)
{
exit();
return;
}
context.enableDepth();
context.enableRGB();
context.enableUser();
context.alternativeViewPointDepthToImage();
}
void draw()
{
frame.setTitle(int(frameRate) + " fps");
context.update();
int[] userMap = context.userMap();
background(0, 0, 0);
mask = loadImage("black640.jpg"); //just a black image
int xSize = context.depthWidth();
int ySize = context.depthHeight();
mask.loadPixels();
for (int y = 0; y < ySize; y++) {
for (int x = 0; x < xSize; x++) {
int index = x + y*xSize;
if (userMap[index]>0) {
mask.pixels[index]=color(255, 255, 255);
}
}
}
mask.updatePixels();
image(mask, 0, 0);
mask.filter(DILATE);
mask.filter(DILATE);
PImage rgb = context.rgbImage();
rgb.mask(mask);
image(rgb, context.depthWidth() + 10, 0);
}

It's good you're aligning the RGB and depth streams.
There are few things that could be improved in terms of efficiency:
No need to reload a black image every single frame (in the draw() loop) since you're modifying all the pixels anyway:
mask = loadImage("black640.jpg"); //just a black image
Also, since you don't need the x,y coordinates as you loop through the user data, you can use a single for loop which should be a bit faster:
for(int i = 0 ; i < numPixels ; i++){
mask.pixels[i] = userMap[i] > 0 ? color(255) : color(0);
}
instead of:
for (int y = 0; y < ySize; y++) {
for (int x = 0; x < xSize; x++) {
int index = x + y*xSize;
if (userMap[index]>0) {
mask.pixels[index]=color(255, 255, 255);
}
}
}
Another hacky thing you could do is retrieve the userImage() from SimpleOpenNI, instead of the userData() and apply a THRESHOLD filter to it, which in theory should give you the same result as above.
For example:
int[] userMap = context.userMap();
background(0, 0, 0);
mask = loadImage("black640.jpg"); //just a black image
int xSize = context.depthWidth();
int ySize = context.depthHeight();
mask.loadPixels();
for (int y = 0; y < ySize; y++) {
for (int x = 0; x < xSize; x++) {
int index = x + y*xSize;
if (userMap[index]>0) {
mask.pixels[index]=color(255, 255, 255);
}
}
}
could be:
mask = context.userImage();
mask.filter(THRESHOLD);
In terms of filtering, if you want to shrink the silhouette you should ERODE and bluring should give you a bit of that Photoshop like feathering.
Note that some filter() calls take arguments (like BLUR), but others don't like the ERODE/DILATE morphological filters, but you can still roll your own loops to deal with that.
I also recommend having some sort of easy to tweak interface (it can be fancy slider or a simple keyboard shortcut) when playing with filters.
Here's a rough attempt at the refactored sketch with the above comments:
import SimpleOpenNI.*;
SimpleOpenNI context;
PImage mask;
int numPixels = 640*480;
int dilateAmt = 1;
int erodeAmt = 1;
int blurAmt = 0;
void setup()
{
size(640*2, 480);
context = new SimpleOpenNI(this);
if (context.isInit() == false)
{
exit();
return;
}
context.enableDepth();
context.enableRGB();
context.enableUser();
context.alternativeViewPointDepthToImage();
mask = createImage(640,480,RGB);
}
void draw()
{
frame.setTitle(int(frameRate) + " fps");
context.update();
int[] userMap = context.userMap();
background(0, 0, 0);
//you don't need to keep reloading the image every single frame since you're updating all the pixels bellow anyway
// mask = loadImage("black640.jpg"); //just a black image
// mask.loadPixels();
// int xSize = context.depthWidth();
// int ySize = context.depthHeight();
// for (int y = 0; y < ySize; y++) {
// for (int x = 0; x < xSize; x++) {
// int index = x + y*xSize;
// if (userMap[index]>0) {
// mask.pixels[index]=color(255, 255, 255);
// }
// }
// }
//a single loop is usually faster than a nested loop and you don't need the x,y coordinates anyway
for(int i = 0 ; i < numPixels ; i++){
mask.pixels[i] = userMap[i] > 0 ? color(255) : color(0);
}
//erode
for(int i = 0 ; i < erodeAmt ; i++) mask.filter(ERODE);
//dilate
for(int i = 0 ; i < dilateAmt; i++) mask.filter(DILATE);
//blur
mask.filter(BLUR,blurAmt);
mask.updatePixels();
//preview the mask after you process it
image(mask, 0, 0);
PImage rgb = context.rgbImage();
rgb.mask(mask);
image(rgb, context.depthWidth() + 10, 0);
//print filter values for debugging purposes
fill(255);
text("erodeAmt: " + erodeAmt + "\tdilateAmt: " + dilateAmt + "\tblurAmt: " + blurAmt,15,15);
}
void keyPressed(){
if(key == 'e') erodeAmt--;
if(key == 'E') erodeAmt++;
if(key == 'd') dilateAmt--;
if(key == 'D') dilateAmt++;
if(key == 'b') blurAmt--;
if(key == 'B') blurAmt++;
//constrain values
if(erodeAmt < 0) erodeAmt = 0;
if(dilateAmt < 0) dilateAmt = 0;
if(blurAmt < 0) blurAmt = 0;
}
Unfortunately I can't test with an actual sensor right now, so please use the concepts explained, but bare in mind the full sketch code isn't tested.
This above sketch (if it runs) should allow you to use keys to control the filter parameters (e/E to decrease/increase erosion, d/D for dilation, b/B for blur). Hopefully you'll get satisfactory results.
When working with SimpleOpenNI in general I advise recording an .oni file (check out the RecorderPlay example for that) of a person for the most common use case. This will save you some time on the long run when testing and will allow you to work remotely with the sensor detached. One thing to bare in mind, the depth resolution is reduced to half on recordings (but using a usingRecording boolean flag should keep things safe)
The last and probably most important point is about the quality of the end result. Your resulting image can't be that much better if the source image isn't easy to work with to begin with. The depth data from the original Kinect sensor isn't great. The Asus sensors feel a wee bit more stable, but still the difference is negligible in most cases. If you are going to stick to one of these sensors, make sure you've got a clear background and decent lighting (without too much direct warm light (sunlight, incandescent lightbulbs, etc.) since they may interfere with the sensor)
If you want a more accurate user cut and the above filtering doesn't get the results you're after, consider switching to a better sensor like KinectV2. The depth quality is much better and the sensor is less susceptible to direct warm light. This may mean you need to use Windows (I see there's a KinectPV2 wrapper available) or OpenFrameworks(c++ collections of libraries similar to Processing) with ofxKinectV2

I've tried built-in erode-dilate-blur in processing. But they are very inefficient. Every time I increment blurAmount in img.filter(BLUR,blurAmount), my FPS decreases by 5 frames.
So I decided to try opencv. It is much better in comparison. The result is satisfactory.
import SimpleOpenNI.*;
import processing.video.*;
import gab.opencv.*;
SimpleOpenNI context;
OpenCV opencv;
PImage mask;
int numPixels = 640*480;
int dilateAmt = 1;
int erodeAmt = 1;
int blurAmt = 1;
Movie mov;
void setup(){
opencv = new OpenCV(this, 640, 480);
size(640*2, 480);
context = new SimpleOpenNI(this);
if (context.isInit() == false) {
exit();
return;
}
context.enableDepth();
context.enableRGB();
context.enableUser();
context.alternativeViewPointDepthToImage();
mask = createImage(640, 480, RGB);
mov = new Movie(this, "wild.mp4");
mov.play();
mov.speed(5);
mov.volume(0);
}
void movieEvent(Movie m) {
m.read();
}
void draw() {
frame.setTitle(int(frameRate) + " fps");
context.update();
int[] userMap = context.userMap();
background(0, 0, 0);
mask.loadPixels();
for (int i = 0; i < numPixels; i++) {
mask.pixels[i] = userMap[i] > 0 ? color(255) : color(0);
}
mask.updatePixels();
opencv.loadImage(mask);
opencv.gray();
for (int i = 0; i < erodeAmt; i++) {
opencv.erode();
}
for (int i = 0; i < dilateAmt; i++) {
opencv.dilate();
}
if (blurAmt>0) {//blur with 0 amount causes error
opencv.blur(blurAmt);
}
mask = opencv.getSnapshot();
image(mask, 0, 0);
PImage rgb = context.rgbImage();
rgb.mask(mask);
image(mov, context.depthWidth() + 10, 0);
image(rgb, context.depthWidth() + 10, 0);
fill(255);
text("erodeAmt: " + erodeAmt + "\tdilateAmt: " + dilateAmt + "\tblurAmt: " + blurAmt, 15, 15);
}
void keyPressed() {
if (key == 'e') erodeAmt--;
if (key == 'E') erodeAmt++;
if (key == 'd') dilateAmt--;
if (key == 'D') dilateAmt++;
if (key == 'b') blurAmt--;
if (key == 'B') blurAmt++;
//constrain values
if (erodeAmt < 0) erodeAmt = 0;
if (dilateAmt < 0) dilateAmt = 0;
if (blurAmt < 0) blurAmt = 0;
}