Develop Reference

Neural Network returning NaN as output

I am trying to write my first neural network to play the game connect four.
Im using Java and deeplearning4j.
I tried to implement a genetic algorithm, but when i train the network for a while, the outputs of the network jump to NaN and I am unable to tell where I messed up so badly for this to happen..
I will post all 3 classes below, where Game is the game logic and rules, VGFrame the UI and Main all the nn stuff.
I have a pool of 35 neural networks and each iteration i let the best 5 live and breed and randomize the newly created ones a little.
To evaluate the networks I let them battle each other and give points to the winner and points for loosing later.
Since I penalize putting a stone into a column thats already full I expected the neural networks at least to be able to play the game by the rules after a while but they cant do this.
I googled the NaN problem and it seems to be an expoding gradient problem, but from my understanding this shouldn't occur in a genetic algorithm?
Any ideas where I could look for the error or whats generally wrong with my implementation?
Main
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.Random;
import org.deeplearning4j.nn.api.OptimizationAlgorithm;
import org.deeplearning4j.nn.conf.MultiLayerConfiguration;
import org.deeplearning4j.nn.conf.NeuralNetConfiguration;
import org.deeplearning4j.nn.conf.layers.DenseLayer;
import org.deeplearning4j.nn.conf.layers.OutputLayer;
import org.deeplearning4j.nn.multilayer.MultiLayerNetwork;
import org.deeplearning4j.nn.weights.WeightInit;
import org.nd4j.linalg.activations.Activation;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.learning.config.Nesterovs;
public class Main {
final int numRows = 7;
final int numColums = 6;
final int randSeed = 123;
MultiLayerNetwork[] models;
static Random random = new Random();
private static final Logger log = LoggerFactory.getLogger(Main.class);
final float learningRate = .8f;
int batchSize = 64; // Test batch size
int nEpochs = 1; // Number of training epochs
// --
public static Main current;
Game mainGame = new Game();
public static void main(String[] args) {
current = new Main();
current.frame = new VGFrame();
current.loadWeights();
}
private VGFrame frame;
private final double mutationChance = .05;
public Main() {
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER)
.activation(Activation.RELU).seed(randSeed)
.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(new Nesterovs(0.1, 0.9))
.list()
.layer(new DenseLayer.Builder().nIn(42).nOut(30).activation(Activation.RELU)
.weightInit(WeightInit.XAVIER).build())
.layer(new DenseLayer.Builder().nIn(30).nOut(15).activation(Activation.RELU)
.weightInit(WeightInit.XAVIER).build())
.layer(new OutputLayer.Builder(LossFunction.NEGATIVELOGLIKELIHOOD).nIn(15).nOut(7)
.activation(Activation.SOFTMAX).weightInit(WeightInit.XAVIER).build())
.build();
models = new MultiLayerNetwork[35];
for (int i = 0; i < models.length; i++) {
models[i] = new MultiLayerNetwork(conf);
models[i].init();
}
}
public void addChip(int i, boolean b) {
if (mainGame.gameState == 0)
mainGame.addChip(i, b);
if (mainGame.gameState == 0) {
float[] f = Main.rowsToInput(mainGame.rows);
INDArray input = Nd4j.create(f);
INDArray output = models[0].output(input);
for (int i1 = 0; i1 < 7; i1++) {
System.out.println(i1 + ": " + output.getDouble(i1));
}
System.out.println("----------------");
mainGame.addChip(Main.getHighestOutput(output), false);
}
getFrame().paint(getFrame().getGraphics());
}
public void newGame() {
mainGame = new Game();
getFrame().paint(getFrame().getGraphics());
}
public void startTraining(int iterations) {
// --------------------------
for (int gameNumber = 0; gameNumber < iterations; gameNumber++) {
System.out.println("Iteration " + gameNumber + " of " + iterations);
float[] evaluation = new float[models.length];
for (int i = 0; i < models.length; i++) {
for (int j = 0; j < models.length; j++) {
if (i != j) {
Game g = new Game();
g.playFullGame(models[i], models[j]);
if (g.gameState == 1) {
evaluation[i] += 45;
evaluation[j] += g.turnNumber;
}
if (g.gameState == 2) {
evaluation[j] += 45;
evaluation[i] += g.turnNumber;
}
}
}
}
float[] evaluationSorted = evaluation.clone();
Arrays.sort(evaluationSorted);
// keep the best 4
int n1 = 0, n2 = 0, n3 = 0, n4 = 0, n5 = 0;
for (int i = 0; i < evaluation.length; i++) {
if (evaluation[i] == evaluationSorted[evaluationSorted.length - 1])
n1 = i;
if (evaluation[i] == evaluationSorted[evaluationSorted.length - 2])
n2 = i;
if (evaluation[i] == evaluationSorted[evaluationSorted.length - 3])
n3 = i;
if (evaluation[i] == evaluationSorted[evaluationSorted.length - 4])
n4 = i;
if (evaluation[i] == evaluationSorted[evaluationSorted.length - 5])
n5 = i;
}
models[0] = models[n1];
models[1] = models[n2];
models[2] = models[n3];
models[3] = models[n4];
models[4] = models[n5];
for (int i = 3; i < evaluationSorted.length; i++) {
// random parent/keep w8ts
double r = Math.random();
if (r > .3) {
models[i] = models[random.nextInt(3)].clone();
} else if (r > .1) {
models[i].setParams(breed(models[random.nextInt(3)], models[random.nextInt(3)]));
}
// Mutate
INDArray params = models[i].params();
models[i].setParams(mutate(params));
}
}
}
private INDArray mutate(INDArray params) {
double[] d = params.toDoubleVector();
for (int i = 0; i < d.length; i++) {
if (Math.random() < mutationChance)
d[i] += (Math.random() - .5) * learningRate;
}
return Nd4j.create(d);
}
private INDArray breed(MultiLayerNetwork m1, MultiLayerNetwork m2) {
double[] d = m1.params().toDoubleVector();
double[] d2 = m2.params().toDoubleVector();
for (int i = 0; i < d.length; i++) {
if (Math.random() < .5)
d[i] += d2[i];
}
return Nd4j.create(d);
}
static int getHighestOutput(INDArray output) {
int x = 0;
for (int i = 0; i < 7; i++) {
if (output.getDouble(i) > output.getDouble(x))
x = i;
}
return x;
}
static float[] rowsToInput(byte[][] rows) {
float[] f = new float[7 * 6];
for (int i = 0; i < 6; i++) {
for (int j = 0; j < 7; j++) {
// f[j + i * 7] = rows[j][i] / 2f;
f[j + i * 7] = (rows[j][i] == 0 ? .5f : rows[j][i] == 1 ? 0f : 1f);
}
}
return f;
}
public void saveWeights() {
log.info("Saving model");
for (int i = 0; i < models.length; i++) {
File resourcesDirectory = new File("src/resources/model" + i);
try {
models[i].save(resourcesDirectory, true);
} catch (IOException e) {
e.printStackTrace();
}
}
}
public void loadWeights() {
if (new File("src/resources/model0").exists()) {
for (int i = 0; i < models.length; i++) {
File resourcesDirectory = new File("src/resources/model" + i);
try {
models[i] = MultiLayerNetwork.load(resourcesDirectory, true);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
System.out.println("col: " + models[0].params().shapeInfoToString());
}
public VGFrame getFrame() {
return frame;
}
}
VGFrame
import java.awt.Color;
import java.awt.Graphics;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import javax.swing.BorderFactory;
import javax.swing.JButton;
import javax.swing.JFrame;
import javax.swing.JPanel;
import javax.swing.JTextField;
public class VGFrame extends JFrame {
JTextField iterations;
/**
*
*/
private static final long serialVersionUID = 1L;
public VGFrame() {
super("Vier Gewinnt");
this.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
this.setSize(1300, 800);
this.setVisible(true);
JPanel panelGame = new JPanel();
panelGame.setBorder(BorderFactory.createLineBorder(Color.black, 2));
this.add(panelGame);
var handler = new Handler();
var menuHandler = new MenuHandler();
JButton b1 = new JButton("1");
JButton b2 = new JButton("2");
JButton b3 = new JButton("3");
JButton b4 = new JButton("4");
JButton b5 = new JButton("5");
JButton b6 = new JButton("6");
JButton b7 = new JButton("7");
b1.addActionListener(handler);
b2.addActionListener(handler);
b3.addActionListener(handler);
b4.addActionListener(handler);
b5.addActionListener(handler);
b6.addActionListener(handler);
b7.addActionListener(handler);
panelGame.add(b1);
panelGame.add(b2);
panelGame.add(b3);
panelGame.add(b4);
panelGame.add(b5);
panelGame.add(b6);
panelGame.add(b7);
JButton buttonTrain = new JButton("Train");
JButton buttonNewGame = new JButton("New Game");
JButton buttonSave = new JButton("Save Weights");
JButton buttonLoad = new JButton("Load Weights");
iterations = new JTextField("1000");
buttonTrain.addActionListener(menuHandler);
buttonNewGame.addActionListener(menuHandler);
buttonSave.addActionListener(menuHandler);
buttonLoad.addActionListener(menuHandler);
iterations.addActionListener(menuHandler);
panelGame.add(iterations);
panelGame.add(buttonTrain);
panelGame.add(buttonNewGame);
panelGame.add(buttonSave);
panelGame.add(buttonLoad);
this.validate();
}
#Override
public void paint(Graphics g) {
super.paint(g);
if (Main.current.mainGame.rows == null)
return;
var rows = Main.current.mainGame.rows;
for (int i = 0; i < rows.length; i++) {
for (int j = 0; j < rows[0].length; j++) {
if (rows[i][j] == 0)
break;
g.setColor((rows[i][j] == 1 ? Color.yellow : Color.red));
g.fillOval(80 + 110 * i, 650 - 110 * j, 100, 100);
}
}
}
public void update() {
}
}
class Handler implements ActionListener {
#Override
public void actionPerformed(ActionEvent event) {
if (Main.current.mainGame.playersTurn)
Main.current.addChip(Integer.parseInt(event.getActionCommand()) - 1, true);
}
}
class MenuHandler implements ActionListener {
#Override
public void actionPerformed(ActionEvent event) {
switch (event.getActionCommand()) {
case "New Game":
Main.current.newGame();
break;
case "Train":
Main.current.startTraining(Integer.parseInt(Main.current.getFrame().iterations.getText()));
break;
case "Save Weights":
Main.current.saveWeights();
break;
case "Load Weights":
Main.current.loadWeights();
break;
}
}
}
Game
import org.deeplearning4j.nn.multilayer.MultiLayerNetwork;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.factory.Nd4j;
public class Game {
int turnNumber = 0;
byte[][] rows = new byte[7][6];
boolean playersTurn = true;
int gameState = 0; // 0:running, 1:Player1, 2:Player2, 3:Draw
public boolean isRunning() {
return this.gameState == 0;
}
public void addChip(int x, boolean player1) {
turnNumber++;
byte b = nextRow(x);
if (b == 6) {
gameState = player1 ? 2 : 1;
return;
}
rows[x][b] = (byte) (player1 ? 1 : 2);
gameState = checkWinner(x, b);
}
private byte nextRow(int x) {
for (byte i = 0; i < rows[x].length; i++) {
if (rows[x][i] == 0)
return i;
}
return 6;
}
// 0 continue, 1 Player won, 2 ai won, 3 Draw
private int checkWinner(int x, int y) {
int color = rows[x][y];
// Vertikal
if (getCount(x, y, 1, 0) + getCount(x, y, -1, 0) >= 3)
return rows[x][y];
// Horizontal
if (getCount(x, y, 0, 1) + getCount(x, y, 0, -1) >= 3)
return rows[x][y];
// Diagonal1
if (getCount(x, y, 1, 1) + getCount(x, y, -1, -1) >= 3)
return rows[x][y];
// Diagonal2
if (getCount(x, y, -1, 1) + getCount(x, y, 1, -1) >= 3)
return rows[x][y];
for (byte[] bs : rows) {
for (byte s : bs) {
if (s == 0)
return 0;
}
}
return 3; // Draw
}
private int getCount(int x, int y, int dirX, int dirY) {
int color = rows[x][y];
int count = 0;
while (true) {
x += dirX;
y += dirY;
if (x < 0 | x > 6 | y < 0 | y > 5)
break;
if (color != rows[x][y])
break;
count++;
}
return count;
}
public void playFullGame(MultiLayerNetwork m1, MultiLayerNetwork m2) {
boolean player1 = true;
while (this.gameState == 0) {
float[] f = Main.rowsToInput(this.rows);
INDArray input = Nd4j.create(f);
this.addChip(Main.getHighestOutput(player1 ? m1.output(input) : m2.output(input)), player1);
player1 = !player1;
}
}
}

With a quick look, and based on the analysis of your multiplier variants, it seems like the NaN is produced by an arithmetic underflow, caused by your gradients being too small (too close to absolute 0).
This is the most suspicious part of the code:
f[j + i * 7] = (rows[j][i] == 0 ? .5f : rows[j][i] == 1 ? 0f : 1f);
If rows[j][i] == 1 then 0f is stored. I don't know how this is managed by the neural network (or even java), but mathematically speaking, a finite-sized float cannot include zero.
Even if your code would alter the 0f with some extra salt, those array values' resultants would have some risk of becoming too close to zero. Due to limited precision when representing real numbers, values very close to zero can not be represented, hence the NaN.
These values have a very friendly name: subnormal numbers.
Any non-zero number with magnitude smaller than the smallest normal
number is subnormal.
IEEE_754
As with IEEE 754-1985, The standard recommends 0 for signaling NaNs, 1 for quiet NaNs, so that a signaling NaNs can be quieted by changing only this bit to 1, while the reverse could yield the encoding of an infinity.
Above's text is important here: according to the standard, you are actually specifying a NaN with any 0f value stored.
Even if the name is misleading, Float.MIN_VALUE is a positive value,higher than 0:
The real minimum float value is, in fact: -Float.MAX_VALUE.
Is floating point math subnormal?
Normalizing the gradients
If you check the issue is only because of the 0f values, you could just alter them for other values that represent something similar; Float.MIN_VALUE, Float.MIN_NORMAL, and so on. Something like this, also in other possible parts of the code where this scenario could happen. Take these just as examples, and play with these ranges:
rows[j][i] == 1 ? Float.MIN_VALUE : 1f;
rows[j][i] == 1 ? Float.MIN_NORMAL : Float.MAX_VALUE/2;
rows[j][i] == 1 ? -Float.MAX_VALUE/2 : Float.MAX_VALUE/2;
Even so, this could also lead to a NaN, based on how these values are altered.
If so, you should normalize the values. You could try applying a GradientNormalizer for this. In your network initialization, something like this should be defined, for each layer(or for those who are problematic):
new NeuralNetConfiguration
.Builder()
.weightInit(WeightInit.XAVIER)
(...)
.layer(new DenseLayer.Builder().nIn(42).nOut(30).activation(Activation.RELU)
.weightInit(WeightInit.XAVIER)
.gradientNormalization(GradientNormalization.RenormalizeL2PerLayer) //this
.build())
(...)
There are different normalizers, so choose which one fits your schema best, and which layers should include one. The options are:
GradientNormalization
RenormalizeL2PerLayer
Rescale gradients by dividing by the L2 norm
of all gradients for the layer.
RenormalizeL2PerParamType
Rescale gradients by dividing by the L2
norm of the gradients, separately for each type of parameter within
the layer. This differs from RenormalizeL2PerLayer in that here, each
parameter type (weight, bias etc) is normalized separately. For
example, in a MLP/FeedForward network (where G is the gradient
vector), the output is as follows:
GOut_weight = G_weight / l2(G_weight) GOut_bias = G_bias / l2(G_bias)
ClipElementWiseAbsoluteValue
Clip the gradients on a per-element
basis. For each gradient g, set g <- sign(g) max(maxAllowedValue,|g|).
i.e., if a parameter gradient has absolute value greater than the
threshold, truncate it. For example, if threshold = 5, then values in
range -5<g<5 are unmodified; values <-5 are set to -5; values >5 are
set to 5.
ClipL2PerLayer
Conditional renormalization. Somewhat similar to
RenormalizeL2PerLayer, this strategy scales the gradients if and only
if the L2 norm of the gradients (for entire layer) exceeds a specified
threshold. Specifically, if G is gradient vector for the layer, then:
GOut = G if l2Norm(G) < threshold (i.e., no change) GOut =
threshold * G / l2Norm(G)
ClipL2PerParamType
Conditional renormalization. Very
similar to ClipL2PerLayer, however instead of clipping per layer, do
clipping on each parameter type separately. For example in a recurrent
neural network, input weight gradients, recurrent weight gradients and
bias gradient are all clipped separately.
Here you can find a complete example of the application of these GradientNormalizers.

I think I finally figured it out. I was trying to visualize the network using deeplearning4j-ui, but got some incompatible versions errors. After changing versions I got a new error, stating the networks input is expecting a 2d array and I found on the internet that this is expected across all versions.
So i changed
float[] f = new float[7 * 6];
Nd4j.create(f);
to
float[][] f = new float[1][7 * 6];
Nd4j.createFromArray(f);
And the NaN values finally disappeared. #aran So I guess assuming incorrect inputs was definitly the right direction. Thank you so much for your help :)

How to show images in a large frequency in JavaFX?

My application generates heatmap images as fast as the CPU can (around 30-60 per second) and I want to display them in a single "live heatmap". In AWT/Swing, I could just paint them into a JPanel which worked like a charm.
Recently, I switched to JavaFX and want to achieve the same here; at first, I tried with a Canvas, which was slow but okay-ish but had a severe memory leak problem, causing the application to crash. Now, I tried the ImageView component - which apparently is way too slow as the image gets quite laggy (using ImageView.setImage on every new iteration). As far as I understand, setImage does not guarantee that the image is actually displayed when the function finishes.
I am getting the impression that I am on the wrong track, using those components in a manner they are not made to. How can I display my 30-60 Images per second?
EDIT: A very simple test application. You will need the JHeatChart library.
Note that on a desktop machine, I get around 70-80 FPS and the visualization is okay and fluid, but on a smaller raspberry pi (my target machine), I get around 30 FPS but an extremly stucking visualization.
package sample;
import javafx.application.Application;
import javafx.embed.swing.SwingFXUtils;
import javafx.scene.Scene;
import javafx.scene.image.ImageView;
import javafx.scene.layout.VBox;
import javafx.stage.Stage;
import org.tc33.jheatchart.HeatChart;
import java.awt.*;
import java.awt.geom.AffineTransform;
import java.awt.image.AffineTransformOp;
import java.awt.image.BufferedImage;
import java.util.LinkedList;
public class Main extends Application {
ImageView imageView = new ImageView();
final int scale = 15;
#Override
public void start(Stage primaryStage) {
Thread generator = new Thread(() -> {
int col = 0;
LinkedList<Long> fps = new LinkedList<>();
while (true) {
fps.add(System.currentTimeMillis());
double[][] matrix = new double[48][128];
for (int i = 0; i < 48; i++) {
for (int j = 0; j < 128; j++) {
matrix[i][j] = col == j ? Math.random() : 0;
}
}
col = (col + 1) % 128;
HeatChart heatChart = new HeatChart(matrix, 0, 1);
heatChart.setShowXAxisValues(false);
heatChart.setShowYAxisValues(false);
heatChart.setLowValueColour(java.awt.Color.black);
heatChart.setHighValueColour(java.awt.Color.white);
heatChart.setAxisThickness(0);
heatChart.setChartMargin(0);
heatChart.setCellSize(new Dimension(1, 1));
long currentTime = System.currentTimeMillis();
fps.removeIf(elem -> currentTime - elem > 1000);
System.out.println(fps.size());
imageView.setImage(SwingFXUtils.toFXImage((BufferedImage) scale(heatChart.getChartImage(), scale), null));
}
});
VBox box = new VBox();
box.getChildren().add(imageView);
Scene scene = new Scene(box, 1920, 720);
primaryStage.setScene(scene);
primaryStage.show();
generator.start();
}
public static void main(String[] args) {
launch(args);
}
private static Image scale(Image image, int scale) {
BufferedImage res = new BufferedImage(image.getWidth(null) * scale, image.getHeight(null) * scale,
BufferedImage.TYPE_INT_ARGB);
AffineTransform at = new AffineTransform();
at.scale(scale, scale);
AffineTransformOp scaleOp =
new AffineTransformOp(at, AffineTransformOp.TYPE_NEAREST_NEIGHBOR);
return scaleOp.filter((BufferedImage) image, res);
}
}

Your code updates the UI from a background thread, which is definitely not allowed. You need to ensure you update from the FX Application Thread. You also want to try to "throttle" the actual UI updates to occur no more than once per JavaFX frame rendering. The easiest way to do this is with an AnimationTimer, whose handle() method is invoked each time a frame is rendered.
Here's a version of your code which does that:
import java.awt.Dimension;
import java.awt.Image;
import java.awt.geom.AffineTransform;
import java.awt.image.AffineTransformOp;
import java.awt.image.BufferedImage;
import java.util.LinkedList;
import java.util.concurrent.atomic.AtomicReference;
import org.tc33.jheatchart.HeatChart;
import javafx.animation.AnimationTimer;
import javafx.application.Application;
import javafx.embed.swing.SwingFXUtils;
import javafx.scene.Scene;
import javafx.scene.image.ImageView;
import javafx.scene.layout.VBox;
import javafx.stage.Stage;
public class Main extends Application {
ImageView imageView = new ImageView();
final int scale = 15;
#Override
public void start(Stage primaryStage) {
AtomicReference<BufferedImage> image = new AtomicReference<>();
Thread generator = new Thread(() -> {
int col = 0;
LinkedList<Long> fps = new LinkedList<>();
while (true) {
fps.add(System.currentTimeMillis());
double[][] matrix = new double[48][128];
for (int i = 0; i < 48; i++) {
for (int j = 0; j < 128; j++) {
matrix[i][j] = col == j ? Math.random() : 0;
}
}
col = (col + 1) % 128;
HeatChart heatChart = new HeatChart(matrix, 0, 1);
heatChart.setShowXAxisValues(false);
heatChart.setShowYAxisValues(false);
heatChart.setLowValueColour(java.awt.Color.black);
heatChart.setHighValueColour(java.awt.Color.white);
heatChart.setAxisThickness(0);
heatChart.setChartMargin(0);
heatChart.setCellSize(new Dimension(1, 1));
long currentTime = System.currentTimeMillis();
fps.removeIf(elem -> currentTime - elem > 1000);
System.out.println(fps.size());
image.set((BufferedImage) scale(heatChart.getChartImage(), scale));
}
});
VBox box = new VBox();
box.getChildren().add(imageView);
Scene scene = new Scene(box, 1920, 720);
primaryStage.setScene(scene);
primaryStage.show();
generator.setDaemon(true);
generator.start();
AnimationTimer animation = new AnimationTimer() {
#Override
public void handle(long now) {
BufferedImage img = image.getAndSet(null);
if (img != null) {
imageView.setImage(SwingFXUtils.toFXImage(img, null));
}
}
};
animation.start();
}
public static void main(String[] args) {
launch(args);
}
private static Image scale(Image image, int scale) {
BufferedImage res = new BufferedImage(image.getWidth(null) * scale, image.getHeight(null) * scale,
BufferedImage.TYPE_INT_ARGB);
AffineTransform at = new AffineTransform();
at.scale(scale, scale);
AffineTransformOp scaleOp = new AffineTransformOp(at, AffineTransformOp.TYPE_NEAREST_NEIGHBOR);
return scaleOp.filter((BufferedImage) image, res);
}
}
Using the AtomicReference to wrap the buffered image ensures that it is safely shared between the two threads.
On my machine, this generates about 130 images per second; note that not all are displayed, as only the latest one is shown each time the JavaFX graphics framework displays a frame (which is typically throttled at 60fps).
If you want to ensure you show all images that are generated, i.e. you throttle the image generation by the JavaFX framerate, then you can use a BlockingQueue to store the images:
// AtomicReference<BufferedImage> image = new AtomicReference<>();
// Size of the queue is a trade-off between memory consumption
// and smoothness (essentially works as a buffer size)
BlockingQueue<BufferedImage> image = new ArrayBlockingQueue<>(5);
// ...
// image.set((BufferedImage) scale(heatChart.getChartImage(), scale));
try {
image.put((BufferedImage) scale(heatChart.getChartImage(), scale));
} catch (InterruptedException exc) {
Thread.currentThread.interrupt();
}
and
#Override
public void handle(long now) {
BufferedImage img = image.poll();
if (img != null) {
imageView.setImage(SwingFXUtils.toFXImage(img, null));
}
}
The code is pretty inefficient, as you generate a new matrix, new HeatChart, etc, on every iteration. This causes many objects to be created on the heap and quickly discarded, which can cause the GC to be run too often, particularly on a small-memory machine. That said, I ran this with the maximum heap size set at 64MB, (-Xmx64m), and it still performed fine. You may be able to optimize the code, but using the AnimationTimer as shown above, generating images more quickly will not cause any additional stress on the JavaFX framework. I would recommend investigating using the mutability of HeatChart (i.e. setZValues()) to avoid creating too many objects, and/or using PixelBuffer to directly write data to the image view (this would need to be done on the FX Application Thread).
Here's a different example, which (almost) completely minimizes object creation, using one off-screen int[] array to compute data, and one on-screen int[] array to display it. There's a little low-level threading details to ensure the on-screen array is only seen in a consistent state. The on-screen array is used to underly a PixelBuffer, which in turn is used for a WritableImage.
This class generates the image data:
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Consumer;
public class ImageGenerator {
private final int width;
private final int height;
// Keep two copies of the data: one which is not exposed
// that we modify on the fly during computation;
// another which we expose publicly.
// The publicly exposed one can be viewed only in a complete
// state if operations on it are synchronized on this object.
private final int[] privateData ;
private final int[] publicData ;
private final long[] frameTimes ;
private int currentFrameIndex ;
private final AtomicLong averageGenerationTime ;
private final ReentrantLock lock ;
private static final double TWO_PI = 2 * Math.PI;
private static final double PI_BY_TWELVE = Math.PI / 12; // 15 degrees
public ImageGenerator(int width, int height) {
super();
this.width = width;
this.height = height;
privateData = new int[width * height];
publicData = new int[width * height];
lock = new ReentrantLock();
this.frameTimes = new long[100];
this.averageGenerationTime = new AtomicLong();
}
public void generateImage(double angle) {
// compute in private data copy:
int minDim = Math.min(width, height);
int minR2 = minDim * minDim / 4;
for (int x = 0; x < width; x++) {
int xOff = x - width / 2;
int xOff2 = xOff * xOff;
for (int y = 0; y < height; y++) {
int index = x + y * width;
int yOff = y - height / 2;
int yOff2 = yOff * yOff;
int r2 = xOff2 + yOff2;
if (r2 > minR2) {
privateData[index] = 0xffffffff; // white
} else {
double theta = Math.atan2(yOff, xOff);
double delta = Math.abs(theta - angle);
if (delta > TWO_PI - PI_BY_TWELVE) {
delta = TWO_PI - delta;
}
if (delta < PI_BY_TWELVE) {
int green = (int) (255 * (1 - delta / PI_BY_TWELVE));
privateData[index] = (0xff << 24) | (green << 8); // green, fading away from center
} else {
privateData[index] = 0xff << 24; // black
}
}
}
}
// copy computed data to public data copy:
lock.lock();
try {
System.arraycopy(privateData, 0, publicData, 0, privateData.length);
} finally {
lock.unlock();
}
frameTimes[currentFrameIndex] = System.nanoTime() ;
int nextIndex = (currentFrameIndex + 1) % frameTimes.length ;
if (frameTimes[nextIndex] > 0) {
averageGenerationTime.set((frameTimes[currentFrameIndex] - frameTimes[nextIndex]) / frameTimes.length);
}
currentFrameIndex = nextIndex ;
}
public void consumeData(Consumer<int[]> consumer) {
lock.lock();
try {
consumer.accept(publicData);
} finally {
lock.unlock();
}
}
public long getAverageGenerationTime() {
return averageGenerationTime.get() ;
}
}
And here's the UI:
import java.nio.IntBuffer;
import javafx.animation.AnimationTimer;
import javafx.application.Application;
import javafx.scene.Scene;
import javafx.scene.control.Label;
import javafx.scene.image.ImageView;
import javafx.scene.image.PixelFormat;
import javafx.scene.image.PixelWriter;
import javafx.scene.image.WritableImage;
import javafx.scene.layout.BorderPane;
import javafx.stage.Stage;
public class AnimationApp extends Application {
private final int size = 400 ;
private IntBuffer buffer ;
#Override
public void start(Stage primaryStage) throws Exception {
// background image data generation:
ImageGenerator generator = new ImageGenerator(size, size);
// Generate new image data as fast as possible:
Thread thread = new Thread(() -> {
while( true ) {
long now = System.currentTimeMillis() ;
double angle = 2 * Math.PI * (now % 10000) / 10000 - Math.PI;
generator.generateImage(angle);
}
});
thread.setDaemon(true);
thread.start();
generator.consumeData(data -> buffer = IntBuffer.wrap(data));
PixelFormat<IntBuffer> format = PixelFormat.getIntArgbPreInstance() ;
PixelBuffer<IntBuffer> pixelBuffer = new PixelBuffer<>(size, size, buffer, format);
WritableImage image = new WritableImage(pixelBuffer);
BorderPane root = new BorderPane(new ImageView(image));
Label fps = new Label("FPS: ");
root.setTop(fps);
Scene scene = new Scene(root);
primaryStage.setScene(scene);
primaryStage.setTitle("Give me a ping, Vasili. ");
primaryStage.show();
AnimationTimer animation = new AnimationTimer() {
#Override
public void handle(long now) {
// Update image, ensuring we only see the underlying
// data in a consistent state:
generator.consumeData(data -> {
pixelBuffer.updateBuffer(pb -> null);
});
long aveGenTime = generator.getAverageGenerationTime() ;
if (aveGenTime > 0) {
double aveFPS = 1.0 / (aveGenTime / 1_000_000_000.0);
fps.setText(String.format("FPS: %.2f", aveFPS));
}
}
};
animation.start();
}
public static void main(String[] args) {
Application.launch(args);
}
}
For a version that doesn't rely on the JavaFX 13 PixelBuffer, you can just modify this class to use a PixelWriter (AIUI this won't be quite as efficient, but works just as smoothly in this example):
// generator.consumeData(data -> buffer = IntBuffer.wrap(data));
PixelFormat<IntBuffer> format = PixelFormat.getIntArgbPreInstance() ;
// PixelBuffer<IntBuffer> pixelBuffer = new PixelBuffer<>(size, size, buffer, format);
// WritableImage image = new WritableImage(pixelBuffer);
WritableImage image = new WritableImage(size, size);
PixelWriter pixelWriter = image.getPixelWriter() ;
and
AnimationTimer animation = new AnimationTimer() {
#Override
public void handle(long now) {
// Update image, ensuring we only see the underlying
// data in a consistent state:
generator.consumeData(data -> {
// pixelBuffer.updateBuffer(pb -> null);
pixelWriter.setPixels(0, 0, size, size, format, data, 0, size);
});
long aveGenTime = generator.getAverageGenerationTime() ;
if (aveGenTime > 0) {
double aveFPS = 1.0 / (aveGenTime / 1_000_000_000.0);
fps.setText(String.format("FPS: %.2f", aveFPS));
}
}
};

How to recognize an image in another image?

So I'm fairly new to Java, so I'm not sure that this my method of doing this is actually a good idea, but basically I am trying to check for an instance of an image inside another image. So for my testing to see if this would work, I had a 200x148 jpg, got the bytes from that, then got the bytes from a screenshot of the window, and got the bytes from that, and then compared them.
Now, since normally the first image wouldn't be in that screenshot, I opened it in my photos app and put it in while the program was sleeping (before it took the screenshot). And yes, I can confirm that the first image was in the screenshot by looking at the screenshot. However, when I compare the strings (to check if a String with all of the byte data of image one is in a String with all of the byte data of image two), it turns out negative.
Here's the code that I'm trying to use so far:
public static void main(String[] args) throws IOException, HeadlessException, AWTException, InterruptedException {
// Get the first image
ByteArrayOutputStream baos = new ByteArrayOutputStream();
ImageIO.write(ImageIO.read(new File("profile.jpg")), "jpg", baos);
byte[] bytes = baos.toByteArray();
String bytes1S = "";
for (int i = 0; i < bytes.length; i++) {
bytes1S += bytes[i];
}
// Give yourself enough time to open the other image
TimeUnit.SECONDS.sleep(6);
// Take the screenshot
BufferedImage image = new Robot().createScreenCapture(new Rectangle(Toolkit.getDefaultToolkit().getScreenSize()));
ImageIO.write(image, "jpg", new File("screenshot.jpg"));
baos = new ByteArrayOutputStream();
ImageIO.write(ImageIO.read(new File("screenshot.jpg")), "jpg", baos);
byte[] bytes2 = baos.toByteArray();
String bytes2S = "";
for (int i = 0; i < bytes2.length; i++) {
bytes2S += bytes2[i];
}
// Check if the second String of bytes contains the first String of bytes.
if (bytes2S.contains(bytes1S))
System.out.println("Yes");
else
System.out.println("No");
}
And for reference, here's the first image, and the screenshot that it took:
What's the reason behind it not detecting the first image in the screenshot, and is there a better way to do this (preferably without another library)?

A brute-force approach is to simply load both images as BufferedImage objects, and then walk through the "main" image, pixel by pixel, and see if the "sub image" can be found there.
I have implemented this a while ago, and will post the code below as a MCVE.
But note: When you save images as JPG files, then they are compressed, and this compression is lossy. This means that the pixels will not have the perfectly same colors, even if they have been equal on the screen. In the example below, this is solved pragmatically, with a "threshold" that defines how different the pixels may be. But this is a bit arbitrary and not so reliable. (A more robust solution would require more effort).
I'd strongly recommend to save the images as PNG files. They use a lossless compression. So for PNG files, you can set threshold=0 in the code below.
import java.awt.Color;
import java.awt.Graphics;
import java.awt.Point;
import java.awt.image.BufferedImage;
import java.net.URL;
import java.util.function.IntBinaryOperator;
import javax.imageio.ImageIO;
import javax.swing.JFrame;
import javax.swing.JPanel;
import javax.swing.SwingUtilities;
public class FindImageInImage
{
public static void main(String[] args) throws Exception
{
BufferedImage mainImage =
ImageIO.read(new URL("https://i.stack.imgur.com/rEouF.jpg"));
BufferedImage subImage =
ImageIO.read(new URL("https://i.stack.imgur.com/wISyn.jpg"));
int threshold = 100;
Point location = findImageLocation(
mainImage, subImage, threshold);
System.out.println("At " + location);
SwingUtilities.invokeLater(() -> showIt(mainImage, subImage, location));
}
private static void showIt(
BufferedImage mainImage, BufferedImage subImage, Point location)
{
JFrame f = new JFrame();
f.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
f.getContentPane().add(new JPanel()
{
#Override
protected void paintComponent(Graphics g)
{
super.paintComponent(g);
g.drawImage(mainImage, 0, 0, null);
if (location != null)
{
g.setColor(Color.RED);
g.drawRect(location.x, location.y,
subImage.getWidth(), subImage.getHeight());
}
}
});
f.setSize(1500, 800);
f.setLocationRelativeTo(null);
f.setVisible(true);
}
static Point findImageLocation(
BufferedImage mainImage,
BufferedImage subImage,
int threshold)
{
return findImageLocation(mainImage, subImage, (rgb0, rgb1) ->
{
int difference = computeDifference(rgb0, rgb1);
if (difference > threshold)
{
return 1;
}
return 0;
});
}
private static int computeDifference(int rgb0, int rgb1)
{
int r0 = (rgb0 & 0x00FF0000) >> 16;
int g0 = (rgb0 & 0x0000FF00) >> 8;
int b0 = (rgb0 & 0x000000FF);
int r1 = (rgb1 & 0x00FF0000) >> 16;
int g1 = (rgb1 & 0x0000FF00) >> 8;
int b1 = (rgb1 & 0x000000FF);
int dr = Math.abs(r0 - r1);
int dg = Math.abs(g0 - g1);
int db = Math.abs(b0 - b1);
return dr + dg + db;
}
static Point findImageLocation(
BufferedImage mainImage,
BufferedImage subImage,
IntBinaryOperator rgbComparator)
{
int w = mainImage.getWidth();
int h = mainImage.getHeight();
for (int x=0; x < w; x++)
{
for (int y = 0; y < h; y++)
{
if (isSubImageAt(mainImage, x, y, subImage, rgbComparator))
{
return new Point(x, y);
}
}
}
return null;
}
static boolean isSubImageAt(
BufferedImage mainImage, int x, int y,
BufferedImage subImage,
IntBinaryOperator rgbComparator)
{
int w = subImage.getWidth();
int h = subImage.getHeight();
if (x + w > mainImage.getWidth())
{
return false;
}
if (y + h > mainImage.getHeight())
{
return false;
}
for (int ix=0; ix < w; ix++)
{
for (int iy = 0; iy < h; iy++)
{
int mainRgb = mainImage.getRGB(x + ix, y + iy);
int subRgb = subImage.getRGB(ix, iy);
if (rgbComparator.applyAsInt(mainRgb, subRgb) != 0)
{
return false;
}
}
}
return true;
}
}

Convolution - Calculating a Neighbour Element Index for a Vectorised Image

Assume the following matrix acts as both an image and a kernel in a matrix convolution operation:
0 1 2
3 4 5
6 7 8
To calculate the neighbour pixel index you would use the following formula:
neighbourColumn = imageColumn + (maskColumn - centerMaskColumn);
neighbourRow = imageRow + (maskRow - centerMaskRow);
Thus the output of convolution would be:
output1 = {0,1,3,4} x {4,5,7,8} = 58
output2 = {0,1,2,3,4,5} x {3,4,5,6,7,8} = 100
output2 = {1,2,4,5} x {3,4,6,7} = 70
output3 = {0,1,3,4,6,7} x {1,2,4,5,7,8} = 132
output4 = {0,1,2,3,4,5,6,7,8} x {0,1,2,3,4,5,6,7,8} = 204
output5 = {1,2,4,5,7,8} x {0,1,3,4,6,7} = 132
output6 = {3,4,6,7} x {1,2,4,5} = 70
output7 = {3,4,5,6,7,8} x {0,1,2,3,4,5} = 100
output8 = {4,5,7,8} x {0,1,3,4} = 58
Thus the output matrix would be:
58 100 70
132 204 132
70 100 58
Now assume the matrix is flattened to give the following vector:
0 1 2 3 4 5 6 7 8
This vector now acts as an image and a kernel in a vector convolution operation for which the ouput should be:
58 100 70 132 204 132 70 100 58
Given the code below how do you calculate the neighbour element index for the vector such that it corresponds with the same neighbour element in the matrix?
public int[] convolve(int[] image, int[] kernel)
{
int imageValue;
int kernelValue;
int outputValue;
int[] outputImage = new int[image.length()];
// loop through image
for(int i = 0; i < image.length(); i++)
{
outputValue = 0;
// loop through kernel
for(int j = 0; j < kernel.length(); j++)
{
neighbour = ?;
// discard out of bound neighbours
if (neighbour >= 0 && neighbour < imageSize)
{
imageValue = image[neighbour];
kernelValue = kernel[j];
outputValue += imageValue * kernelValue;
}
}
outputImage[i] = outputValue;
}
return output;
}

The neighbour index is computed by offsetting the original pixel index by the difference between the index of the current element and half the size of the matrix. For example, to compute the column index:
int neighbourCol = imageCol + col - (size / 2);
I put a working demo on GitHub, trying to keep the whole convolution algorithm as readable as possible:
int[] dstImage = new int[srcImage.width() * srcImage.height()];
srcImage.forEachElement((image, imageCol, imageRow) -> {
Pixel pixel = new Pixel();
forEachElement((filter, col, row) -> {
int neighbourCol = imageCol + col - (size / 2);
int neighbourRow = imageRow + row - (size / 2);
if (srcImage.hasElementAt(neighbourCol, neighbourRow)) {
int color = srcImage.at(neighbourCol, neighbourRow);
int weight = filter.at(col, row);
pixel.addWeightedColor(color, weight);
}
});
dstImage[(imageRow * srcImage.width() + imageCol)] = pixel.rgb();
});

As you are dealing with 2D images, you will have to retain some information about the images, in addition the the plain 1D pixel array. Particularly, you at least need the width of the image (and the mask) in order to find out which indices in the 1D array correspond to which indices in the original 2D image. And as already pointed out by Raffaele in his answer, there are general rules for the conversions between these ("virtual") 2D coordinates and 1D coordinates in such a pixel array:
int pixelX = ...;
int pixelY = ...;
int index = pixelX + pixelY * imageSizeX;
Based on this, you can do your convolution simply on a 2D image. The limits for the pixels that you may access may easily be checked. The loops are simple 2D loops over the image and the mask. It all boils down to the point where you access the 1D data with the 2D coordinates, as described above.
Here is an example. It applies a Sobel filter to the input image. (There may still be something odd with the pixel values, but the convolution itself and the index computations should be right)
import java.awt.Graphics2D;
import java.awt.GridLayout;
import java.awt.image.BufferedImage;
import java.awt.image.DataBuffer;
import java.awt.image.DataBufferByte;
import java.io.File;
import java.io.IOException;
import javax.imageio.ImageIO;
import javax.swing.ImageIcon;
import javax.swing.JFrame;
import javax.swing.JLabel;
import javax.swing.SwingUtilities;
public class ConvolutionWithArrays1D
{
public static void main(String[] args) throws IOException
{
final BufferedImage image =
asGrayscaleImage(ImageIO.read(new File("lena512color.png")));
SwingUtilities.invokeLater(new Runnable()
{
#Override
public void run()
{
createAndShowGUI(image);
}
});
}
private static void createAndShowGUI(BufferedImage image0)
{
JFrame f = new JFrame();
f.getContentPane().setLayout(new GridLayout(1,2));
f.getContentPane().add(new JLabel(new ImageIcon(image0)));
BufferedImage image1 = compute(image0);
f.getContentPane().add(new JLabel(new ImageIcon(image1)));
f.pack();
f.setLocationRelativeTo(null);
f.setVisible(true);
}
private static BufferedImage asGrayscaleImage(BufferedImage image)
{
BufferedImage gray = new BufferedImage(
image.getWidth(), image.getHeight(), BufferedImage.TYPE_BYTE_GRAY);
Graphics2D g = gray.createGraphics();
g.drawImage(image, 0, 0, null);
g.dispose();
return gray;
}
private static int[] obtainGrayscaleIntArray(BufferedImage image)
{
BufferedImage gray = new BufferedImage(
image.getWidth(), image.getHeight(), BufferedImage.TYPE_BYTE_GRAY);
Graphics2D g = gray.createGraphics();
g.drawImage(image, 0, 0, null);
g.dispose();
DataBuffer dataBuffer = gray.getRaster().getDataBuffer();
DataBufferByte dataBufferByte = (DataBufferByte)dataBuffer;
byte data[] = dataBufferByte.getData();
int result[] = new int[data.length];
for (int i=0; i<data.length; i++)
{
result[i] = data[i];
}
return result;
}
private static BufferedImage createImageFromGrayscaleIntArray(
int array[], int imageSizeX, int imageSizeY)
{
BufferedImage gray = new BufferedImage(
imageSizeX, imageSizeY, BufferedImage.TYPE_BYTE_GRAY);
DataBuffer dataBuffer = gray.getRaster().getDataBuffer();
DataBufferByte dataBufferByte = (DataBufferByte)dataBuffer;
byte data[] = dataBufferByte.getData();
for (int i=0; i<data.length; i++)
{
data[i] = (byte)array[i];
}
return gray;
}
private static BufferedImage compute(BufferedImage image)
{
int imagePixels[] = obtainGrayscaleIntArray(image);
int mask[] =
{
1,0,-1,
2,0,-2,
1,0,-1,
};
int outputPixels[] =
Convolution.filter(imagePixels, image.getWidth(), mask, 3);
return createImageFromGrayscaleIntArray(
outputPixels, image.getWidth(), image.getHeight());
}
}
class Convolution
{
public static final int[] filter(
final int[] image, int imageSizeX,
final int[] mask, int maskSizeX)
{
int imageSizeY = image.length / imageSizeX;
int maskSizeY = mask.length / maskSizeX;
int output[] = new int[image.length];
for (int y=0; y<imageSizeY; y++)
{
for (int x=0; x<imageSizeX; x++)
{
int outputPixelValue = 0;
for (int my=0; my< maskSizeY; my++)
{
for (int mx=0; mx< maskSizeX; mx++)
{
int neighborX = x + mx -maskSizeX / 2;
int neighborY = y + my -maskSizeY / 2;
if (neighborX >= 0 && neighborX < imageSizeX &&
neighborY >= 0 && neighborY < imageSizeY)
{
int imageIndex =
neighborX + neighborY * imageSizeX;
int maskIndex = mx + my * maskSizeX;
int imagePixelValue = image[imageIndex];
int maskPixelValue = mask[maskIndex];
outputPixelValue +=
imagePixelValue * maskPixelValue;
}
}
}
outputPixelValue = truncate(outputPixelValue);
int outputIndex = x + y * imageSizeX;
output[outputIndex] = outputPixelValue;
}
}
return output;
}
private static final int truncate(final int pixelValue)
{
return Math.min(255, Math.max(0, pixelValue));
}
}

BufferedImage, int[] pixels and rendering. How do they work they work together?

There is something in the following code that I am unable to understand. After digging through google for a while, I decided it would be better to ask someone.
I am following a game programming tutorial on youtube, and I feel I understand (to some degree) everything I have written, except for some lines which concern the rendering part of the program.
package com.thomas.game;
import java.awt.Canvas;
import java.awt.Dimension;
import java.awt.Graphics;
import java.awt.image.BufferStrategy;
import java.awt.image.BufferedImage;
import java.awt.image.DataBufferInt;
import javax.swing.JFrame;
import com.thomas.game.graphics.Screen;
import com.thomas.game.input.Keyboard;
public class Game extends Canvas implements Runnable {
private static final int WIDTH = 300;
private static final int HEIGHT = (WIDTH / 16) * 9;
private static final int SCALE = 3;
private static final String TITLE = "Game";
private JFrame frame;
private Thread thread;
private Screen screen;
private BufferedImage image;
private Keyboard key;
private int[] pixels;
private boolean running = false;
private int x = 0, y = 0;
public Game() {
setPreferredSize(new Dimension(WIDTH * SCALE, HEIGHT * SCALE));
screen = new Screen(WIDTH, HEIGHT);
frame = new JFrame();
initializeFrame();
image = new BufferedImage(WIDTH, HEIGHT, BufferedImage.TYPE_INT_RGB);
pixels = ((DataBufferInt)image.getRaster().getDataBuffer()).getData();
this is what I don't understand. I get the image raster, from which I get the databuffer. I typecast that databuffer into a (DatabufferInt), which allows me to retrieve an int[] through the getData() method. After this is done, pixel.length has a value of 48600, and every index contains the int value 0. Operating with this int[] makes the program render like it is supposed to. However, if I don't typecast and retrieve the int[] in the above manner, and instead say pixels = new int[48600], i end up with a black screen.
I guess what I want to know is: what is the difference between these two int[], or rather, what makes the first one work? How does it work?
key = new Keyboard();
addKeyListener(key);
setFocusable(true);
}
public void run() {
long lastTime = System.nanoTime();
double nsPerTick = 1E9/60;
double delta = 0;
long now;
int ticks = 0;
int frames = 0;
long timer = System.currentTimeMillis();
while(running) {
now = System.nanoTime();
delta += (now - lastTime) / nsPerTick;
lastTime = now;
while(delta >= 1) {
tick();
ticks++;
delta--;
}
render();
frames++;
if(System.currentTimeMillis() - timer >= 1000) {
timer += 1000;
frame.setTitle(TITLE + " | ups: " + ticks + " fps: " + frames);
ticks = 0;
frames = 0;
}
}
}
private void render() {
BufferStrategy bs = getBufferStrategy(); // retrieves the bufferstrategy from the current component (the instance of Game that calls this method)
if(bs == null) {
createBufferStrategy(3);
return;
}
screen.clear();
screen.render(x, y);
getPixels();
Graphics g = bs.getDrawGraphics(); // retrieves a graphics object from the next in line buffer in the bufferstrategy, this graphics object draws to that buffer
g.drawImage(image, 0, 0, getWidth(), getHeight(), null); // draws the bufferedimage to the available buffer
g.dispose();
bs.show(); // orders the next in line buffer (which the graphics object g is tied to) to show its contents on the canvas
}
private void tick() {
key.update();
if(key.up)
y--;
if(key.down)
y++;
if(key.left)
x--;
if(key.right)
x++;
}
public void initializeFrame() {
frame.setTitle(TITLE);
frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
frame.setResizable(false);
frame.add(this);
frame.pack();
frame.setLocationRelativeTo(null);
frame.setVisible(true);
}
public synchronized void start() {
running = true;
thread = new Thread(this);
thread.start();
}
public synchronized void stop() {
running = false;
try {
thread.join();
} catch(InterruptedException e) {
e.printStackTrace();
}
}
public static void main(String[] args) {
Game game = new Game();
game.start();
}
public void getPixels() {
for(int i = 0; i < pixels.length; i++)
pixels[i] = screen.pixels[i];
}
}
It seems like the bufferedimage gets values from the pixels array. But I don't understand how these two communicate, or how they are connected. I haven't explicitly told the bufferedimage to get its pixels from the pixels array, so how does it know?
I will also attach the Screen class, which is responsible for updating the pixels array.
package com.thomas.game.graphics;
import java.util.Random;
public class Screen {
private int width, height;
public int[] pixels;
private final int MAP_SIZE = 64;
private final int MAP_SIZE_MASK = MAP_SIZE - 1;
private int[] tiles;
private int tileIndex;
private int xx, yy;
private Random r;
public Screen(int w, int h) {
width = w;
height = h;
pixels = new int[width * height];
tiles = new int[MAP_SIZE * MAP_SIZE];
r = new Random(0xffffff);
for(int i = 0; i < tiles.length; i++) {
tiles[i] = r.nextInt();
}
tiles[0] = 0;
}
public void clear() {
for(int i = 0; i < pixels.length; i++)
pixels[i] = 0;
}
public void render(int xOffset, int yOffset) {
for(int y = 0; y < height; y++) {
yy = y + yOffset;
for(int x = 0; x < width; x++) {
xx = x + xOffset;
tileIndex = (yy >> 4 & MAP_SIZE_MASK) * MAP_SIZE + (xx >> 4 & MAP_SIZE_MASK);
pixels[y * width + x] = tiles[tileIndex];
}
}
}
}
I really hope someone can explain this to me, it would be greatly appreciated. The program is working like it is supposed to, but I don't feel comfortable continuing on the tutorial until I grasp this.

Basic types like short, int, long etc are not Objects.
However, int[] is an array. Arrays are objects in java. Java manipulates objects by reference, not value.
In this line you are not creating a new object. You are storing a reference to the object int[] in your variable pixels. Anything you change in pixels, gets changed inside of the int[] object in image:
pixels = ((DataBufferInt)image.getRaster().getDataBuffer()).getData();
I've created an example, try running this code:
public class Data {
private int[] values = {25,14};
public int[] getValues() {
return values;
}
public static void main(String[] args) {
Data d = new Data();
System.out.println(d.getValues()[0]);
int[] values = d.getValues();
values[0] = 15;
System.out.println(d.getValues()[0]);
}
}
Output:
25
15

Note that you have this code...
pixels = ((DataBufferInt)image.getRaster().getDataBuffer()).getData();
while it should be like this...
pixels = ((DataBufferInt)img.getRaster().getDataBuffer()).getData();
Change image to img.
Hope it works!