Develop Reference

Java Backpropagation Algorithm is very slow

I have a big problem. I try to create a neural network and want to train it with a backpropagation algorithm. I found this tutorial here http://mattmazur.com/2015/03/17/a-step-by-step-backpropagation-example/ and tried to recreate it in Java. And when I use the training data he uses, I get the same results as him.
Without backpropagation my TotalError is nearly the same as his. And when I use the back backpropagation 10 000 time like him, than I get the nearly the same error. But he uses 2 Input Neurons, 2 Hidden Neurons and 2 Outputs but I'd like to use this neural network for OCR, so I need definitely more Neurons. But if I use for example 49 Input Neurons, 49 Hidden Neurons and 2 Output Neurons, It takes very long to change the weights to get a small error. (I believe it takes forever.....). I have a learningRate of 0.5. In the constructor of my network, I generate the neurons and give them the same training data like the one in the tutorial and for testing it with more neurons, I gave them random weights, inputs and targets. So can't I use this for many Neurons, does it takes just very long or is something wrong with my code ? Shall I increase the learning rate, the bias or the start weight?
Hopefully you can help me.
package de.Marcel.NeuralNetwork;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Random;
public class Network {
private ArrayList<Neuron> inputUnit, hiddenUnit, outputUnit;
private double[] inHiWeigth, hiOutWeigth;
private double hiddenBias, outputBias;
private double learningRate;
public Network(double learningRate) {
this.inputUnit = new ArrayList<Neuron>();
this.hiddenUnit = new ArrayList<Neuron>();
this.outputUnit = new ArrayList<Neuron>();
this.learningRate = learningRate;
generateNeurons(2,2,2);
calculateTotalNetInputForHiddenUnit();
calculateTotalNetInputForOutputUnit();
}
public double calcuteLateTotalError () {
double e = 0;
for(Neuron n : outputUnit) {
e += 0.5 * Math.pow(Math.max(n.getTarget(), n.getOutput()) - Math.min(n.getTarget(), n.getOutput()), 2.0);
}
return e;
}
private void generateNeurons(int input, int hidden, int output) {
// generate inputNeurons
for (int i = 0; i < input; i++) {
Neuron neuron = new Neuron();
// for testing give each neuron an input
if(i == 0) {
neuron.setInput(0.05d);
} else if(i == 1) {
neuron.setOutput(0.10d);
}
inputUnit.add(neuron);
}
// generate hiddenNeurons
for (int i = 0; i < hidden; i++) {
Neuron neuron = new Neuron();
hiddenUnit.add(neuron);
}
// generate outputNeurons
for (int i = 0; i < output; i++) {
Neuron neuron = new Neuron();
if(i == 0) {
neuron.setTarget(0.01d);
} else if(i == 1) {
neuron.setTarget(0.99d);
}
outputUnit.add(neuron);
}
// generate Bias
hiddenBias = 0.35;
outputBias = 0.6;
// generate connections
double startWeigth = 0.15;
// generate inHiWeigths
inHiWeigth = new double[inputUnit.size() * hiddenUnit.size()];
for (int i = 0; i < inputUnit.size() * hiddenUnit.size(); i += hiddenUnit.size()) {
for (int x = 0; x < hiddenUnit.size(); x++) {
int z = i + x;
inHiWeigth[z] = round(startWeigth, 2, BigDecimal.ROUND_HALF_UP);
startWeigth += 0.05;
}
}
// generate hiOutWeigths
hiOutWeigth = new double[hiddenUnit.size() * outputUnit.size()];
startWeigth += 0.05;
for (int i = 0; i < hiddenUnit.size() * outputUnit.size(); i += outputUnit.size()) {
for (int x = 0; x < outputUnit.size(); x++) {
int z = i + x;
hiOutWeigth[z] = round(startWeigth, 2, BigDecimal.ROUND_HALF_UP);
startWeigth += 0.05;
}
}
}
private double round(double unrounded, int precision, int roundingMode)
{
BigDecimal bd = new BigDecimal(unrounded);
BigDecimal rounded = bd.setScale(precision, roundingMode);
return rounded.doubleValue();
}
private void calculateTotalNetInputForHiddenUnit() {
// calculate totalnetinput for each hidden neuron
for (int s = 0; s < hiddenUnit.size(); s++) {
double net = 0;
int x = (inHiWeigth.length / inputUnit.size());
// calculate toAdd
for (int i = 0; i < x; i++) {
int v = i + s * x;
double weigth = inHiWeigth[v];
double toAdd = weigth * inputUnit.get(i).getInput();
net += toAdd;
}
// add bias
net += hiddenBias * 1;
net = net *-1;
double output = (1.0 / (1.0 + (double)Math.exp(net)));
hiddenUnit.get(s).setOutput(output);
}
}
private void calculateTotalNetInputForOutputUnit() {
// calculate totalnetinput for each hidden neuron
for (int s = 0; s < outputUnit.size(); s++) {
double net = 0;
int x = (hiOutWeigth.length / hiddenUnit.size());
// calculate toAdd
for (int i = 0; i < x; i++) {
int v = i + s * x;
double weigth = hiOutWeigth[v];
double outputOfH = hiddenUnit.get(s).getOutput();
double toAdd = weigth * outputOfH;
net += toAdd;
}
// add bias
net += outputBias * 1;
net = net *-1;
double output = (double) (1.0 / (1.0 + Math.exp(net)));
outputUnit.get(s).setOutput(output);
}
}
private void backPropagate() {
// calculate ouputNeuron weigthChanges
double[] oldWeigthsHiOut = hiOutWeigth;
double[] newWeights = new double[hiOutWeigth.length];
for (int i = 0; i < hiddenUnit.size(); i += 1) {
double together = 0;
double[] newOuts = new double[hiddenUnit.size()];
for (int x = 0; x < outputUnit.size(); x++) {
int z = x * hiddenUnit.size() + i;
double weigth = oldWeigthsHiOut[z];
double target = outputUnit.get(x).getTarget();
double output = outputUnit.get(x).getOutput();
double totalErrorChangeRespectOutput = -(target - output);
double partialDerivativeLogisticFunction = output * (1 - output);
double totalNetInputChangeWithRespect = hiddenUnit.get(x).getOutput();
double puttedAllTogether = totalErrorChangeRespectOutput * partialDerivativeLogisticFunction
* totalNetInputChangeWithRespect;
double weigthChange = weigth - learningRate * puttedAllTogether;
// set new weigth
newWeights[z] = weigthChange;
together += (totalErrorChangeRespectOutput * partialDerivativeLogisticFunction * weigth);
double out = hiddenUnit.get(x).getOutput();
newOuts[x] = out * (1.0 - out);
}
for (int t = 0; t < newOuts.length; t++) {
inHiWeigth[t + i] = (double) (inHiWeigth[t + i] - learningRate * (newOuts[t] * together * inputUnit.get(t).getInput()));
}
hiOutWeigth = newWeights;
}
}
}
And my Neuron Class:
package de.Marcel.NeuralNetwork;
public class Neuron {
private double input, output;
private double target;
public Neuron () {
}
public void setTarget(double target) {
this.target = target;
}
public void setInput (double input) {
this.input = input;
}
public void setOutput(double output) {
this.output = output;
}
public double getInput() {
return input;
}
public double getOutput() {
return output;
}
public double getTarget() {
return target;
}
}

Think about it: you have 10,000 propagations through 49->49->2 neurons. Between the input layer and the hidden layer, you have 49 * 49 links to propagate through, so parts of your code are being executed about 24 million times (10,000 * 49 * 49). That is going to take time. You could try 100 propogations, and see how long it takes, just to give you an idea.
There are a few things that can be done to increase performance, like using a plain array instead of an ArrayList, but this is a better topic for the Code Review site. Also, don't expect this to give drastic improvements.

Your back propagation code has complexity of O(h*o + h^2) * 10000, where h is the number of hidden neurons and o is the number of output neurons. Here's why.
You have a loop that executes for all of your hidden neurons...
for (int i = 0; i < hiddenUnit.size(); i += 1) {
... containing another loop that executes for all the output neurons...
for (int x = 0; x < outputUnit.size(); x++) {
... and an additional inner loop that executes again for all the hidden neurons...
double[] newOuts = new double[hiddenUnit.size()];
for (int t = 0; t < newOuts.length; t++) {
... and you execute all of that ten thousand times. Add on top of this O(i + h + o) [initial object creation] + O(i*h + o*h) [initial weights] + O(h*i) [calculate net inputs] + O(h*o) [calculate net outputs].
No wonder it's taking forever; your code is littered with nested loops. If you want it to go faster, factor these out - for example, combine object creation and initialization - or reduce the number of neurons. But significantly cutting the number of back propagation calls is the best way to make this run faster.

Dynamic programming approach to TSP in Java

I'm a beginner, and I'm trying to write a working travelling salesman problem using dynamic programming approach.
This is the code for my compute function:
public static int compute(int[] unvisitedSet, int dest) {
if (unvisitedSet.length == 1)
return distMtx[dest][unvisitedSet[0]];
int[] newSet = new int[unvisitedSet.length-1];
int distMin = Integer.MAX_VALUE;
for (int i = 0; i < unvisitedSet.length; i++) {
for (int j = 0; j < newSet.length; j++) {
if (j < i) newSet[j] = unvisitedSet[j];
else newSet[j] = unvisitedSet[j+1];
}
int distCur;
if (distMtx[dest][unvisitedSet[i]] != -1) {
distCur = compute(newSet, unvisitedSet[i]) + distMtx[unvisitedSet[i]][dest];
if (distMin > distCur)
distMin = distCur;
}
else {
System.out.println("No path between " + dest + " and " + unvisitedSet[i]);
}
}
return distMin;
}
The code is not giving me the correct answers, and I'm trying to figure out where the error is occurring. I think my error occurs when I add:
distCur = compute(newSet, unvisitedSet[i]) + distMtx[unvisitedSet[i]][dest];
So I've been messing around with that part, moving the addition to the very end right before I return distMin and so on... But I couldn't figure it out.
Although I'm sure it can be inferred from the code, I will state the following facts to clarify.
distMtx stores all the intercity distances, and distances are symmetric, meaning if distance from city A to city B is 3, then the distance from city B to city A is also 3. Also, if two cities don't have any direct paths, the distance value is -1.
Any help would be very much appreciated!
Thanks!
Edit:
The main function reads the intercity distances from a text file. Because I'm assuming the number of cities will always be less than 100, global int variable distMtx is [100][100].
Once the matrix is filled with the necessary information, an array of all the cities are created. The names of the cities are basically numbers. So if I have 4 cities, set[4] = {0, 1, 2, 3}.
In the main function, after distMtx and set is created, first call to compute() is called:
int optRoute = compute(set, 0);
System.out.println(optRoute);
Sample input:
-1 3 2 7
3 -1 10 1
2 10 -1 4
7 1 4 -1
Expected output:
10

Here's a working iterative solution to the TSP with dynamic programming. What would make your life easier is to store the current state as a bitmask instead of in an array. This has the advantage that the state representation is compact and can be cached easily.
I made a video detailing the solution to this problem on Youtube, please enjoy! Code was taken from my github repo
/**
* An implementation of the traveling salesman problem in Java using dynamic
* programming to improve the time complexity from O(n!) to O(n^2 * 2^n).
*
* Time Complexity: O(n^2 * 2^n)
* Space Complexity: O(n * 2^n)
*
**/
import java.util.List;
import java.util.ArrayList;
import java.util.Collections;
public class TspDynamicProgrammingIterative {
private final int N, start;
private final double[][] distance;
private List<Integer> tour = new ArrayList<>();
private double minTourCost = Double.POSITIVE_INFINITY;
private boolean ranSolver = false;
public TspDynamicProgrammingIterative(double[][] distance) {
this(0, distance);
}
public TspDynamicProgrammingIterative(int start, double[][] distance) {
N = distance.length;
if (N <= 2) throw new IllegalStateException("N <= 2 not yet supported.");
if (N != distance[0].length) throw new IllegalStateException("Matrix must be square (n x n)");
if (start < 0 || start >= N) throw new IllegalArgumentException("Invalid start node.");
this.start = start;
this.distance = distance;
}
// Returns the optimal tour for the traveling salesman problem.
public List<Integer> getTour() {
if (!ranSolver) solve();
return tour;
}
// Returns the minimal tour cost.
public double getTourCost() {
if (!ranSolver) solve();
return minTourCost;
}
// Solves the traveling salesman problem and caches solution.
public void solve() {
if (ranSolver) return;
final int END_STATE = (1 << N) - 1;
Double[][] memo = new Double[N][1 << N];
// Add all outgoing edges from the starting node to memo table.
for (int end = 0; end < N; end++) {
if (end == start) continue;
memo[end][(1 << start) | (1 << end)] = distance[start][end];
}
for (int r = 3; r <= N; r++) {
for (int subset : combinations(r, N)) {
if (notIn(start, subset)) continue;
for (int next = 0; next < N; next++) {
if (next == start || notIn(next, subset)) continue;
int subsetWithoutNext = subset ^ (1 << next);
double minDist = Double.POSITIVE_INFINITY;
for (int end = 0; end < N; end++) {
if (end == start || end == next || notIn(end, subset)) continue;
double newDistance = memo[end][subsetWithoutNext] + distance[end][next];
if (newDistance < minDist) {
minDist = newDistance;
}
}
memo[next][subset] = minDist;
}
}
}
// Connect tour back to starting node and minimize cost.
for (int i = 0; i < N; i++) {
if (i == start) continue;
double tourCost = memo[i][END_STATE] + distance[i][start];
if (tourCost < minTourCost) {
minTourCost = tourCost;
}
}
int lastIndex = start;
int state = END_STATE;
tour.add(start);
// Reconstruct TSP path from memo table.
for (int i = 1; i < N; i++) {
int index = -1;
for (int j = 0; j < N; j++) {
if (j == start || notIn(j, state)) continue;
if (index == -1) index = j;
double prevDist = memo[index][state] + distance[index][lastIndex];
double newDist = memo[j][state] + distance[j][lastIndex];
if (newDist < prevDist) {
index = j;
}
}
tour.add(index);
state = state ^ (1 << index);
lastIndex = index;
}
tour.add(start);
Collections.reverse(tour);
ranSolver = true;
}
private static boolean notIn(int elem, int subset) {
return ((1 << elem) & subset) == 0;
}
// This method generates all bit sets of size n where r bits
// are set to one. The result is returned as a list of integer masks.
public static List<Integer> combinations(int r, int n) {
List<Integer> subsets = new ArrayList<>();
combinations(0, 0, r, n, subsets);
return subsets;
}
// To find all the combinations of size r we need to recurse until we have
// selected r elements (aka r = 0), otherwise if r != 0 then we still need to select
// an element which is found after the position of our last selected element
private static void combinations(int set, int at, int r, int n, List<Integer> subsets) {
// Return early if there are more elements left to select than what is available.
int elementsLeftToPick = n - at;
if (elementsLeftToPick < r) return;
// We selected 'r' elements so we found a valid subset!
if (r == 0) {
subsets.add(set);
} else {
for (int i = at; i < n; i++) {
// Try including this element
set |= 1 << i;
combinations(set, i + 1, r - 1, n, subsets);
// Backtrack and try the instance where we did not include this element
set &= ~(1 << i);
}
}
}
public static void main(String[] args) {
// Create adjacency matrix
int n = 6;
double[][] distanceMatrix = new double[n][n];
for (double[] row : distanceMatrix) java.util.Arrays.fill(row, 10000);
distanceMatrix[5][0] = 10;
distanceMatrix[1][5] = 12;
distanceMatrix[4][1] = 2;
distanceMatrix[2][4] = 4;
distanceMatrix[3][2] = 6;
distanceMatrix[0][3] = 8;
int startNode = 0;
TspDynamicProgrammingIterative solver = new TspDynamicProgrammingIterative(startNode, distanceMatrix);
// Prints: [0, 3, 2, 4, 1, 5, 0]
System.out.println("Tour: " + solver.getTour());
// Print: 42.0
System.out.println("Tour cost: " + solver.getTourCost());
}
}

I know this is pretty old question but it might help somebody in the future.
Here is very well written paper on TSP with dynamic programming approach
https://github.com/evandrix/SPOJ/blob/master/DP_Main112/Solving-Traveling-Salesman-Problem-by-Dynamic-Programming-Approach-in-Java.pdf

I think you have to make some changes in your program.
Here there is an implementation
http://www.sanfoundry.com/java-program-implement-traveling-salesman-problem-using-nearest-neighbour-algorithm/

NumberOfDiscIntersections overflow in codility test

In the codility test NumberOfDiscIntersections I am getting perf 100% and correctness 87% with the one test failing being
overflow
arithmetic overflow tests
got -1 expected 2
I can't see what is causing that given that I am using long which is 64-bit. And even if I can get it to 100% perf 100% correctness I am wondering if there is a better way to do this that is not as verbose in Java.
edit: figured out a much better way to do with with two arrays rather than a pair class
// you can also use imports, for example:
import java.util.*;
// you can use System.out.println for debugging purposes, e.g.
// System.out.println("this is a debug message");
class Solution {
public int solution(int[] A) {
int j = 0;
Pair[] arr = new Pair[A.length * 2];
for (int i = 0; i < A.length; i++) {
Pair s = new Pair(i - A[i], true);
arr[j] = s;
j++;
Pair e = new Pair(i + A[i], false);
arr[j] = e;
j++;
}
Arrays.sort(arr, new Pair(0, true));
long numIntersect = 0;
long currentCount = 0;
for (Pair p: arr) {
if (p.start) {
numIntersect += currentCount;
if (numIntersect > 10000000) {
return -1;
}
currentCount++;
} else {
currentCount--;
}
}
return (int) numIntersect;
}
static private class Pair implements Comparator<Pair> {
private long x;
private boolean start;
public Pair(long x, boolean start) {
this.x = x;
this.start = start;
}
public int compare(Pair p1, Pair p2) {
if (p1.x < p2.x) {
return -1;
} else if (p1.x > p2.x) {
return 1;
} else {
if (p1.start && p2.start == false) {
return -1;
} else if (p1.start == false && p2.start) {
return 1;
} else {
return 0;
}
}
}
}
}

Look at this line:
Pair s = new Pair(i + A[i], true);
This is equivalent with Pair s = new Pair((long)(i + A[i]) , true);
As i is integer, and A[i] is also integer, so this can cause overflow, as value in A[i] can go up to Integer.MAX_VALUE, and the cast to long happened after add operation is completed.
To fix:
Pair s = new Pair((long)i + (long)A[i], true);
Note: I have submitted with my fixed and got 100%
https://codility.com/demo/results/demoRRBY3Q-UXH/

My todays solution. O(N) time complexity. Simple assumption that number of availble pairs in next point of the table is difference between total open circle to that moment (circle) and circles that have been processed before. Maybe it's to simple :)
public int solution04(int[] A) {
final int N = A.length;
final int M = N + 2;
int[] left = new int[M]; // values of nb of "left" edges of the circles in that point
int[] sleft = new int[M]; // prefix sum of left[]
int il, ir; // index of the "left" and of the "right" edge of the circle
for (int i = 0; i < N; i++) { // counting left edges
il = tl(i, A);
left[il]++;
}
sleft[0] = left[0];
for (int i = 1; i < M; i++) {// counting prefix sums for future use
sleft[i]=sleft[i-1]+left[i];
}
int o, pairs, total_p = 0, total_used=0;
for (int i = 0; i < N; i++) { // counting pairs
ir = tr(i, A, M);
o = sleft[ir]; // nb of open till right edge
pairs = o -1 - total_used;
total_used++;
total_p += pairs;
}
if(total_p > 10000000){
total_p = -1;
}
return total_p;
}
int tl(int i, int[] A){
int tl = i - A[i]; // index of "begin" of the circle
if (tl < 0) {
tl = 0;
} else {
tl = i - A[i] + 1;
}
return tl;
}
int tr(int i, int[] A, int M){
int tr; // index of "end" of the circle
if (Integer.MAX_VALUE - i < A[i] || i + A[i] >= M - 1) {
tr = M - 1;
} else {
tr = i + A[i] + 1;
}
return tr;
}

My take on this, O(n):
public int solution(int[] A) {
int[] startPoints = new int[A.length];
int[] endPoints = new int[A.length];
int tempPoint;
int currOpenCircles = 0;
long pairs = 0;
//sum of starting and end points - how many circles open and close at each index?
for(int i = 0; i < A.length; i++){
tempPoint = i - A[i];
startPoints[tempPoint < 0 ? 0 : tempPoint]++;
tempPoint = i + A[i];
if(A[i] < A.length && tempPoint < A.length) //first prevents int overflow, second chooses correct point
endPoints[tempPoint]++;
}
//find all pairs of new circles (combinations), then make pairs with exiting circles (multiplication)
for(int i = 0; i < A.length; i++){
if(startPoints[i] >= 2)
pairs += (startPoints[i] * (startPoints[i] - 1)) / 2;
pairs += currOpenCircles * startPoints[i];
currOpenCircles += startPoints[i];
currOpenCircles -= endPoints[i];
if(pairs > 10000000)
return -1;
}
return (int) pairs;
}

The explanation to Helsing's solution part:
if(startPoints[i] >= 2) pairs += (startPoints[i] * (startPoints[i] - 1)) / 2;
is based on mathematical combinations formula:
Cn,m = n! / ((n-m)!.m!
for pairs, m=2 then:
Cn,2 = n! / ((n-2)!.2
Equal to:
Cn,2 = n.(n-1).(n-2)! / ((n-2)!.2
By simplification:
Cn,2 = n.(n-1) / 2

Not a very good performance, but using streams.
List<Long> list = IntStream.range(0, A.length).mapToObj(i -> Arrays.asList((long) i - A[i], (long) i + A[i]))
.sorted((o1, o2) -> {
int f = o1.get(0).compareTo(o2.get(0));
return f == 0 ? o1.get(1).compareTo(o2.get(1)) : f;
})
.collect(ArrayList<Long>::new,
(acc, val) -> {
if (acc.isEmpty()) {
acc.add(0l);
acc.add(val.get(1));
} else {
Iterator it = acc.iterator();
it.next();
while (it.hasNext()) {
long el = (long) it.next();
if (val.get(0) <= el) {
long count = acc.get(0);
acc.set(0, ++count);
} else {
it.remove();
}
}
acc.add(val.get(1));
}
},
ArrayList::addAll);
return (int) (list.isEmpty() ? 0 : list.get(0) > 10000000 ? -1 : list.get(0));

This one in Python passed all "Correctness tests" and failed all "Performance tests" due to O(n²), so I got 50% score. But it is very simple to understand. I just used the right radius (maximum) and checked if it was bigger or equal to the left radius (minimum) of the next circles. I also avoided to use sort and did not check twice the same circle. Later I will try to improve performance, but the problem here for me was the algorithm. I tried to find a very easy solution to help explain the concept. Maybe this will help someone.
def solution(A):
n = len(A)
cnt = 0
for j in range(1,n):
for i in range(n-j):
if(i+A[i]>=i+j-A[i+j]):
cnt+=1
if(cnt>1e7):
return -1
return cnt

Implementation of Temporal Difference Learning in Java

The below code is my implementation of temporal difference learning. The agent who uses the TD algorithm plays more than 750,000 games against an agent that uses mini-max procedure to play the game, But the problem is the TD-agent does not learn... What is wrong with this implementation?
updateToNextState is called when the agent choose a next move.
public void updateToNextState(int[] currentState, double[] nextStateOutput) {
double[] outputOfNext = nextStateOutput;
double[] outputOfCurrent = getOutput(currentState);
double[] error = getDifferenceOfOutputs(outputOfNext, outputOfCurrent);
lastHandledState = currentState;
for (int j = 0; j < layers[HIDDEN].neurons.length; j++) {
for (int k = 0; k < layers[OUTPUT].neurons.length; k++) {
double toBeUpdatedValueForJToK = BETA * error[k]
* eligibilityTraces.getEjk(j, k);
layers[HIDDEN].neurons[j].updateWeightToNeuron(
layers[OUTPUT].neurons[k].getNeuronId(),
toBeUpdatedValueForJToK);
for (int i = 0; i < layers[INPUT].neurons.length; i++) {
double toBeUpdatedValueForIToJ = ALPHA * error[k]
* eligibilityTraces.getEijk(i, j, k);
layers[INPUT].neurons[i].updateWeightToNeuron(
layers[HIDDEN].neurons[j].getNeuronId(),
toBeUpdatedValueForIToJ);
}
}
}
updateEligibilityTraces(currentState);
}
private void updateEligibilityTraces(int[] currentState) {
// to ensure that the values in neurons are originated from current
// state
feedForward(currentState);
for (int j = 0; j < layers[HIDDEN].neurons.length; j++) {
for (int k = 0; k < layers[OUTPUT].neurons.length; k++) {
double toBeUpdatedValueForJK = gradient(layers[OUTPUT].neurons[k])
* layers[HIDDEN].neurons[j].output;
eligibilityTraces.updateEjk(j, k, toBeUpdatedValueForJK);
for (int i = 0; i < layers[INPUT].neurons.length; i++) {
double toBeUpdatedValueForIJK = gradient(layers[OUTPUT].neurons[k])
* gradient(layers[HIDDEN].neurons[j])
* layers[INPUT].neurons[i].output
* layers[HIDDEN].neurons[j]
.getWeightToNeuron(layers[OUTPUT].neurons[k]
.getNeuronId());
eligibilityTraces.updateEijk(i, j, k,
toBeUpdatedValueForIJK);
}
}
}
}
private double gradient(Neuron neuron) {
return neuron.output * (1 - neuron.output);
}
public void updateToNextWhenOpponentEndsGame(double[] outputOfEndState) {
updateToNextState(lastHandledState, outputOfEndState);
}
private double[] getDifferenceOfOutputs(double[] outputNext,
double[] outputCurrent) {
double[] differencesVector = new double[outputNext.length];
for (int i = 0; i < outputNext.length; i++) {
double difference = outputNext[i] - outputCurrent[i];
differencesVector[i] = difference;
}
return differencesVector;
}
I have used this link as guide line. I have tried different values for ALPHA & BETA, amount of hidden neurons. Eligibility traces are initialized to 0.

The problem mainly is that you cannot tune your neural network function approximator, and from what you said I can assume that "it does not learn", means that the algorithm does not converge.
This happens when we use using TD and NN together. And this happened to me previously, and I searched for it for a long time. The lesson that I have learned is as follows:
According to Richard Sutton: Do not try to use Neural Networks as function approximators, together with TD methods, unless you know exactly how to tune your neural network. Otherwise, this will cause lots of problems.
To know more about it find Sutton's talk on youtube.

Hidden Markov Model, Clarification on a Previous Implementation

I'm experimenting with hidden markov models. I really have no prior experience working with them so I decided to check out a few examples of implementations.
Looking at the below implementation, I was a bit confused about the purpose of the Baum-Welch algorithm (found under the train method) taking a variable steps. I understand providing a training set, but not providing steps. Does anyone have an explanation for this, since I don't understand it from the documentation.
Here's a link to the original code http://cs.nyu.edu/courses/spring04/G22.2591-001/BW%20demo/HMM.java since the code isn't being presented very nicely in my post.
import java.text.*;
/** This class implements a Hidden Markov Model, as well as
the Baum-Welch Algorithm for training HMMs.
#author Holger Wunsch (wunsch#sfs.nphil.uni-tuebingen.de)
*/
public class HMM {
/** number of states */
public int numStates;
/** size of output vocabulary */
public int sigmaSize;
/** initial state probabilities */
public double pi[];
/** transition probabilities */
public double a[][];
/** emission probabilities */
public double b[][];
/** initializes an HMM.
#param numStates number of states
#param sigmaSize size of output vocabulary
*/
public HMM(int numStates, int sigmaSize) {
this.numStates = numStates;
this.sigmaSize = sigmaSize;
pi = new double[numStates];
a = new double[numStates][numStates];
b = new double[numStates][sigmaSize];
}
/** implementation of the Baum-Welch Algorithm for HMMs.
#param o the training set
#param steps the number of steps
*/
public void train(int[] o, int steps) {
int T = o.length;
double[][] fwd;
double[][] bwd;
double pi1[] = new double[numStates];
double a1[][] = new double[numStates][numStates];
double b1[][] = new double[numStates][sigmaSize];
for (int s = 0; s < steps; s++) {
/* calculation of Forward- und Backward Variables from the
current model */
fwd = forwardProc(o);
bwd = backwardProc(o);
/* re-estimation of initial state probabilities */
for (int i = 0; i < numStates; i++)
pi1[i] = gamma(i, 0, o, fwd, bwd);
/* re-estimation of transition probabilities */
for (int i = 0; i < numStates; i++) {
for (int j = 0; j < numStates; j++) {
double num = 0;
double denom = 0;
for (int t = 0; t <= T - 1; t++) {
num += p(t, i, j, o, fwd, bwd);
denom += gamma(i, t, o, fwd, bwd);
}
a1[i][j] = divide(num, denom);
}
}
/* re-estimation of emission probabilities */
for (int i = 0; i < numStates; i++) {
for (int k = 0; k < sigmaSize; k++) {
double num = 0;
double denom = 0;
for (int t = 0; t <= T - 1; t++) {
double g = gamma(i, t, o, fwd, bwd);
num += g * (k == o[t] ? 1 : 0);
denom += g;
}
b1[i][k] = divide(num, denom);
}
}
pi = pi1;
a = a1;
b = b1;
}
}
/** calculation of Forward-Variables f(i,t) for state i at time
t for output sequence O with the current HMM parameters
#param o the output sequence O
#return an array f(i,t) over states and times, containing
the Forward-variables.
*/
public double[][] forwardProc(int[] o) {
int T = o.length;
double[][] fwd = new double[numStates][T];
/* initialization (time 0) */
for (int i = 0; i < numStates; i++)
fwd[i][0] = pi[i] * b[i][o[0]];
/* induction */
for (int t = 0; t <= T-2; t++) {
for (int j = 0; j < numStates; j++) {
fwd[j][t+1] = 0;
for (int i = 0; i < numStates; i++)
fwd[j][t+1] += (fwd[i][t] * a[i][j]);
fwd[j][t+1] *= b[j][o[t+1]];
}
}
return fwd;
}
The other two questions I have are in regards to the Forward method, which implements the Forward part of the Forward-Backward algorithm. From reading up on HMMs, I gather that, after training my model, I should use something like this method to predict future observations. So is the param O (representing output sequences) just the sequence of observations up until this point?
With some experimentation with this method, I'm returned what the documentation says are Forward-Variables, which just look like a bunch of probabilities. How are these translated into future observations?
I'm delving into regions of programming that are quite difficult for me, so I really appreciate your help in helping me understand this stuff!