Develop Reference

Apply LOOCV in java splitting with a specific condition

I have a csv file containing 24231 rows. I would like to apply LOOCV based on the project name instead of the observations of the whole dataset.
So if my dataset contains information for 15 projects, I would like to have the training set based on 14 projects and the test set based on the other project.
I was relying on weka's API, is there anything that automates this process?

For non-numeric attributes, Weka allows you to retrieve the unique values via Attribute.numValues() (how many are there) and Attribute.value(int) (the -th value).
package weka;
import weka.core.Attribute;
import weka.core.Instance;
import weka.core.Instances;
import weka.core.converters.ConverterUtils;
public class LOOByValue {
/**
* 1st arg: ARFF file to load
* 2nd arg: 0-based index in ARFF to use for class
* 3rd arg: 0-based index in ARFF to use for LOO
*
* #param args the command-line arguments
* #throws Exception if loading/processing of data fails
*/
public static void main(String[] args) throws Exception {
// load data
Instances full = ConverterUtils.DataSource.read(args[0]);
full.setClassIndex(Integer.parseInt(args[1]));
int looCol = Integer.parseInt(args[2]);
Attribute looAtt = full.attribute(looCol);
if (looAtt.isNumeric())
throw new IllegalStateException("Attribute cannot be numeric!");
// iterate unique values to create train/test splits
for (int i = 0; i < looAtt.numValues(); i++) {
String value = looAtt.value(i);
System.out.println("\n" + (i+1) + "/" + full.attribute(looCol).numValues() + ": " + value);
Instances train = new Instances(full, full.numInstances());
Instances test = new Instances(full, full.numInstances());
for (int n = 0; n < full.numInstances(); n++) {
Instance inst = full.instance(n);
if (inst.stringValue(looCol).equals(value))
test.add((Instance) inst.copy());
else
train.add((Instance) inst.copy());
}
train.compactify();
test.compactify();
// TODO do something with the data
System.out.println("train size: " + train.numInstances());
System.out.println("test size: " + test.numInstances());
}
}
}
With Weka's anneal UCI dataset and the surface-quality for leave-one-out, you can generate something like this:
1/5: ?
train size: 654
test size: 244
2/5: D
train size: 843
test size: 55
3/5: E
train size: 588
test size: 310
4/5: F
train size: 838
test size: 60
5/5: G
train size: 669
test size: 229

Python3: how to run java class file from python

I read JPype documentation from this link: http://jpype.readthedocs.io/en/latest/, but i am not sure that i can use JPype or it is better that i choose another way to run java class from python3. Also i have to point that i get a little confused for doing with JPype.
i did this:
import urllib.request
import os
import tempfile
import sys
import fileinput
import logging
import JPype as jp
# here i have python code and do something else
"""
my python code
"""
#from this point till end of the code i did for my question
logging.basicConfig(filename="ERROR.txt", level= logging.ERROR)
try:
logging.debug('we are in the main try loop')
jp.startJVM("C:/Users/user/AppData/Local/Programs/Python/Python36/ClassWithTest.java", "-ea")
test_class = jp.JClass("ClassWithTest")
a = testAll()
file_java_class = open("OUTPUT.txt", "w")
file_java_class.write(a)
except Exception as e1:
logging.error(str(e1))
jp.shutdownJVM()
But it has problem and shows me this error: 7, in import
jpype as jp ModuleNotFoundError: No module named 'jpype'
========================================================================
I have to point that my java class is in my python path and my java
class is:
public class ClassWithTest{
public ClassWithTest(){
/* Va invocato */
System.out.println(insertLength("Tests ready to run"));
}
public void testAbs(){
/* Va invocato */
System.out.println(insertLength("Invocation of " + Thread.currentThread().getStackTrace()[1].getMethodName()));
System.out.println(insertLength("abs(-2) = 2: " + testResult(Math.abs(-2), 2)));
System.out.println(insertLength("abs(-3) = -2: " + testResult(Math.abs(-3), -2)));
}
public void testRound(){
/* Va invocato */
System.out.println(insertLength("Invocation of " + Thread.currentThread().getStackTrace()[1].getMethodName()));
System.out.println(insertLength("round(0.7) = 1: " + testResult(Math.round(0.7), 1)));
System.out.println(insertLength("round(0.2) = 1: " + testResult(Math.round(0.2), 1)));
}
public void testMin(){
/* Va invocato */
System.out.println(insertLength("Invocation of " + Thread.currentThread().getStackTrace()[1].getMethodName()));
System.out.println(insertLength("min(7,3) = 3: " + testResult(Math.min(7,3), 3)));
System.out.println(insertLength("min(5,7) = 7: " + testResult(Math.min(5,7), 7)));
}
public void testMax(){
/* Va invocato */
System.out.println(insertLength("Invocation of " + Thread.currentThread().getStackTrace()[1].getMethodName()));
System.out.println(insertLength("max(7,14) = 14: " + testResult(Math.max(7,14), 14)));
System.out.println(insertLength("max(5,7) = 5: " + testResult(Math.max(5,7), 5)));
}
public void TestDiv(){
/* Non andreabbe chiamato */
System.out.println(insertLength("Invocation of " + Thread.currentThread().getStackTrace()[1].getMethodName()));
System.out.println(insertLength("14 / 2 = 7: " + testResult(14 / 7, 3)));
// assert(14 / 0 == 10));
}
public void testMult(int x){
/* Non andreabbe chiamato */
System.out.println(insertLength("Invocation of " + Thread.currentThread().getStackTrace()[1].getMethodName()));
System.out.printf("2 * %d = %s:", x, testResult(2 * x, 2 * x)));
// assert(false));
}
public void otherTest(){
/* Non andreabbe chiamato */
System.out.println(insertLength("Invocation of " + Thread.currentThread().getStackTrace()[1].getMethodName()));
System.out.printf("1 = 1:", testResult(1, 1)));
// assert(false));
}
private void testAll(){
/* Non dovrebbe essere chiamato */
System.out.println(insertLength("Invocation of " + Thread.currentThread().getStackTrace()[1].getMethodName()));
testAbs());
testRound());
testMin());
testMax());
}
private static String testResult(double result, double expected){
return (result == expected ? "ok" : "no"));
}
public static String insertLength(Object obj){
String line = obj.toString());
return String.format("[%d]> %s", line.length(), line));
}
public static void main(String[] args){
(new ClassWithTest()).testAll());
}
}
i do not know why this not work?!! i read jython, but i can not use
jython, because i did not understand well. could you help me:
1- how can i run java class from python 2- save the out put of
execution in text file 3- save the possible error messages in
another text file and then terminate. thank you

I normally use JYthon. Making use of Java from within Jython applications is about as seamless as using external Jython modules within a Jython script.
Here is a simple example:
from java.lang import Math
Math.max(4, 7)
Math.pow(10,5)
from java.lang import System as javasystem
javasystem.out.println("Hello")
Hello

Tensorflow in Android: How do i use my linear regression model to predict a value in an android application?

I currently have a ipynb file (ipython notebook) that contains a linear regression code / model(im not entirely sure if it's a model) that I've created earlier.
How do i implement this model in an android application such that if I were to input a value of 'x' in a text box, it'll output in a textview the predicted value of 'y'. Function: Y = mx + b.
I've tried looking at different tutorials, but they were mostly not "step-by-step" guides, which made it really hard to understand, I'm a beginner at coding.
Here's my code for the model:
import tensorflow as tf
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
rng = np.random
from numpy import genfromtxt
from sklearn.datasets import load_boston
# Parameters
learning_rate = 0.0001
training_epochs = 1000
display_step = 50
n_samples = 222
X = tf.placeholder("float") # create symbolic variables
Y = tf.placeholder("float")
filename_queue = tf.train.string_input_producer(["C://Users//Shiina//battdata.csv"],shuffle=False)
reader = tf.TextLineReader() # skip_header_lines=1 if csv has headers
key, value = reader.read(filename_queue)
# Default values, in case of empty columns. Also specifies the type of the
# decoded result.
record_defaults = [[1.], [1.]]
height, soc= tf.decode_csv(
value, record_defaults=record_defaults)
features = tf.stack([height])
# Set model weights
W = tf.Variable(rng.randn(), name="weight")
b = tf.Variable(rng.randn(), name="bias")
# Construct a linear model
pred_soc = tf.add(tf.multiply(height, W), b) # XW + b <- y = mx + b where W is gradient, b is intercept
# Mean squared error
cost = tf.reduce_sum(tf.pow(pred_soc-soc, 2))/(2*n_samples)
# Gradient descent
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)
# Initializing the variables
init = tf.global_variables_initializer()
with tf.Session() as sess:
# Start populating the filename queue.
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
sess.run(init)
# Fit all training data
for epoch in range(training_epochs):
_, cost_value = sess.run([optimizer,cost])
#Display logs per epoch step
if (epoch+1) % display_step == 0:
c = sess.run(cost)
print( "Epoch:", '%04d' % (epoch+1), "cost=", "{:.9f}".format(c), \
"W=", sess.run(W), "b=", sess.run(b))
print("Optimization Finished!")
training_cost = sess.run(cost)
print ("Training cost=", training_cost, "W=", sess.run(W), "b=", sess.run(b), '\n')
#Plot data after completing training
train_X = []
train_Y = []
for i in range(n_samples): #Your input data size to loop through once
X, Y = sess.run([height, pred_soc]) # Call pred, to get the prediction with the updated weights
train_X.append(X)
train_Y.append(Y)
#Graphic display
plt.plot(train_X, train_Y, 'ro', label='Original data')
plt.ylabel("SoC")
plt.xlabel("Height")
plt.axis([0, 1, 0, 100])
plt.plot(train_X, train_Y, linewidth=2.0)
plt.legend()
plt.show()
coord.request_stop()
coord.join(threads)

Weka output predictions

I've used the Weka GUI for training and testing a file (making predictions), but can't do the same with the API. The error I'm getting says there's a different number of attributes in the train and test files. In the GUI, this can be solved by checking "Output predictions".
How to do something similar using the API? do you know of any samples out there?
import weka.classifiers.bayes.NaiveBayes;
import weka.classifiers.meta.FilteredClassifier;
import weka.classifiers.trees.J48;
import weka.core.Instances;
import weka.core.converters.ConverterUtils.DataSource;
import weka.filters.Filter;
import weka.filters.unsupervised.attribute.NominalToBinary;
import weka.filters.unsupervised.attribute.Remove;
public class WekaTutorial
{
public static void main(String[] args) throws Exception
{
DataSource trainSource = new DataSource("/tmp/classes - edited.arff"); // training
Instances trainData = trainSource.getDataSet();
DataSource testSource = new DataSource("/tmp/classes_testing.arff");
Instances testData = testSource.getDataSet();
if (trainData.classIndex() == -1)
{
trainData.setClassIndex(trainData.numAttributes() - 1);
}
if (testData.classIndex() == -1)
{
testData.setClassIndex(testData.numAttributes() - 1);
}
String[] options = weka.core.Utils.splitOptions("weka.filters.unsupervised.attribute.StringToWordVector -R first-last -W 1000 -prune-rate -1.0 -N 0 -stemmer weka.core.stemmers.NullStemmer -M 1 "
+ "-tokenizer \"weka.core.tokenizers.WordTokenizer -delimiters \" \\r\\n\\t.,;:\\\'\\\"()?!\"");
Remove remove = new Remove();
remove.setOptions(options);
remove.setInputFormat(trainData);
NominalToBinary filter = new NominalToBinary();
NaiveBayes nb = new NaiveBayes();
FilteredClassifier fc = new FilteredClassifier();
fc.setFilter(filter);
fc.setClassifier(nb);
// train and make predictions
fc.buildClassifier(trainData);
for (int i = 0; i < testData.numInstances(); i++)
{
double pred = fc.classifyInstance(testData.instance(i));
System.out.print("ID: " + testData.instance(i).value(0));
System.out.print(", actual: " + testData.classAttribute().value((int) testData.instance(i).classValue()));
System.out.println(", predicted: " + testData.classAttribute().value((int) pred));
}
}
}
Error:
Exception in thread "main" java.lang.IllegalArgumentException: Src and Dest differ in # of attributes: 2 != 17152
This was not an issue for the GUI.

You need to ensure that categories in train and test sets are compatible, try to
combine train and test sets
List item
preprocess them
save them as arff
open two empty files
copy the header from the top to line "#data"
copy in training set into first file and test set into second file

Java Exceptions on weka k fold programme

I would like to perform a 10 fold cross validation on my data and I used the weka java programme. However, I encountered exception problems.
Here is the exceptions:
---Registering Weka Editors---
Trying to add database driver (JDBC): jdbc.idbDriver - Error, not in CLASSPATH?
Exception in thread "main" java.lang.IllegalArgumentException: No suitable converter found for ''!
at weka.core.converters.ConverterUtils$DataSource.<init>(ConverterUtils.java:137)
at weka.core.converters.ConverterUtils$DataSource.read(ConverterUtils.java:441)
at crossvalidationmultipleruns.CrossValidationMultipleRuns.main(CrossValidationMultipleRuns.java:45)
C:\Users\TomXavier\AppData\Local\NetBeans\Cache\8.1\executor-snippets\run.xml:53: Java returned: 1
BUILD FAILED (total time: 1 second)
Here is the programme I used:
import weka.core.Instances;
import weka.core.converters.ConverterUtils.DataSource;
import weka.core.Utils;
import weka.classifiers.Classifier;
import weka.classifiers.Evaluation;
import java.util.Random;
/**
* Performs a single run of cross-validation.
*
* Command-line parameters:
* <ul>
* <li>-t filename - the dataset to use</li>
* <li>-x int - the number of folds to use</li>
* <li>-s int - the seed for the random number generator</li>
* <li>-c int - the class index, "first" and "last" are accepted as well;
* "last" is used by default</li>
* <li>-W classifier - classname and options, enclosed by double quotes;
* the classifier to cross-validate</li>
* </ul>
*
* Example command-line:
* <pre>
* java CrossValidationSingleRun -t anneal.arff -c last -x 10 -s 1 -W "weka.classifiers.trees.J48 -C 0.25"
* </pre>
*
* #author FracPete (fracpete at waikato dot ac dot nz)
*/
public class CrossValidationSingleRun {
/**
* Performs the cross-validation. See Javadoc of class for information
* on command-line parameters.
*
* #param args the command-line parameters
* #throws Excecption if something goes wrong
*/
public static void main(String[] args) throws Exception {
// loads data and set class index
Instances data = DataSource.read(Utils.getOption("C:/Users/TomXavier/Documents/MATLAB/total_data.arff", args));
String clsIndex = Utils.getOption("first", args);
if (clsIndex.length() == 0)
clsIndex = "last";
if (clsIndex.equals("first"))
data.setClassIndex(0);
else if (clsIndex.equals("last"))
data.setClassIndex(data.numAttributes() - 1);
else
data.setClassIndex(Integer.parseInt(clsIndex) - 1);
// classifier
String[] tmpOptions;
String classname;
tmpOptions = Utils.splitOptions(Utils.getOption("weka.classifiers.trees.J48", args));
classname = tmpOptions[0];
tmpOptions[0] = "";
Classifier cls = (Classifier) Utils.forName(Classifier.class, classname, tmpOptions);
// other options
int seed = Integer.parseInt(Utils.getOption("1", args));
int folds = Integer.parseInt(Utils.getOption("10", args));
// randomize data
Random rand = new Random(seed);
Instances randData = new Instances(data);
randData.randomize(rand);
if (randData.classAttribute().isNominal())
randData.stratify(folds);
// perform cross-validation
Evaluation eval = new Evaluation(randData);
for (int n = 0; n < folds; n++) {
Instances train = randData.trainCV(folds, n);
Instances test = randData.testCV(folds, n);
// the above code is used by the StratifiedRemoveFolds filter, the
// code below by the Explorer/Experimenter:
// Instances train = randData.trainCV(folds, n, rand);
// build and evaluate classifier
Classifier clsCopy = Classifier.makeCopy(cls);
clsCopy.buildClassifier(train);
eval.evaluateModel(clsCopy, test);
}
// output evaluation
System.out.println();
System.out.println("=== Setup ===");
System.out.println("Classifier: " + cls.getClass().getName() + " " + Utils.joinOptions(cls.getOptions()));
System.out.println("Dataset: " + data.relationName());
System.out.println("Folds: " + folds);
System.out.println("Seed: " + seed);
System.out.println();
System.out.println(eval.toSummaryString("=== " + folds + "-fold Cross-validation ===", false));
}
}
Is there any solution for this problem?
Many thanks!