public void writeEnvToCsv(int customerId, ZipOutputStream zos, List<SummaryDetailsResponse> summaryDetailsResponseList, QueryHelper queryHelper) {
//Headers size = size(No of distinct entities) * 2 + 1
final CsvWriter writer = new CsvWriter(new OutputStreamWriter(zos), new CsvWriterSettings());
final String[] headers = new String[summaryDetailsResponseList.get(0).getEntities().size() * 2 + 1];
String header;
int index = 0, i, k;
headers[index++] = Entities;
for (i = 0; i < summaryDetailsResponseList.get(0).getEntities().size(); i++) {
header = summaryDetailsResponseList.get(0).getEntities().get(i).getEntityType();
if (valueMap.get(header) != null) {
header = valueMap.get(header);
}
headers[index++] = header + " " + Count;
headers[index++] = header + "(s)";
}
writer.writeHeaders(headers);
for (index = 0; index < summaryDetailsResponseList.size(); index++) {
final String[] values = new String[summaryDetailsResponseList.get(index).getEntities().size() * 2 + 1];
i = 0;
k = 0;
values[i++] = summaryDetailsResponseList.get(index).getRecordName();
for (; i <= summaryDetailsResponseList.get(index).getEntities().size() * 2; ) {
values[i++] = String.valueOf(summaryDetailsResponseList.get(index).getEntities().get(k).getEntityData().get(0).getTotal());
try {
StringBuilder strConcat = new StringBuilder();
List<String> hostNames = queryHelper.getHostNames(customerId,
String.valueOf(summaryDetailsResponseList.get(index).getEntities().get(k).getEntityData().get(0).getQuery()));
for (String host : hostNames) {
if (strConcat.toString().equals(""))
strConcat.append(host);
else
strConcat.append(", ").append(host);
}
values[i++] = strConcat.toString();
} catch (Exception e) {
e.printStackTrace();
}
k++;
}
writer.writeRow(CSVUtils.escapeCSVRow(values));
}
writer.flush();
}
Basically I want to write UT to verify this function. Can we verify the CSVWriter that it has written correct data for headers and rows or ZipOutputStream in any way.
I was able to verify the records writer has written if I pass writer to the function instead of creating it inside the function but I don't want to do this.
I have code that needs finds the N-number of optimal combinations taking exactly 1 "player" from 8 ArrayLists of "players". Each ArrayList is anywhere from 20 - 40. Which results in a huge run times with that many iterations. Before attempting to optimize run time, the code was fully functioning, just with undesirable run time. I decided the best way to do this was the use of Executor Services and Java Streams, despite me being new to both. This is a minimized version of my code below:
public static void main(String [] args){
//Lineup is an object which I store the combination of players in
ArrayList <Lineup> currBoard = new ArrayList<Lineup>();
final ExecutorService executorService = Executors.newFixedThreadPool(nThreads);
String threadId = Thread.currentThread().getName();
int tid = Integer.parseInt(threadId.substring(threadId.length() - 1));
int threadWorkLoad = newPG.size() / 4;
//int threadStart = (int)(tid - 1) * threadWorkLoad;
final List<Future<?>> futures = new ArrayList<>();
System.out.println(newPG.size());
for(int i=0; i < nThreads; i++){
Future<?> future = executorService.submit(() -> {
//System.out.println(pg.getName());
currBoard.addAll(parallelFunction(nThreads, amounts, newPG, newSG, newSF, newPF, newC, newG, newF, newL));
});
futures.add(future);
}
executorService.shutdown();
try{
executorService.awaitTermination(5000, TimeUnit.MILLISECONDS);
}catch(InterruptedException e){
System.out.println("oof");
}
try {
for (Future<?> future : futures) {
future.get(); // do anything you need, e.g. isDone(), ...
}
} catch (InterruptedException | ExecutionException e) {
e.printStackTrace();
}
}
And here is the function which I iterate through the possible lineups
public ArrayList<Lineup> parallelFunction(int nThreads, int amounts, ArrayList<Player> newPG, ArrayList<Player> newSG, ArrayList<Player> newSF,
ArrayList<Player> newPF, ArrayList<Player> newC, ArrayList<Player> newG, ArrayList<Player> newF,
ArrayList<Player> newL) {
ArrayList<Player> temp = new ArrayList<Player>();
ArrayList<Lineup> threadBoard = new ArrayList<Lineup>();
String threadId = Thread.currentThread().getName();
int tid = Integer.parseInt(threadId.substring(threadId.length() - 1));
int threadWorkLoad = (newPG.size() + nThreads - 1) / nThreads;
int threadStart = (int)(tid - 1) * threadWorkLoad;
int threadStop = Math.min(threadStart + threadWorkLoad, newPG.size());
//ReentrantLock lock = new ReentrantLock();
ArrayList<Player> tempPG = new ArrayList<Player>();
for(int i = threadStart; i <= threadStop; i++){
tempPG.add(newPG.get(i));
}
Stream<Player> pgStream = StreamSupport.stream(tempPG.spliterator(), true);
pgStream.forEach(pg -> {
System.out.println("TID: " + tid + ", PG: " + pg.getName() + ", iterations: " + counting);
Stream<Player> sgStream = StreamSupport.stream(newSG.spliterator(), true);
sgStream.forEach(sg -> {
//System.out.println(tid + ", " + sg.getName());
Stream<Player> sfStream = StreamSupport.stream(newSF.spliterator(), true);
sfStream.forEach(sf -> {
Stream<Player> pfStream = StreamSupport.stream(newPF.spliterator(), true);
pfStream.forEach(pf -> {
Stream<Player> cStream = StreamSupport.stream(newC.spliterator(), true);
cStream.forEach(c -> {
Stream<Player> gStream = StreamSupport.stream(newG.spliterator(), true);
gStream.forEach(g -> {
if ((!(g.getName().equals(sg.getName()) || g.getName().equals(pg.getName())))
&& ((pg.getSalary() + sg.getSalary() + sf.getSalary() + pf.getSalary()
+ c.getSalary() + 3000 + 3000 + 3000) < 50000)) {
Stream<Player> fStream = StreamSupport.stream(newF.spliterator(), true);
fStream.forEach(f -> {
if ((!(f.getName().equals(sf.getName()) || f.getName().equals(pf.getName())))
&& ((g.getSalary() + pg.getSalary() + sg.getSalary() + sf.getSalary()
+ pf.getSalary() + c.getSalary() + 3000 + 3000) < 50000)) {
Stream<Player> pStream = StreamSupport.stream(newL.spliterator(), true);
pStream.forEach(p -> {
if (!(p.getName().equals(pg.getName()) || p.getName().equals(sg.getName())
|| p.getName().equals(sf.getName())
|| p.getName().equals(pf.getName())
|| p.getName().equals(c.getName())
|| p.getName().equals(g.getName())
|| p.getName().equals(f.getName()))) {
double currScore = 0;
double totalSal = 0;
totalSal = pg.getSalary() + sg.getSalary() + sf.getSalary()
+ pf.getSalary() + c.getSalary() + f.getSalary() + g.getSalary()
+ p.getSalary();
currScore = pg.getProjection() + sg.getProjection() + sf.getProjection()
+ pf.getProjection() + c.getProjection() + f.getProjection()
+ g.getProjection() + p.getProjection();
if (totalSal <= 50000.0) {
counting += 1;
temp.clear();
temp.add(pg);
temp.add(sg);
temp.add(sf);
temp.add(pf);
temp.add(c);
temp.add(g);
temp.add(f);
temp.add(p);
ArrayList<Lineup> tempBoard = new ArrayList<Lineup>();
Lineup aLine = new Lineup(temp, currScore);
if (threadBoard.size() < amounts) {
if (!alreadyIn(threadBoard, temp)) {
threadBoard.add(aLine);
}
} else if (currScore > threadBoard.get(amounts - 1).totalScore) {
// Collections.sort(currBoard, Lineup.TotComp);
if (!alreadyIn(threadBoard, temp)) {
for (int i = 0; i < threadBoard.size() - 1; i++) {
tempBoard.add(threadBoard.get(i));
}
threadBoard.clear();
threadBoard.addAll(tempBoard);
tempBoard.clear();
threadBoard.add(aLine);
}
}
Collections.sort(threadBoard, Lineup.TotComp);
}
// temp.clear();
} // p if
});
pStream.close();
} // f if
});
fStream.close();
} // g if
});
gStream.close();
});
cStream.close();
});
pfStream.close();
});
sfStream.close();
});
sgStream.close();
});
pgStream.close();
return threadBoard;
}
This causes this error after a decent amount of iterations, java.util.concurrent.ExecutionException: java.util.ConcurrentModificationException: java.util.ConcurrentModificationException
Is there anything that I did wrong with the executorService or streams to cause this.
Or does anyone have any suggestions to improve the speed of this code, as it still unsatisfactory with speed until it crashes.
Edit
main:
ArrayList<Lineup> currBoard = new ArrayList<Lineup>();
int nThreads = Runtime.getRuntime().availableProcessors();
final ExecutorService executorService = Executors.newFixedThreadPool(nThreads);
String threadId = Thread.currentThread().getName();
int tid = Integer.parseInt(threadId.substring(threadId.length() - 1));
int threadWorkLoad = newPG.size() / 4;
//int threadStart = (int)(tid - 1) * threadWorkLoad;
final List<Future<ThreadBoard>> futures = new ArrayList<Future<ThreadBoard>>();
for(int i=0; i < nThreads; i++){
Future<ThreadBoard> future = executorService.submit(() -> {
//System.out.println(pg.getName());
return parallelFunction(nThreads, amounts, newPG, newSG, newSF, newPF, newC, newG, newF, newL);
});
futures.add(future);
}
executorService.shutdown();
try{
executorService.awaitTermination(5000, TimeUnit.MILLISECONDS);
}catch(InterruptedException e){
System.out.println("oof");
}
try {
for (Future<ThreadBoard> future : futures) {
currBoard.addAll(future.get().getCurrentLineup()); // do anything you need, e.g. isDone(), ...
}
} catch (InterruptedException | ExecutionException e) {
e.printStackTrace();
}
parallelFunction:
public ThreadBoard parallelFunction(int nThreads, int amounts, ArrayList<Player> newPG, ArrayList<Player> newSG, ArrayList<Player> newSF,
ArrayList<Player> newPF, ArrayList<Player> newC, ArrayList<Player> newG, ArrayList<Player> newF,
ArrayList<Player> newL) {
ThreadBoard threadBoard = new ThreadBoard();
String threadId = Thread.currentThread().getName();
int tid = Integer.parseInt(threadId.substring(threadId.length() - 1));
int threadWorkLoad = (newPG.size() + nThreads - 1) / nThreads;
int threadStart = (int)(tid - 1) * threadWorkLoad;
int threadStop = Math.min(threadStart + threadWorkLoad, newPG.size());
//ReentrantLock lock = new ReentrantLock();
ArrayList<Player> tempPG = new ArrayList<Player>();
for(int i = threadStart; i <= threadStop; i++){
tempPG.add(newPG.get(i));
}
Stream<Player> pgStream = StreamSupport.stream(tempPG.spliterator(), false);
pgStream.forEach(pg -> {
System.out.println("TID: " + tid + ", PG: " + pg.getName() + ", iterations: " + counting);
Stream<Player> sgStream = StreamSupport.stream(newSG.spliterator(), false);
sgStream.forEach(sg -> {
//System.out.println(tid + ", " + sg.getName());
Stream<Player> sfStream = StreamSupport.stream(newSF.spliterator(), false);
sfStream.forEach(sf -> {
Stream<Player> pfStream = StreamSupport.stream(newPF.spliterator(), false);
pfStream.forEach(pf -> {
Stream<Player> cStream = StreamSupport.stream(newC.spliterator(), false);
cStream.forEach(c -> {
Stream<Player> gStream = StreamSupport.stream(newG.spliterator(), false);
gStream.forEach(g -> {
if ((!(g.getName().equals(sg.getName()) || g.getName().equals(pg.getName())))
&& ((pg.getSalary() + sg.getSalary() + sf.getSalary() + pf.getSalary()
+ c.getSalary() + 3000 + 3000 + 3000) < 50000)) {
Stream<Player> fStream = StreamSupport.stream(newF.spliterator(), false);
fStream.forEach(f -> {
if ((!(f.getName().equals(sf.getName()) || f.getName().equals(pf.getName())))
&& ((g.getSalary() + pg.getSalary() + sg.getSalary() + sf.getSalary()
+ pf.getSalary() + c.getSalary() + 3000 + 3000) < 50000)) {
Stream<Player> pStream = StreamSupport.stream(newL.spliterator(), false);
pStream.forEach(p -> {
if (!(p.getName().equals(pg.getName()) || p.getName().equals(sg.getName())
|| p.getName().equals(sf.getName())
|| p.getName().equals(pf.getName())
|| p.getName().equals(c.getName())
|| p.getName().equals(g.getName())
|| p.getName().equals(f.getName()))) {
double currScore = 0;
double totalSal = 0;
totalSal = pg.getSalary() + sg.getSalary() + sf.getSalary()
+ pf.getSalary() + c.getSalary() + f.getSalary() + g.getSalary()
+ p.getSalary();
currScore = pg.getProjection() + sg.getProjection() + sf.getProjection()
+ pf.getProjection() + c.getProjection() + f.getProjection()
+ g.getProjection() + p.getProjection();
if (totalSal <= 50000.0) {
counting += 1;
ArrayList<Player> temp = new ArrayList<Player>();
temp.clear();
temp.add(pg);
temp.add(sg);
temp.add(sf);
temp.add(pf);
temp.add(c);
temp.add(g);
temp.add(f);
temp.add(p);
Lineup aLine = new Lineup(temp, currScore);
threadBoard.addLineup(aLine, currScore, amounts, temp);
// if (threadBoard.size() < amounts) {
// //if (!alreadyIn(threadBoard, temp)) {
// threadBoard.add(aLine);
// //}
// } else if (currScore > threadBoard.get(amounts - 1).totalScore) {
// // Collections.sort(currBoard, Lineup.TotComp);
// //if (!alreadyIn(threadBoard, temp)) {
// for (int i = 0; i < threadBoard.size() - 1; i++) {
// tempBoard.add(threadBoard.get(i));
// }
// threadBoard.clear();
// threadBoard.addAll(tempBoard);
// tempBoard.clear();
// threadBoard.add(aLine);
// //}
// }
//Collections.sort(threadBoard.getCurrentLineup(), Lineup.TotComp);
}
//temp.clear();
} // p if
});
pStream.close();
} // f if
});
fStream.close();
} // g if
});
gStream.close();
});
cStream.close();
});
pfStream.close();
});
sfStream.close();
});
sgStream.close();
});
pgStream.close();
return threadBoard;
}
ThreadBoard class:
public class ThreadBoard {
private List<Lineup> lineups;
public ThreadBoard(){
this.lineups = new ArrayList<>();
}
public synchronized void addLineup(Lineup aLine, double currScore, int amounts, ArrayList<Player> temp) {
if (lineups.size() < amounts) {
if (!this.alreadyIn(lineups, temp)) {
lineups.add(aLine);
}
} else if (currScore > lineups.get(amounts - 1).totalScore) {
// Collections.sort(currBoard, Lineup.TotComp);
if (!this.alreadyIn(lineups, temp)) {
lineups.set(amounts - 1, aLine);
}
}
Collections.sort(lineups, Lineup.TotComp);
}
public List<Lineup> getCurrentLineup() {
return lineups;
}
public boolean alreadyIn(List<Lineup> board, ArrayList<Player> temp) {
boolean found = false;
for (int i = 0; i < board.size(); i++) {
ArrayList <Player> check = board.get(i).getL();
boolean pg = (check.contains(temp.get(0)));
boolean sg = (check.contains(temp.get(1)));
boolean pf = (check.contains(temp.get(2)));
boolean sf = (check.contains(temp.get(3)));
boolean c = (check.contains(temp.get(4)));
boolean g = (check.contains(temp.get(5)));
boolean f = (check.contains(temp.get(6)));
boolean all = (check.contains(temp.get(7)));
if (pg && sg && pf && sf && c && g && f && all) {
found = true;
}
}
return found;
}
}
The problem with the ArrayList<Lineup> threadBoard is that it is used to collect, sort and limit the results from different parallel streams. IMHO an ArrayList is not the correct data structure for these tasks. I would create a separate class that does these steps:
public class ThreadBoard {
private List<Lineup> currentLineup = new ArrayList<>();
public void addLineup(Lineup aLine, double currScore) {
if (currentLineup.size() < amounts) {
if (!alreadyIn(currentLineup, temp)) {
currentLineup.add(aLine);
}
} else if (currScore > currentLineup.get(amounts - 1).totalScore) {
// Collections.sort(currBoard, Lineup.TotComp);
if (!alreadyIn(currentLineup, temp)) {
currentLineup.set(amounts - 1, aLine);
Collections.sort(currentLineup, Lineup.TotComp);
}
}
public List<Lineup> getCurrentLineup() {
return currentLineup;
}
}
To make this thread safe you can declare the addLineup() method as synchronized.
However I'm not sure if all those parallel streams help the performance (since you already split the complete job into chunks that are processed in parallel).
Maybe it would make more sense to split the complete job into more, smaller chunks and then use only sequential streams to process a chunk. In the end your computer has only a limited amount of cores (maybe 4, maybe 16) and splitting a CPU bound job into many more parts (and by using nested parallel streams you are doing that) than there are cores doesn't make much sense.
I'm trying to write a program that uses Imgur's API to download images based on an account name.
private volatile int threadCount;
private volatile double totalFileSize;
private volatile List<String> albums = new ArrayList<>();
private volatile Map<JSONObject, String> images = new HashMap<>();
private final ExecutorService executorService = Executors.newFixedThreadPool(100, (Runnable r) -> {
Thread t = Executors.defaultThreadFactory().newThread(r);
t.setDaemon(true);
return t;
});
private void downloadAlbums(List<String> albums) {
threadCount = 0;
albums.forEach((albumHash) -> {
if (hasRemainingRequests()) {
incThreadCount();
executorService.execute(() -> {
try {
String responseString;
String dirTitle;
String albumUrl = URL_ALBUM + albumHash;
String query = String.format("client_id=%s", URLEncoder.encode(CLIENT_ID, CHARSET));
URLConnection connection = new URL(albumUrl + "?" + query).openConnection();
connection.setRequestProperty("Accept-Charset", CHARSET);
InputStream response = connection.getInputStream();
try (Scanner scanner = new Scanner(response)) {
responseString = scanner.useDelimiter("\\A").next();
JSONObject obj = new JSONObject(responseString).getJSONObject("data");
dirTitle = obj.getString("title");
String temp = "";
// Get save path from a TextField somewhere else on the GUI
ObservableList<Node> nodes = primaryStage.getScene().getRoot().getChildrenUnmodifiable();
for (Node node : nodes) {
if (node instanceof VBox) {
ObservableList<Node> vNodes = ((VBox) node).getChildrenUnmodifiable();
for (Node vNode : vNodes) {
if (vNode instanceof DestinationBrowser) {
temp = ((DestinationBrowser) vNode).getDestination().trim();
}
}
}
}
final String path = temp + "\\" + formatPath(accountName) + "\\" + formatPath(dirTitle);
JSONArray arr = obj.getJSONArray("images");
arr.forEach((jsonObject) -> {
totalFileSize += ((JSONObject) jsonObject).getDouble("size");
images.put((JSONObject) jsonObject, path);
});
}
} catch (IOException ex) {
//
} catch (Exception ex) {
//
} finally {
decThreadCount();
if (threadCount == 0) {
Platform.runLater(() -> {
DecimalFormat df = new DecimalFormat("#.#");
Alert alert = new Alert(Alert.AlertType.CONFIRMATION);// 714833218
alert.setHeaderText("Found " + images.size() + " images (" + (totalFileSize < 1000000000 ? df.format(totalFileSize / 1000000) + " MB)" : df.format(totalFileSize / 1000000000) + " GB)"));
alert.setContentText("Proceed with download and save images?");
Optional<ButtonType> alertResponse = alert.showAndWait();
if (alertResponse.get() == ButtonType.OK) {
progressBar.setTotalWork(images.size());
executorService.execute(() -> {
for (JSONObject obj : images.keySet()) {
(new File(images.get(obj))).mkdirs();
downloadImage(obj, images.get(obj));
}
});
}
});
}
}
});
}
});
}
albums is a List of codes that are required to send a GET request to Imgur in order to receive that album's images. The data returned is then used in another method which downloads the images themselves.
Now, all of this works fine, but when the program is making all the GET requests the JavaFX thread hangs (GUI becomes unresponsive). And after all the GET requests have been executed, the JavaFX thread stops hanging and the alert shows up with the correct information.
I just don't understand why the GUI becomes unresponsive when I'm not (I believe I'm not) blocking it's thread and I'm using an ExecutorService to do all the network requests.
ExecutorService serv = Executors.newFixedThreadPool(5);
CompletionService<Integer> servive = new ExecutorCompletionService<>(serv);
for (int i = 0; i < 5; i++) {
servive.submit(new MyTask(i));
}
for (int i = 0; i < 5; i++) {
Future<Integer> f = null;
try {
f = servive.take();
} catch (InterruptedException e) {
e.printStackTrace();
}
//f.get()
}
I want to know which task or callable was completed in the first result(f.get())
How do I do that, or what other functions?
Well, its totally depends upon JVM and task how its going to be handle by JVM.
I have a line in my class that is supposed to make a call to another class.
I put a break point on this line and that break point is never reached, instead
I get an error saying:
Can't create handler inside thread that has not called Looper.prepare(
)
I have placed my code below. I am getting the error on the line where I try to call the AppTimeUsageClass.
public void storeAppTimeUsageData(AppTimeUsage stats) {
//Context context = this;
List<AppTimeUsage> items = new ArrayList<>();
try {
appTimeUsageDao.insertOrReplace(stats);
} catch (Exception e) {
Log.e("Error", "Some exception occurred", e);
Log.e("APP_TAG", "STACKTRACE");
Log.e("APP_TAG", Log.getStackTraceString(e));
}
String sql = "SELECT * FROM APP_TIME_USAGE ";
Cursor c = appTimeUsageDao.getDatabase().rawQuery(sql, null);
int offset = 0;
int d ;
int cd ;
String e = "";
while (c.moveToNext()) {
AppTimeUsage atu = new AppTimeUsage(
c.getLong(0),
new Date (),
d = c.getInt(2),
e = c.getString(3)
// break;
);
items.add(atu);
stats = atu ;
}
UploadAppTimeUsageActivity aapTimeClass = new UploadAppTimeUsageActivity();
aapTimeClass.postAppTimeUsage(stats);
}
This is the code of the service that calls the "storeAppTimeUsageData" method
private boolean addToStatistics(String target )
{
Context context = this;
SQLiteOpenHelper helper = new DaoMaster.DevOpenHelper(getApplicationContext(), Globals.DatabaseName, null);
DaoMaster master = new DaoMaster(helper.getWritableDatabase());
daoSession = master.newSession();
appTimeUsageDao = DeviceInsightApp.getSession(this, true).getAppTimeUsageDao();
AppTimeUsage appStats;
boolean changed = false;
Date now = new Date();
if(!TextUtils.isEmpty(target))
{
if(!target.equals(foreground))
{
int i;
// timeCheckVariable = i ;
if(foreground != null && split != null)
{
// TODO: calculate time difference from current moment
// to the moment when previous foreground process was activated
i = packages.indexOf(foreground);
timeCheckVariable = i ;
long delta = (now.getTime() - split.getTime()) / 1000;
Long time = (Long)processList.get(i).get(ProcessList.COLUMN_PROCESS_TIME);
if(time != null)
{
// TODO: add the delta to statistics of 'foreground'
time += delta;
}
else
{
time = new Long(delta);
}
processList.get(i).put(ProcessList.COLUMN_PROCESS_TIME, time);
int x = time.intValue( );
String applicationName = (String)processList.get(i).get(ProcessList.COLUMN_PROCESS_NAME);
appStats = new AppTimeUsage(null , new Date(), x ,applicationName);
**//Call to storeAppTimeUsageData below**
new DataUsageRecorder(context).storeAppTimeUsageData(appStats);
}
//update count of process activation for new 'target'
i = packages.indexOf(target);
Integer count = (Integer)processList.get(i).get(ProcessList.COLUMN_PROCESS_COUNT);
if(count != null) count++;
else
{
count = new Integer(1);
}
processList.get(i).put(ProcessList.COLUMN_PROCESS_COUNT, count);
foreground = target;
split = now;
changed = true;
}
}
//Long checkTimeNow = (Long)processList.get(timeCheckVariable).get(ProcessList.COLUMN_PROCESS_TIME);
return changed;
}