Develop Reference

How to make the main thread finish last

I have a parser that works with the help of Executor.newCachedThreadPool(), and faced the fact that the main thread in which the record to the JSON file is written is executed before the child ones. As a result, we have an empty file...
I know the topic of multithreading rather poorly and can not understand ge wrong. I tried to use the Join () method on the main thread, but in the end the program just hangs up when approaching this part
Main.java
import model.Product;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
public class Main {
public static void main(String[] args) throws InterruptedException {
String rootUrl = "example.com";
System.out.println("Started parsing: " + rootUrl);
long m = System.currentTimeMillis();
HtmlParser htmlParser = new HtmlParser();
List<Product> productList = new CopyOnWriteArrayList<>();
htmlParser.parse(rootUrl, productList);
Printer.printToJson(productList);
System.out.println("Finish: completed in " + ((double) System.currentTimeMillis() - m) / 1000 + " seconds");
}
}
HtmlParser.java
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import ua.bala.model.Product;
import java.io.IOException;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;
public class HtmlParser {
private static AtomicInteger httpRequestsCounter = new AtomicInteger(0);
public static AtomicInteger getHttpRequestsCounter() {
return httpRequestsCounter;
}
public void parse(String url, List<Product> productList) {
try {
Document page = getPage(url);
parsePage(page, productList);
} catch (IOException e) {
e.printStackTrace();
}
}
private static Document getPage(String url) throws IOException {
Document document = Jsoup.connect(url).get();
httpRequestsCounter.getAndIncrement();
return document;
}
private void parsePage(Document page, List<Product> productList) {
Elements productElements = page.select("a.dgBQdu");
ExecutorService service = Executors.newCachedThreadPool();
for (Element element: productElements){
service.execute(() -> {
Long articleID = Long.parseLong(element.attr("id"));
String name = "NAME";
String brand = "BRAND";
BigDecimal price = new BigDecimal(BigInteger.ZERO);
Set<String> colors = new HashSet<>();
String url = "https://www.aboutyou.de" + element.attr("href");
Document innerPage;
try {
innerPage = getPage(url);
Element innerElement = innerPage.selectFirst("[data-test-id='BuyBox']");
name = innerElement.selectFirst("div.dZjUXd").text();
brand = innerElement.selectFirst("[data-test-id='BrandLogo']").attr("alt");
colors = new HashSet<>(innerElement.select("span.jlvxcb-1").eachText());
String priceStr = innerElement.selectFirst("div.dWWxvw > span").text().replace("ab ","").replace(" EUR","").replace(",", ".");
price = new BigDecimal(priceStr);
} catch (IOException e) {
e.printStackTrace();
}
Product product = new Product(articleID, name, brand, colors, price, url);
addProduct(product, productList);
});
}
service.shutdown();
}
private synchronized void addProduct(Product product, List<Product> productList){
System.out.println("Product " + product.getID() + " parsed");
System.out.print(product);
productList.add(product);
System.out.printf("Product %d added to list\n%n", product.getID());
}
}
Printer.java
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import model.Product;
import java.io.*;
import java.util.Comparator;
import java.util.List;
public class Printer {
private static final String path = "";
private static final String fileName = "productsOutput";
public static void printToJson(List<Product> products){
products.sort(Comparator.comparing(Product::getID));
System.out.println("Product list start printing to JSON");
try (final Writer writer = new FileWriter(path + fileName + ".json")) {
Gson gson = new GsonBuilder().create();
gson.toJson(products, writer);
System.out.println("Product list printed to JSON");
System.out.printf("Amount of triggered HTTP requests: %s%nAmount of extracted products: %s%n",
HtmlParser.getHttpRequestsCounter(), products.size());
} catch (IOException e) {
e.printStackTrace();
}
}
}
Printer.java
package model;
import lombok.*;
import java.math.BigDecimal;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
#NoArgsConstructor
#Getter
#Setter
public class Product {
private static AtomicLong productsCounter = new AtomicLong(1);
private Long ID;
private Long articleID;
private String name;
private String brand;
private BigDecimal price;
private Set<String> colors;
private String url;
{
ID = productsCounter.getAndIncrement();
}
public Product(Long articleID, String name, String brand, Set<String> colors, BigDecimal price, String url) {
this.articleID = articleID;
this.name = name;
this.brand = brand;
this.price = price;
this.colors = colors;
this.url = url;
}
public static AtomicLong getProductsCounter() {
return productsCounter;
}
#Override
public String toString() {
return String.format("%d\t%d\t%s\t%s\t%s\t%s\t%s\n", ID, articleID, name, brand, price, colors, url);
}
}

One way to get the main thread to wait for the worker threads to finish would be to have HTMLParser return its ExecutorService so that Main.main can call awaitTermination(...) on it.
Or ... if you didn't want to expose the service to the Main class, you could add a "wait until it is done" method to the HTMLParser classes API.
Note: it is probably a bad idea for each call to parsePage to create and then tear down its own thread pool executor service. You should probably create one thread pool for each HTMLParser instance, and reuse it in each parsePage call.
In addition, doing this will make solving your problem with main easier.

There are several ways to overcome this. Using observables, or blocking main thread or using an interface without blocking the main thread. For me, interface would be a good choice. If you are familiar with java interface, you can implement an interface in order to print recent parsed products. Here is how step by step:
Interface class:
public interface ProductsListener {
void onProductsReady(List<Product> products);
}
MainImpl class (NOT Main class itself):
public class MainImpl implements ProductListener {
// When product list loading is done this func will be called
void onProductsRead(List<Product> products) {
Printer.printToJson(productList);
}
}
In Main class:
public class Main {
public static void main(String[] args) throws InterruptedException {
MainImpl listener = new MainImpl();
htmlParser.setProductListener(listener);
// Rest of the code...
}
}
In HtmlParser class:
public class HtmlParser {
private MainImpl productListener;
//...
public void setProductListener(MainImpl listener) {
// Alternatively you can do it in a constructor
productListener = listener;
}
//...
private void parsePage(Document page, List<Product> productList) {
Elements productElements = page.select("a.dgBQdu");
int parseCount = 0;
ExecutorService service = Executors.newCachedThreadPool();
for (Element element: productElements){
service.execute(() -> {
Long articleID = Long.parseLong(element.attr("id"));
String name = "NAME";
String brand = "BRAND";
BigDecimal price = new BigDecimal(BigInteger.ZERO);
Set<String> colors = new HashSet<>();
String url = "https://www.aboutyou.de" + element.attr("href");
Document innerPage;
try {
innerPage = getPage(url);
Element innerElement = innerPage.selectFirst("[data-test-id='BuyBox']");
name = innerElement.selectFirst("div.dZjUXd").text();
brand = innerElement.selectFirst("[data-test-id='BrandLogo']").attr("alt");
colors = new HashSet<>(innerElement.select("span.jlvxcb-1").eachText());
String priceStr = innerElement.selectFirst("div.dWWxvw > span").text().replace("ab ","").replace(" EUR","").replace(",", ".");
price = new BigDecimal(priceStr);
} catch (IOException e) {
e.printStackTrace();
}
Product product = new Product(articleID, name, brand, colors, price, url);
addProduct(product, productList);
parseCount++; // Count each element that has been parsed
// Check if all elements have been parsed
if(parseCount >= productElements.size()) {
// All products are done, notify the listener class
productListener.onProductsReady(productList);
}
});
}
}
Not tested, but the interface logic must work.

Static access of Enums in Java?

I have two enums AttributesProperty and ResourceAttributesMapping each referencing each other as shown below:
import org.apache.commons.collections.map.HashedMap;
import test.xsd.Attribute;
public enum AttributesProperty {
ACP_RESOURCETYPE(ResourceAttributesMapping.ACCESSCONTROLPOLICY,"resourceType"),
ACP_RESOURCEID(ResourceAttributesMapping.ACCESSCONTROLPOLICY,"resourceID"),
.............
private ResourceAttributesMapping resource;
private String attributeName;
private AttributesProperty(ResourceAttributesMapping resource, String attributeName)
{
this.resource = resource;
this.attributeName = attributeName;
}
public static Map<ResourceAttributesMapping, List<Attribute>> getResTypeMapForAttrName(List<Attribute> attributesList) {
Map<ResourceAttributesMapping, List<Attribute>> attributeMap = new HashedMap();
List<Attribute> attrList = null;
List<ResourceAttributesMapping> resTypes = new ArrayList<ResourceAttributesMapping>();
for (Attribute attr : attributesList) {
boolean flag = false;
for (AttributesProperty attrProp : AttributesProperty.values()) {
if (attr.getName().equals(attrProp.getAttributeName())) {
if (attributeMap.get(attrProp.getResourceType()) == null) {
.........// Some logic
}
}
Referenced Enum: ResourceAttributesMapping.java
This enum as seen in imports statically importing the AttributesProperty enum.
import static xsd.enums.AttributesProperty.*;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import org.apache.commons.collections.map.HashedMap;
public enum ResourceAttributesMapping
{
ACCESSCONTROLPOLICY(1, Arrays.asList(ACP_RESOURCETYPE,ACP_RESOURCEID,............)),
AE(2, Arrays.asList(AE_RESOURCETYPE,AE_RESOURCEID,...........)),
....
private int resourceType;
private List<AttributesProperty> attrList;
private ResourceAttributesMapping(int resourceType, List<AttributesProperty> attrList) {
this.resourceType = resourceType;
this.attrList = attrList;
}
public int getM2MResourceTypes() {
return resourceType;
}
public List<AttributesProperty> getAttributeList(){
return attrList;
}
......// accessor methods
}
This code was running succesfully on JDK 8, WildFly 15 but after migrating to JDK 11 WildFly 20, I am getting a NullPointerException in the AttributesProperty 's enums accessor method:
public static Map<ResourceAttributesMapping, List<Attribute>> getResTypeMapForAttrName(List<Attribute> attributesList) {
Map<ResourceAttributesMapping, List<Attribute>> attributeMap = new HashedMap();
List<Attribute> attrList = null;
List<ResourceAttributesMapping> resTypes = new ArrayList<ResourceAttributesMapping>();
for (Attribute attr : attributesList) {
boolean flag = false;
for (AttributesProperty attrProp : AttributesProperty.values()) {
if (attr.getName().equals(attrProp.getAttributeName())) {
if (attributeMap.get(attrProp.getResourceType()) == null) {
.........// Some logic
attrProp.getResourceType() is giving null.
The test scenario is such that on first iteration of running the accesor methods in the enum it runs successfully, however on second iteration I get the NullPointerException.
Is there some change in Enums accessing from JDK 8 to 11?
Or the static access of second enum causing the issue.

In which way I can write down for-loop with List in Java?

Is it possible to use Lambda expression or better way to write down the for-loop?
public TaskDTO convertToDTO(Task task) {
for (int i = 0; i < task.getPrecedingTasks().size(); i++)
this.precedingTasks.add(task.getPrecedingTasks().get(i).getName());
}

Your solution is good as:
task.getPrecedingTasks().stream().map(Task::getName).forEach(this.precedingTasks::add);
But since you are just retrieving the part of the Task, map and then collect as a list as:
this.precedingTasks = task.getPrecedingTasks().stream().map(Task::getName).collect(Collectors.toList());
Isn't it more straightforward and easier to understand? Since stream here is to do the mapping/converting and then collecting.
And also in this way, you don't need to do the initialisation for the this.precedingTasks as
this.precedingTasks = new ArrayList<>(); // to ensure it's not null;
Anyway, just personal preference here.

This is a complete example, where I put System.out.println ... you should use a this.precedingTasks.addAll( ...
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
public class Main {
public static void main(String[] args) {
Task t1 = new Task("myTask", Arrays.asList(new Task("innerTask1"), new Task("innerTask2")));
System.out.println(t1.precedingTasks.stream().map(Task::getName).collect(Collectors.toList()));
}
static class Task {
private String name;
private List<Task> precedingTasks = new ArrayList<>();
public Task(String name) {
this.name = name;
}
public Task(String name, List<Task> precedingTasks) {
this.name = name;
this.precedingTasks = precedingTasks;
}
public String getName() {
return name;
}
public List<Task> getPrecedingTasks() {
return precedingTasks;
}
}
}
The output is
[innerTask1, innerTask2]

I found the correct solution in my case:
task.getPrecedingTasks().stream().map(Task::getName).forEach(this.precedingTasks::add);
Thanks, for tips : )

Java decoupling issue

am new to Java. I am trying to decouple the Responder class from the WeatherSystem Class. But I get an error at Public NewResponder in the Responder class (invalid method declaration; return type required), I am really stuck at this point. I have tried changing all the class points at NewResponder and responder but can't rectify it. Could anyone point out why I am getting this issue, please?
(I also have InputReader class but that's not included below).
WeatherSystem Class
import java.util.HashSet;
public class WeatherSystem
{
private InputReader reader;
private NewResponder responder;
public WeatherSystem(NewResponder responder)
{
reader = new InputReader();
this.responder = new Responder();
}
public void start()
{
boolean finished = false;
printWelcome();
while(!finished) {
HashSet<String> input = reader.getInput();
if(input.contains("exit")) {
finished = true;
}
else {
String response = this.responder.generateResponse(input);
System.out.println(response);
}
}
printGoodbye();
.............................................
Class Responder
import java.util.HashMap;
import java.util.HashSet;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.Random;
import WeatherSystem.NewResponder
public class Responder implements NewResponder
{
private HashMap<String, String> responseMap;
private ArrayList<String> defaultResponses;
private Random randomGenerator;
public NewResponder()
{
responseMap = new HashMap<>();
defaultResponses = new ArrayList<>();
fillResponseMap();
fillDefaultResponses();
randomGenerator = new Random();
}
public String generateResponse(HashSet<String> words)
{
for (String word : words) {
String response = this.responseMap.get(word);
if(response != null) {
return response;
}
}
return pickDefaultResponse();
....................................................

You really need to check whether you need interface NewResponder inside WeatherSystem. Which you are implementing in Responder class with wrong constructor name.

Array with a blank reult

why can execute my code without any issues and it gives me the correct results.
package com.opennlp.demo;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.HashSet;
import java.util.Set;
import opennlp.tools.cmdline.parser.ParserTool;
import opennlp.tools.parser.Parse;
import opennlp.tools.parser.Parser;
import opennlp.tools.parser.ParserFactory;
import opennlp.tools.parser.ParserModel;
public class ParserTest {
public static Set<String> nounPhrases = new HashSet<>();
private static String line = "I need a PHP note if";
public void getNounPhrases(Parse p) {
if (p.getType().equals("NN") || p.getType().equals("NNS") || p.getType().equals("NNP") || p.getType().equals("NNPS")) {
nounPhrases.add(p.getCoveredText());
//System.out.println(p.getCoveredText());
}
for (Parse child : p.getChildren()) {
getNounPhrases(child);
System.out.println(child.toString()+"lol");
}
}
public void parserAction() throws Exception {
InputStream is = new FileInputStream("en-parser-chunking.bin");
ParserModel model = new ParserModel(is);
Parser parser = ParserFactory.create(model);
Parse topParses[] = ParserTool.parseLine(line, parser, 1);
for (Parse p : topParses){
//p.show();
getNounPhrases(p);
}
}
public static void main(String[] args) throws Exception {
new ParserTest().parserAction();
System.out.println("List of Noun Parse : "+nounPhrases);
}
}
below is my second code.. it gives me a blank Array in console.I removed main method from above class and I tried to print the nounPhrases inside another class. it shows a blank array. I think in second code my parserAction method is not executing. How can I do execute that method without a main method in another class ?
my second code
package com.opennlp.demo;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.HashSet;
import java.util.Set;
import opennlp.tools.cmdline.parser.ParserTool;
import opennlp.tools.parser.Parse;
import opennlp.tools.parser.Parser;
import opennlp.tools.parser.ParserFactory;
import opennlp.tools.parser.ParserModel;
public class ParserTest {
public static Set<String> nounPhrases = new HashSet<>();
//private static String line = "i need a Java book which has JSP also";
private static String line = "I need a PHP note if";
public void getNounPhrases(Parse p) {
if (p.getType().equals("NN") || p.getType().equals("NNS") || p.getType().equals("NNP") || p.getType().equals("NNPS")) {
nounPhrases.add(p.getCoveredText());
//System.out.println(p.getCoveredText());
}
for (Parse child : p.getChildren()) {
getNounPhrases(child);
System.out.println(child.toString()+"lol");
}
}
public void parserAction() throws Exception {
InputStream is = new FileInputStream("en-parser-chunking.bin");
ParserModel model = new ParserModel(is);
Parser parser = ParserFactory.create(model);
Parse topParses[] = ParserTool.parseLine(line, parser, 1);
for (Parse p : topParses){
//p.show();
getNounPhrases(p);
}
}
}
I called the nounPhrases in another class like below. But it shows a blank result. How to fix this ? I need to do this without a main method in this class.
ParserTest pt = new ParserTest();
pt.parserAction();
System.out.println(ParserTest.nounPhrases);