Develop Reference

Apache HttpClient doesn't get content of pages using pagination links. I get 200 status but html has no content

I'm trying to web crawl pages of content with Apache HttpClient. I get status 200 when requesting the next page using the link from pagination but HTML shows 500 in the body and no content. Postman works fine and gets content even when using links from pagination.
Main class
public static void main(String[] args) {
String url = "https://www.cochranelibrary.com/cdsr/reviews/topics";
MyContentFetcher myContentFetcher = new MyContentFetcher();
MyParser myParser = new MyParser();
try {
// Load Topic list page
String topicsPage = myContentFetcher.loadHTML(url);
// Getting all the topics.
Map<Integer, MyNode> topics = myParser.getTopicList(topicsPage);
// Print all the topics and ask user to choose one
for (int id : topics.keySet())
System.out.println("-> " + id + " <- " + topics.get(id).getTopic());
System.out.println("********************");
System.out.print("Enter ID number from the list above to get reviews or enter anything else to exit:\n");
BufferedReader reader = new BufferedReader(new InputStreamReader(System.in));
String id = reader.readLine();
// Validate user input, get the link and topic and cout the choice.
if (isNumber(id)) {
int idNum = Integer.parseInt(id);
if (idNum <= topics.size() && idNum > 0) {
String topic = topics.get(idNum).getTopic();
String link = topics.get(idNum).getLink();
System.out.println("You picked: " + topic + link + "\n***************************");
// Loading first page of reviews
myParser.loadReviews(myContentFetcher.loadHTML(link), topic);
// Getting links to other pages
Queue<String> paginationLinks = myParser.getLinks();
// --------------> WORKS FINE UNTIL HERE <--------------
// Problem starts here....
// Load list of reviews for chosen topic
while(!paginationLinks.isEmpty()) {
String page = myContentFetcher.loadHTML(paginationLinks.remove());
myParser.loadReviews(page, topic);
}
}
}
System.out.println("Exiting...");
} catch (IOException e) {
System.out.println("There was a problem...");
}
!!!! Here is the class that fetches HTML. I'm probably doing something wrong here...
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import java.io.IOException;
import java.util.Scanner;
public class MyContentFetcher {
public MyContentFetcher() {
}
String loadHTML(String url) throws IOException {
// Create configurations for
RequestConfig config = RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.setCookieSpec(CookieSpecs.STANDARD)
.build();
// Creating a HttpClient object
CloseableHttpClient httpClient = HttpClients.custom()
.setDefaultRequestConfig(config)
.build();
// Creating a HttpGet object
HttpGet httpget = new HttpGet(url);
httpget.setHeader("User-Agent", "Mozilla/5.0 (Linux; Android 8.1.0; Pixel Build/OPM4.171019.021.D1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.109 Mobile Safari/537.36 EdgA/42.0.0.2057");
CloseableHttpResponse httpResponse = httpClient.execute(httpget);
Scanner sc = new Scanner(httpResponse.getEntity().getContent());
StringBuilder page = new StringBuilder("");
while(sc.hasNext())
page.append(sc.nextLine()).append(" ");
httpResponse.close();
httpClient.close();
return page.toString();
}
}
Here is the parser. Parser doesn't have any problems (Parse perfectly fine and as needed)
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
public class MyParser {
private Map<String, String> topics;
private Document htmlPage;
private Element reviewBlock;
public MyParser(){}
// Loads all topics from the Cochrane Library into a map -> (Topic Name, Link)
public Map<Integer, MyNode> getTopicList(String page) {
Map<Integer, MyNode> topics= new HashMap<Integer, MyNode>();
htmlPage = Jsoup.parse(page);
// Get 'a' element that is inside 'li' with a class name of browse-by-list-item
int i = 1;
MyNode info;
for(Element element : htmlPage.body().select("li.browse-by-list-item > a")) {
info = new MyNode(element.select("button").text(),
element.select("a").attr("href").trim());
topics.put(i, info);
i++;
}
return topics;
}
// Loads Reviews
public void loadReviews(String page, String topic) throws IOException {
htmlPage = Jsoup.parse(page);
// Get all review blocks
System.out.println("**************************\n" + page + "\n**************************\n");
for(Element element : htmlPage.body().select(".search-results-item-body")){
reviewBlock = element;
String review = getLink() + " | " + topic + " | " + getTitle() + " | " + getAuthor() + " | " + getDate();
System.out.println(review);
}
}
Queue<String> getLinks(){
System.out.println("GETTING LINKS");
Queue<String> links = new LinkedList<>();
for(Element element : htmlPage.body().select("li.pagination-page-list-item > a")) {
links.add(element.attr("href"));
}
return links;
}
private String getLink(){
return "https://www.cochranelibrary.com" + reviewBlock.select("a").attr("href");
}
public String getTitle(){
return reviewBlock.selectFirst("a").text();
}
public String getAuthor(){
return reviewBlock.selectFirst("div.search-result-authors").text();
}
public String getDate(){
String result = reviewBlock.select("div.search-result-date > div").text();
try {
SimpleDateFormat fmt = new SimpleDateFormat("dd MMMM yyyy", Locale.US);
Date d = fmt.parse(result);
fmt.applyPattern("yyyy-MM-dd");
result = fmt.format(d);
} catch (ParseException e) {
System.out.println("Failed parsing the date...");
}
return result;
}

Had I had the permissions, this would have been just a comment.
I ran your loadHtml function with the URL you provided and the result I get is more or less equal to the html of the page.
Could you please provide more details about the httpclient library? I'm using Java 12 (I'm positive it would work with Java 8 too) with this dependency
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.11</version>
</dependency>

To solve this problem I created a session and used cookie store and used CloseableHttpResponse to close the response after each time I fetch the page. Here is the fragment of the code that make it work:
RequestConfig config = RequestConfig.custom()
.setCircularRedirectsAllowed(true)
.build();
httpClient = HttpClients.custom()
.setDefaultRequestConfig(config)
.setMaxConnPerRoute(100)
.build();
CookieStore cookieStore = new BasicCookieStore();
httpContext = new BasicHttpContext();
httpContext.setAttribute(HttpClientContext.COOKIE_STORE, cookieStore);
HttpGet httpget = new HttpGet(url);
httpget.setHeader("User-Agent", "Whatever");
StringBuilder page = new StringBuilder("");
try {
CloseableHttpResponse response = httpClient.execute(httpget, httpContext);
System.out.println(response.getStatusLine());
Scanner sc = new Scanner(response.getEntity().getContent());
while (sc.hasNext())
page.append(sc.nextLine()).append(" ");
sc.close();
response.close();
} catch (IOException e) {
e.printStackTrace();
}
return page.toString();

Crawling amazon.com

I'm crawling amazon products and principle it's going fine.
I have three classes from this nice tutorial:
http://www.netinstructions.com/how-to-make-a-simple-web-crawler-in-java/
I added the files to the following code (class Spider):
import java.io.FileNotFoundException;
import java.util.*;
public class Spider {
public static final int MAX_PAGES_TO_SEARCH = 10000;
private Set<String> pagesVisited = new HashSet<String>();
private List<String> pagesToVisit = new LinkedList<String>();
public void search(String url) {
while (this.pagesVisited.size() < MAX_PAGES_TO_SEARCH) {
String currentUrl;
SpiderLeg leg = new SpiderLeg();
if (this.pagesToVisit.isEmpty()) {
//System.out.println("abc");
currentUrl = url;
this.pagesVisited.add(url);
} else {
//System.out.println("def");
currentUrl = this.nextUrl();
}
try {
Thread.sleep(10000);
leg.crawl(currentUrl); // Lots of stuff happening here. Look at the crawl method in
} catch (FileNotFoundException e) {
System.out.println("Oops, FileNotFoundException caught");
} catch (InterruptedException e) {
e.printStackTrace();
}
this.pagesToVisit.addAll(leg.getLinks());
//System.out.println("Test");
}
System.out.println("\n**Done** Visited " + this.pagesVisited.size() + " web page(s)");
SpiderLeg leg = new SpiderLeg();
leg.calcAdjMatrix();
for (int i = 0; i < leg.adjMatrix.length; i++) {
System.out.println(Arrays.toString(leg.adjMatrix[i]));
}
}
private String nextUrl() {
String nextUrl;
do {
if (this.pagesToVisit.isEmpty()){
return "https://www.amazon.de/Proband-Thriller-Guido-Kniesel/dp/1535287004/ref=sr_1_1?s=books&ie=UTF8&qid=1478247246&sr=1-1&keywords=%5B%5D";
}
nextUrl = this.pagesToVisit.remove(0);
} while (this.pagesVisited.contains(nextUrl));
this.pagesVisited.add(nextUrl);
return nextUrl;
}
}
class SpiderLeg:
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.*;
import java.util.*;
public class SpiderLeg {
// We'll use a fake USER_AGENT so the web server thinks the robot is a normal web browser.
private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36";
private static List<String> links = new LinkedList<String>();
private static String graphLink;
private Document htmlDocument;
private static double counter = 0;
static Map<String, Set<String>> adjMap = new HashMap<String, Set<String>>();
static int[][] adjMatrix;
static List<String> mapping;
public boolean crawl(String url) throws FileNotFoundException {
if (url.isEmpty()) {
return false;
}
try{
Connection connection = Jsoup.connect(url).ignoreContentType(true).userAgent(USER_AGENT);
Document htmlDocument = connection.get();
this.htmlDocument = htmlDocument;
if(connection.response().statusCode() == 200){
// 200 is the HTTP OK status code
// indicating that everything is great.
counter++;
double progress;
progress = (counter/Spider.MAX_PAGES_TO_SEARCH)*100;
System.out.println("\n**Visiting** Received web page at " + url);
System.out.println("\n**Progress** " + progress + "%");
}
if(!connection.response().contentType().contains("text/html")) {
System.out.println("**Failure** Retrieved something other than HTML");
return false;
}
//Elements linksOnPage = htmlDocument.select("a[href*=/gp/product/]");
Elements linksOnPage = htmlDocument.select("a[href*=/dp/]");
Elements salesRank = htmlDocument.select("span.zg_hrsr_rank");
Elements category = htmlDocument.select("span.zg_hrsr_ladder a");
String categoryString = category.html();
String salesRankString = salesRank.html();
salesRankString = salesRankString.replace("\n", " ");
categoryString = categoryString.replace("\n", " ");
//System.out.println(categoryString);
System.out.println("Found (" + linksOnPage.size() + ") links");
PrintWriter pw = new PrintWriter(new FileWriter("Horror.csv", true));
StringBuilder sb = new StringBuilder();
int beginIndex = url.indexOf(".de/");
int endIndex = url.indexOf("/dp");
String title = url.substring(beginIndex+4,endIndex);
if(!adjMap.containsKey(title)){
if(categoryString.contains("Horror")){
adjMap.put(title, new HashSet<String>());
sb.append(title);
sb.append(',');
sb.append(salesRankString);
sb.append(',');
sb.append(categoryString);
sb.append(',');
for(Element link : linksOnPage){
String graphLink = link.attr("abs:href");
if(!graphLink.contains("one-click")){
if(!graphLink.contains("Kindle")){
if(!graphLink.contains("unsticky")){
this.links.add(graphLink);
//adjMap.get(url).add(graphLink);
adjMap.get(title).add(cutTitle(graphLink));
sb.append(graphLink);
sb.append(',');
}
}
}
}
sb.append('\n');
pw.write(sb.toString());
pw.close();
}
}
System.out.println("done!");
return true;
}
catch(IOException ioe) {
// We were not successful in our HTTP request
System.out.println("Error in out HTTP request " + ioe);
return false;
}
}
public static void calcAdjMatrix(){
Set<String> allMyURLs = new HashSet(adjMap.keySet());
for(String s: adjMap.keySet()){
allMyURLs.addAll(adjMap.get(s));
System.out.println(s + "\t" + adjMap.get(s));
}
int dim = allMyURLs.size();
adjMatrix = new int[dim][dim];
List<String> nodes_list = new ArrayList<>();
for(String s: allMyURLs){
nodes_list.add(s);
}
for(String s: nodes_list){
Set<String> outEdges = adjMap.get(s);
int i = nodes_list.indexOf(s);
if(outEdges != null){
for(String s1: outEdges){
int j = nodes_list.indexOf(s1);
adjMatrix[i][j] = 1;
}
}
}
}
public String cutTitle(String url) throws FileNotFoundException{
int beginIndex = url.indexOf(".de/");
int endIndex = url.indexOf("/dp");
String title;
if(url.contains(".de") && url.contains("/dp")){
title = url.substring(beginIndex+4,endIndex);
}else{
title = "wrong url";
}
return title;
}
public boolean searchForWord(String searchWord) {
if(this.htmlDocument == null) {
System.out.println("ERROR! Call crawl() before performing analysis on the document");
return false;
}
System.out.println("Searching for the word " + searchWord + "...");
String bodyText = this.htmlDocument.body().text();
return bodyText.toLowerCase().contains(searchWord.toLowerCase());
}
public List<String> getLinks(){
return this.links;
}
}
class SpiderTest:
public class SpiderTest {
public static void main(String[] args) {
Spider spider = new Spider();
spider.search("https://www.amazon.de/Wille-geschehe-Psychothriller-Guido-Kniesel/dp/1537455389/ref=pd_sim_14_1?_encoding=UTF8&psc=1&refRID=CQPDDGY4BJ4D8THNNSZ6");
}
}
Now the problem is, that after 100 URLs I think, that amazon is banning me from the server. The program doesn't find URLs anymore.
Does anyone has an idea how I can fix that?

Well, don't be rude and crawl them then.
Check their robots.txt (wiki) to see what they allow you to do. Don't be surprised if they ban you if you go places they don't want you to go.

The problem is very common when you try to crawl big websites that don't want to be crawled. They basically block you for a period of time to prevent their data being crawled or stolen.
With that being said, you have two options, either make each request from a different IP/server which will make your requests look legit and avoid the ban, or go for the easiest way which is to use a service that does that for you.
I've done both and the first one is complex, takes time and needs maintenance (you have to build a network of servers), the second option is usually not free but very fast to implement and guarantees that all your requests will always return data and you won't be banned.
There are some services on the internet that does that. I've used proxycrawl (which also has a free tier) in the past which works very good. They have an API that you can call and you only can use your same code, just changing the url you call.
This would be an example for amazon:
GET https://api.proxycrawl.com?token=yourtoken&url=https://amazon.com
And you would get always a response, even if you crawl 1000 pages a second, you will never be banned as you will be calling that proxy instead which does all the magic for you.
I hope it helps :)

You can try using proxy servers to prevent being blocked. There are services providing working proxies. I have good experience using https://gimmeproxy.com which specifically has proxies supporting amazon.
To get proxy working with Amazon, you need just to make the following request:
https://gimmeproxy.com/api/getProxy?api_key=your_api_key&websites=amazon
You will get JSON response with all proxy data which you can use later as needed:
{
"supportsHttps": true,
"protocol": "socks5",
"ip": "116.182.122.182",
"port": "1915",
"get": true,
"post": true,
"cookies": true,
"referer": true,
"user-agent": true,
"anonymityLevel": 1,
"websites": {
"example": true,
"google": false,
"amazon": true
},
"country": "BR",
"tsChecked": 1517952910,
"curl": "socks5://116.182.122.182:1915",
"ipPort": "116.182.122.182:1915",
"type": "socks5",
"speed": 37.78,
"otherProtocols": {}
}

HtmlViewService in OBIEE Web Service API is not rendering report (it's only showing spinning loader)

I have a Java application that leverages the OBIEE Web Service API to consume data from the BI Server. I am able to XMLViewService and the WebCatalogService just fine, but I can't quite get the HtmlViewService to properly render a report in the Java app. The report just shows the spinning loader, but never actually renders the report. I'm pretty sure it has to do with the fact that the Java app and the BI Server are on different domains. This is what the API documentation says:
In situations where Oracle BI Web Services and the third-party Web server do not belong to the same Domain Name Service (DNS) domain, users may get JavaScript errors related to browser security constraints for cross-domain scripting. To avoid these issues, use the setBridge() method to modify callback URLs to point to the third-party Web server. Be aware that a Web component executed by the third-party Web server to re-route requests to Oracle BI Web Services is not provided. This function would need to be fulfilled by the third-party application.
Several years ago, I did this same type of integration using .NET/C# and ran in the the same issue because the .NET app and the BI Server were on different domains. As a result, I had to create an HTTP Handler (.ashx file) as well as use the setBridge() method to to solve the issue.
The challenge that I'm having is that I can't find a servlet bridge example for Java. And I'm not too confident in porting the .NET/.ASHX code to a Java servlet/bridge. Does anyone have any code examples or direction they could provide to point me in the right direction? Here's a snippet of code to show you what I'm doing to pull back the report data:
// define report path
ReportRef reportRef = new ReportRef();
reportRef.setReportPath(reportFolder + "/" + reportName);
// set page params
StartPageParams pageParams = new StartPageParams();
pageParams.setDontUseHttpCookies(true);
// set report params
String pageId = htmlService.startPage(pageParams, sawSessionId);
String reportId = pageId + reportName;
htmlService.addReportToPage(pageId, reportId, reportRef, null, null, null, sawSessionId);
// get report html
StringBuffer reportHtml = new StringBuffer();
reportHtml.append(htmlService.getHtmlForReport(pageId, reportId, sawSessionId));
// return html
return reportHtml.toString();
This is the error that is coming back in the browser:
XMLHttpRequest cannot load http://myobiserver.com/analytics/saw.dll?ajaxGo. No 'Access-Control-Allow-Origin' header is present on the requested resource. Origin 'http://myjavaapp.com' is therefore not allowed access.
Per requested, here is my .NET/.ASHX bridge:
using System.Collections.Specialized;
using System.Net;
using System.Text;
using System.Web;
using System;
using System.Collections;
using System.Configuration;
using System.Collections.Specialized;
using System.Web;
using System.Text;
using System.Net;
using System.IO;
using System.Diagnostics;
/*
This is a ASP.NET handler that handles communication
between the SharePoint site and OracleBI.
It will be deployed to:
C:\Program Files\Common Files\Microsoft Shared\web server extensions\12\TEMPLATE\LAYOUTS\OracleBI
*/
public class OracleBridge: IHttpHandler
{
public bool IsReusable {get{return true;}}
public OracleBridge()
{
}
string getServer()
{
string strServer = "http://<enter-domain>/analytics/saw.dll";
int index = strServer.LastIndexOf("/");//split off saw.dll
if (index >=0)
return strServer.Substring(0,index+1);
else
return strServer;
}
public void ProcessRequest(HttpContext context)
{
HttpWebRequest req = forwardRequest(context);
forwardResponse(context,req);
}
private HttpWebRequest forwardRequest(HttpContext context)
{
string strURL = makeURL(context);
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(strURL);
req.Method = context.Request.HttpMethod;
NameValueCollection headers = context.Request.Headers;
req.Accept = headers.Get("Accept");
req.Expect = headers.Get("Expect");
req.ContentType = headers.Get("Content-Type");
string strModifiedSince = headers.Get("If-Modified-Since");
if (strModifiedSince != null && strModifiedSince.Length != 0)
req.IfModifiedSince = DateTime.Parse(strModifiedSince);
req.Referer = headers.Get("Referer");
req.UserAgent = headers.Get("User-Agent");
if (!req.Method.Equals("GET"))
{
CopyStreams(context.Request.InputStream,req.GetRequestStream());
}
return req;
}
private void forwardResponse(HttpContext context, HttpWebRequest req)
{
HttpWebResponse resp =null;
try
{
resp = (HttpWebResponse)req.GetResponse();
}
catch(WebException e)
{
resp = (HttpWebResponse)e.Response;
}
context.Response.StatusCode = (int)resp.StatusCode;
for (int i = 0; i < resp.Cookies.Count; i++)
{
Cookie c = resp.Cookies[i];
HttpCookie hc = new HttpCookie(c.Name, c.Value);
hc.Path = c.Path;
hc.Domain = getServer();
context.Response.Cookies.Add(hc);
}
context.Response.ContentType = resp.ContentType;
CopyStreams(resp.GetResponseStream(), context.Response.OutputStream);
}
private string makeURL(HttpContext context)
{
string strQuery = context.Request.Url.Query;
string[] arrParams = strQuery.Split('?','&');
StringBuilder resultingParams = new StringBuilder();
string strURL=null;
foreach(string strParam in arrParams )
{
string[] arrNameValue = strParam.Split('=');
if (!arrNameValue[0].Equals("RedirectURL"))
{
if (strParam.Length != 0)
{
if (resultingParams.Length != 0)
resultingParams.Append("&");
resultingParams.Append(strParam);
}
}
else if (arrNameValue.Length >1)
strURL = HttpUtility.UrlDecode(arrNameValue[1]);
}
if (strURL ==null)
throw new Exception("Invalid URL format. requestURL parameter is missing");
String sAppendChar = strURL.Contains("?") ? "&" : "?";
if (strURL.StartsWith("http:") || strURL.StartsWith("https:"))
{
String tmpURL = strURL + sAppendChar + resultingParams.ToString();
return tmpURL;
}
else
{
String tmpURL = getServer() + strURL + sAppendChar + resultingParams.ToString();
return tmpURL;
}
}
private void CopyStreams(Stream inStr,Stream outStr)
{
byte[] buf = new byte[4096];
try
{
do
{
int iRead = inStr.Read(buf,0,4096);
if (iRead == 0)
break;
outStr.Write(buf,0,iRead);
}
while (true);
}
finally
{
outStr.Close();
}
}
}

Using the BridgeServlet from link (http://pastebin.com/NibVnBLb) posted in previous answer did not work for us. In our web portal, embedding Oracle BI dashboard using BridgeServlet above, redirected us to OBI login page and console log showed incorrect resources(js/css) web links (local URL instead of OBIEE URL's).
Instead we used this class (with some minor adjustments) https://gist.github.com/rafaeltuelho/9376341#file-obieehttpservletbridge-java.
Tested with Java 11, Oracle Business Intelligence 12.2.1.4.0, WSDL v12 of OBIEE (http://OBIEE-server:port/analytics/saw.dll/wsdl/v12), with SSO disabled.
Here it is the BridgeServlet class:
package com.abs.bi;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLEncoder;
import java.util.Enumeration;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import javax.servlet.ServletException;
import javax.servlet.http.Cookie;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
/**
* Servlet implementation class OBIEEBridge
*/
public class BridgeServlet extends HttpServlet {
private static final long serialVersionUID = 1L;
/**
* #see HttpServlet#HttpServlet()
*/
public BridgeServlet() {
super();
}
/**
* #see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse
* response)
*/
protected void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
try {
this.processRequest(request, response);
} catch (Exception e) {
throw new ServletException(e);
}
}
/**
* #see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse
* response)
*/
protected void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
try {
this.processRequest(request, response);
} catch (Exception e) {
throw new ServletException(e);
}
}
protected void processRequest(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
HttpURLConnection urlCon = forwardRequest(request);
forwardResponse(response, urlCon);
}
#SuppressWarnings("unchecked")
private String decodeURL(HttpServletRequest request) {
StringBuffer bufURL = new StringBuffer("");
Map<String, String[]> params = request.getParameterMap();
String[] arrURL = params.get("RedirectURL");
String strURL = arrURL == null || arrURL.length == 0 ? null : arrURL[0];
bufURL.append(strURL);
int nQIndex = strURL.lastIndexOf('?');
if (params != null && !params.isEmpty()) {
bufURL.append(((nQIndex >= 0) ? "&" : "?"));
Set<String> keys = params.keySet();
Iterator<String> it = keys.iterator();
while (it.hasNext()) {
try {
String strKey = it.next();
if (strKey.equalsIgnoreCase("RedirectURL")) {
continue;
}
String strEncodedKey = URLEncoder.encode(strKey, "UTF-8");
String[] paramValues = params.get(strKey);
for (String paramValue : paramValues) {
bufURL.append(strEncodedKey);
bufURL.append("=");
bufURL.append(URLEncoder.encode(paramValue, "UTF-8"));
bufURL.append("&");
}
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
}
bufURL.deleteCharAt(bufURL.length() - 1);
}
return bufURL.toString();
}
#SuppressWarnings("unchecked")
private HttpURLConnection forwardRequest(HttpServletRequest request) throws IOException {
String strURL = decodeURL(request);
String[] arrURL = strURL.split("&", 2);
String baseURL = arrURL[0];
URL url = new URL(baseURL);
HttpURLConnection con = (HttpURLConnection) url.openConnection();
String strMethod = request.getMethod();
con.setRequestMethod(strMethod);
Enumeration<String> en = request.getHeaderNames();
String strHeader;
while (en.hasMoreElements()) {
strHeader = en.nextElement();
String strHeaderValue = request.getHeader(strHeader);
con.setRequestProperty(strHeader, strHeaderValue);
}
// is not a HTTP GET request
if (strMethod.compareTo("GET") != 0) {
con.setDoOutput(true);
con.setDoInput(true);
con.setUseCaches(false);
DataOutputStream forwardStream = new DataOutputStream(con.getOutputStream());
try {
String urlParameters = arrURL[1];
forwardStream.writeBytes(urlParameters);
forwardStream.flush();
} finally {
forwardStream.close();
}
}
return con;
}
private void forwardResponse(HttpServletResponse response, HttpURLConnection con) throws IOException {
int nContentLen = -1;
String strKey;
String strValue;
try {
response.setStatus(con.getResponseCode());
for (int i = 1; true; ++i) {
strKey = con.getHeaderFieldKey(i);
strValue = con.getHeaderField(i);
if (strKey == null) {
break;
}
if (strKey.equals("Content-Length")) {
nContentLen = Integer.parseInt(con.getHeaderField(i));
continue;
}
if (strKey.equalsIgnoreCase("Connection") || strKey.equalsIgnoreCase("Server")
|| strKey.equalsIgnoreCase("Transfer-Encoding") || strKey.equalsIgnoreCase("Content-Length")) {
continue; // skip certain headers
}
if (strKey.equalsIgnoreCase("Set-Cookie")) {
String[] cookieStr1 = strValue.split(";");
String[] cookieStr2 = cookieStr1[0].split("=");
// String[] cookieStr3 = cookieStr1[1].split("=");
/*
* Change the Set-Cookie HTTP Header to remove the 'path' attribute. Thus the
* browser can accept the ORA_BIPS_NQID cookie from Oracle BI Server
*/
Cookie c = new Cookie(cookieStr2[0], cookieStr2[1]);
c.setPath("/");
response.addCookie(c);
} else {
response.setHeader(strKey, strValue);
}
}
copyStreams(con.getInputStream(), response.getOutputStream(), nContentLen);
} finally {
response.getOutputStream().close();
con.getInputStream().close();
}
}
private void copyStreams(InputStream inputStream, OutputStream forwardStream, int nContentLen) throws IOException {
byte[] buf = new byte[1024];
int nCount = 0;
int nBytesToRead = 1024;
int nTotalCount = 0;
do {
if (nContentLen != -1)
nBytesToRead = nContentLen - nTotalCount > 1024 ? 1024 : nContentLen - nTotalCount;
if (nBytesToRead == 0)
break;
// try to read some bytes from src stream
nCount = inputStream.read(buf, 0, nBytesToRead);
if (nCount < 0)
break;
nTotalCount += nCount;
// try to write some bytes in target stream
forwardStream.write(buf, 0, nCount);
} while (true);
}
}
AbsServiceUtils which contains SAWSessionService and HtmlViewService web service calls to Oracle BI server:
package com.abs.bi;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.jsp.JspWriter;
import oracle.bi.web.soap.AuthResult;
import oracle.bi.web.soap.HtmlViewService;
import oracle.bi.web.soap.HtmlViewServiceSoap;
import oracle.bi.web.soap.ReportHTMLLinksMode;
import oracle.bi.web.soap.ReportHTMLOptions;
import oracle.bi.web.soap.ReportRef;
import oracle.bi.web.soap.SAWLocale;
import oracle.bi.web.soap.SAWSessionParameters;
import oracle.bi.web.soap.SAWSessionService;
import oracle.bi.web.soap.SAWSessionServiceSoap;
import oracle.bi.web.soap.StartPageParams;
public class AbsServiceUtils {
private AbsServiceUtils() {
}
public static URL buildWsdlUrl() throws MalformedURLException {
return new URL(IAbsService.BASEWSDLURL);
}
public static void writeBiContent(HttpServletRequest request, JspWriter out, String biReport) throws IOException {
String userAgent = request.getHeader("User-Agent");
Locale userLocale = request.getLocale();
String bridgeServletContextPath = request.getContextPath() + "/bridgeservlet";
String reportHtml = writeBiContent(biReport, userAgent, userLocale, bridgeServletContextPath);
if (out != null) {
out.println(reportHtml);
}
}
public static String writeBiContent(String biReport, String userAgent, Locale userLocale,
String bridgeServletContextPath) throws MalformedURLException {
HtmlViewService htmlViewService = new HtmlViewService(buildWsdlUrl());
HtmlViewServiceSoap htmlClient = htmlViewService.getHtmlViewService();
SAWSessionService sAWSessionService = new SAWSessionService(buildWsdlUrl());
SAWSessionServiceSoap myPort = sAWSessionService.getSAWSessionServiceSoap();
SAWSessionParameters sessionparams = new SAWSessionParameters();
sessionparams.setUserAgent(userAgent);
SAWLocale sawlocale = new SAWLocale();
sawlocale.setLanguage(userLocale.getLanguage());
sawlocale.setCountry(userLocale.getCountry());
sessionparams.setLocale(sawlocale);
sessionparams.setAsyncLogon(false);
AuthResult result = myPort.logonex(IAbsService.BIUSERNAME, IAbsService.BIPASSWORD, sessionparams);
String sessionID = result.getSessionID();
List<String> keepAliveSessionList = new ArrayList<>(1);
keepAliveSessionList.add(sessionID);
myPort.keepAlive(keepAliveSessionList);
StartPageParams spparams = new StartPageParams();
spparams.setDontUseHttpCookies(true);
String pageID = htmlClient.startPage(spparams, sessionID);
/**
* This method will set the path to the servlet which will act like a bridge to
* retrieve all the OBIEE resources like the javascript, CSS and the report.
*/
if (bridgeServletContextPath != null) {
htmlClient.setBridge(bridgeServletContextPath, sessionID);
}
ReportHTMLOptions htmlOptions = new ReportHTMLOptions();
htmlOptions.setEnableDelayLoading(false);
htmlOptions.setLinkMode(ReportHTMLLinksMode.IN_PLACE.value());
ReportRef reportref = new ReportRef();
reportref.setReportPath(IAbsService.BIROOTPATH + biReport);
StartPageParams startpageparams = new StartPageParams();
startpageparams.setDontUseHttpCookies(false);
htmlClient.addReportToPage(pageID, biReport.replace(" ", ""), reportref, null, null, htmlOptions, sessionID);
String reportHtml = htmlClient.getHeadersHtml(pageID, sessionID);
reportHtml = reportHtml + htmlClient.getHtmlForReport(pageID, biReport.replace(" ", ""), sessionID);
reportHtml = reportHtml + htmlClient.getCommonBodyHtml(pageID, sessionID);
return reportHtml;
}
}
IAbsService:
package com.abs.bi;
public interface IAbsService {
public static final String BASEWSDLURL = "http://<OracleBIServer:port>/analytics/saw.dll/wsdl/v12";
public static final String BIUSERNAME = "USER";
public static final String BIPASSWORD = "PASS";
public static final String BIROOTPATH = "/shared/sharedfolder/";
public static final String BIREPORTNAME = "report";
}

Use the link http://pastebin.com/NibVnBLb to check out the bridge code for java. Hope this might be helful.

any frameworks to handle REST Client server list? [closed]

Closed. This question does not meet Stack Overflow guidelines. It is not currently accepting answers.
Questions asking us to recommend or find a tool, library or favorite off-site resource are off-topic for Stack Overflow as they tend to attract opinionated answers and spam. Instead, describe the problem and what has been done so far to solve it.
Closed 9 years ago.
Improve this question
The idea is that REST Client can be configured with list of REST Servers. So the servers list will rotate on the REST Client in a round robin fashion.
e.g. REST Client Application. I'll be configuring server list (REST_SERVER1,REST_SERVER2, REST_SERVER3)
1 request -> REST_SERVER1
2 request -> REST_SERVER2
3 request -> REST_SERVER3
4 request -> REST_SERVER1
I searched so much I couldn't found a proper framework that supports this functionality.

I'd just wrap your client then repeat the request until it succeeds. That way you can neatly keep track of which servers are working. Here's some code I adapted from our system...
import java.io.IOException;
import java.net.URL;
import java.util.Calendar;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
public class RestClient {
private final String mediaType = "application/json";
private RestServer[] servers = new RestServer[] {new RestServer("server1", 8080), new RestServer("server2", 8080)};
protected RestClient() {
}
protected ClientResponse post(String methodUrl, Object postData) throws IOException {
return doRequest(methodUrl, postData, true);
}
protected ClientResponse get(String methodUrl) throws IOException {
return doRequest(methodUrl, null, false);
}
private ClientResponse doRequest(String methodUrl, Object postData, boolean isPost) throws IOException {
Client client = Client.create();
for (RestServer restServer : servers) {
if (!restServer.shouldTry()) {
System.out.println(restServer + " not ready");
continue;
}
System.out.println("Trying with " + restServer);
try {
URL url = new URL("http", restServer.getHost(), restServer.getPort(), '/' + methodUrl);
WebResource webResource = client.resource(url.toString());
System.out.println("Calling " + url);
ClientResponse response = isPost
? webResource.type(mediaType).post(ClientResponse.class, postData)
: webResource.type(mediaType).get(ClientResponse.class);
if (response.getStatus() < 300) {
restServer.succeeded();
return response;
}
restServer.failed();
} catch (Exception ex) {
System.out.println(restServer + " failed with exception " + ex.getMessage());
restServer.failed();
}
}
// No servers worked
return null;
}
}
class RestServer {
private final int TIME_TO_WAIT_BEFORE_TRYING_AGAIN = 1000 * 30; // 30 seconds
private String host;
private int port;
private Calendar lastAttempted;
private boolean lastCallFailed;
public RestServer(String host, int port) {
this.host = host;
this.port = port;
lastAttempted = Calendar.getInstance();
}
public String getHost() {
return host;
}
public int getPort() {
return port;
}
public void failed() {
lastCallFailed = true;
lastAttempted = Calendar.getInstance();
}
public void succeeded() {
lastCallFailed = false;
lastAttempted = Calendar.getInstance();
}
public boolean shouldTry() {
if (!lastCallFailed)
return true;
return Calendar.getInstance().compareTo(lastAttempted) > TIME_TO_WAIT_BEFORE_TRYING_AGAIN;
}
#Override
public String toString() {
return new StringBuilder(host).append(':').append(port).toString();
}
}

On the fear adding a bunch of components to complicate your application setup, I might consider a quick-and-dirty, pure Java solution.
Here's something fun I came up with using Spring's RestTemplate. If you are comfortable with interceptors, aspects, other things that can wrap up a method call, you can apply these principles to wrap up all of the various RestTemplate REST calls. See the RestTemplate javadoc
import org.junit.Test;
import org.springframework.http.HttpMethod;
import org.springframework.web.client.*;
import org.springframework.web.util.UriTemplate;
import org.springframework.web.util.UriUtils;
import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Iterator;
import java.util.Map;
public class Stuff {
// Make this configurable.
Collection<String> serverList;
// or do something a little smarter, like this interface. I'll use this in this example.
ServerLookup serverLookup;
interface ServerLookup {
Iterator<String> getValidServerListIterator();
void markUnreachableServer(String url);
}
// Do it externally around RestTemplate...
#Test
public void testNormalRestTemplate() throws Exception {
RestTemplate restTemplate = new RestTemplate();
Iterator<String> serverIterator = serverLookup.getValidServerListIterator();
while (serverIterator.hasNext()) {
String server = serverIterator.next();
try {
Object obj = restTemplate.getForObject(server + "/objectIdentifier/511", Object.class);
break;
} catch (ResourceAccessException e) {
serverLookup.markUnreachableServer(server);
}
}
}
// or you can try to 'enhance' RestTemplate to contain the retry logic within. It's a bit hacky, but more fun.
#Test
public void testMyRestTemplate() {
RestTemplate rt = new MyRestTemplate();
Object obj = rt.getForObject("/objectIdentifier/511", Object.class);
rt.delete("/objectIdentifier/511");
}
// Here's a way to (hackily) augment RestTemplate with retry functionality
class MyRestTemplate extends RestTemplate {
// Unfortunately RestTemplate probably wasn't designed for much extensibility. URI objects can't be made from
// URL fragments, so these two methods are the 'furthest in' that we can override and cover all RestTemplate
// REST methods.
#Override
public <T> T execute(String url, HttpMethod method, RequestCallback requestCallback,
ResponseExtractor<T> responseExtractor, Object... urlVariables) throws RestClientException {
Iterator<String> serverIterator = serverLookup.getValidServerListIterator();
while (serverIterator.hasNext()) {
String server = serverIterator.next();
// prefix the URL fragment passed in with a server
String fullUrl = server + url;
UriTemplate uriTemplate = new HttpUrlTemplate(fullUrl);
URI expanded = uriTemplate.expand(urlVariables);
try {
return doExecute(expanded, method, requestCallback, responseExtractor);
} catch (ResourceAccessException e) {
serverLookup.markUnreachableServer(server);
}
}
throw new RuntimeException("Unable to reach any servers in the server list for " + url);
}
#Override
public <T> T execute(String url, HttpMethod method, RequestCallback requestCallback,
ResponseExtractor<T> responseExtractor, Map<String, ?> urlVariables) throws RestClientException {
Iterator<String> serverIterator = serverLookup.getValidServerListIterator();
while (serverIterator.hasNext()) {
String server = serverIterator.next();
// prefix the URL fragment passed in with a server
String fullUrl = server + url;
UriTemplate uriTemplate = new HttpUrlTemplate(fullUrl);
URI expanded = uriTemplate.expand(urlVariables);
try {
return doExecute(expanded, method, requestCallback, responseExtractor);
} catch (ResourceAccessException e) {
serverLookup.markUnreachableServer(server);
}
}
throw new RuntimeException("Unable to reach any servers in the server list for " + url);
}
/** Exact duplicate of the inner class of RestTemplate. Can not touch privates. */
class HttpUrlTemplate extends UriTemplate {
public HttpUrlTemplate(String uriTemplate) {
super(uriTemplate);
}
#Override
protected URI encodeUri(String uri) {
try {
String encoded = UriUtils.encodeHttpUrl(uri, "UTF-8");
return new URI(encoded);
}
catch (UnsupportedEncodingException ex) {
// should not happen, UTF-8 is always supported
throw new IllegalStateException(ex);
}
catch (URISyntaxException ex) {
throw new IllegalArgumentException("Could not create HTTP URL from [" + uri + "]: " + ex, ex);
}
}
}
}
}

using dbpedia spotlight in java or scala

Does anyone know where to find a little how to on using dbpedia spotlight in java or scala? Or could anyone explain how it's done? I can't find any information on this...

The DBpedia Spotlight wiki pages would be a good place to start.
And I believe the installation page has listed the most popular ways (using a jar, or set up a web service) to use the application.
It includes instructions on using the Java/Scala API with your own installation, or calling the Web Service.
There are some additional data needed to be downloaded to run your own server for full service, good time to make a coffee for yourself.

you need download dbpedia spotlight (jar file) after that u can use next two classes ( author pablomendes ) i only make some change .
public class db extends AnnotationClient {
//private final static String API_URL = "http://jodaiber.dyndns.org:2222/";
private static String API_URL = "http://spotlight.dbpedia.org:80/";
private static double CONFIDENCE = 0.0;
private static int SUPPORT = 0;
private static String powered_by ="non";
private static String spotter ="CoOccurrenceBasedSelector";//"LingPipeSpotter"=Annotate all spots
//AtLeastOneNounSelector"=No verbs and adjs.
//"CoOccurrenceBasedSelector" =No 'common words'
//"NESpotter"=Only Per.,Org.,Loc.
private static String disambiguator ="Default";//Default ;Occurrences=Occurrence-centric;Document=Document-centric
private static String showScores ="yes";
#SuppressWarnings("static-access")
public void configiration(double CONFIDENCE,int SUPPORT,
String powered_by,String spotter,String disambiguator,String showScores){
this.CONFIDENCE=CONFIDENCE;
this.SUPPORT=SUPPORT;
this.powered_by=powered_by;
this.spotter=spotter;
this.disambiguator=disambiguator;
this.showScores=showScores;
}
public List<DBpediaResource> extract(Text text) throws AnnotationException {
LOG.info("Querying API.");
String spotlightResponse;
try {
String Query=API_URL + "rest/annotate/?" +
"confidence=" + CONFIDENCE
+ "&support=" + SUPPORT
+ "&spotter=" + spotter
+ "&disambiguator=" + disambiguator
+ "&showScores=" + showScores
+ "&powered_by=" + powered_by
+ "&text=" + URLEncoder.encode(text.text(), "utf-8");
LOG.info(Query);
GetMethod getMethod = new GetMethod(Query);
getMethod.addRequestHeader(new Header("Accept", "application/json"));
spotlightResponse = request(getMethod);
} catch (UnsupportedEncodingException e) {
throw new AnnotationException("Could not encode text.", e);
}
assert spotlightResponse != null;
JSONObject resultJSON = null;
JSONArray entities = null;
try {
resultJSON = new JSONObject(spotlightResponse);
entities = resultJSON.getJSONArray("Resources");
} catch (JSONException e) {
//throw new AnnotationException("Received invalid response from DBpedia Spotlight API.");
}
LinkedList<DBpediaResource> resources = new LinkedList<DBpediaResource>();
if(entities!=null)
for(int i = 0; i < entities.length(); i++) {
try {
JSONObject entity = entities.getJSONObject(i);
resources.add(
new DBpediaResource(entity.getString("#URI"),
Integer.parseInt(entity.getString("#support"))));
} catch (JSONException e) {
LOG.error("JSON exception "+e);
}
}
return resources;
}
}
second class
/**
* #author pablomendes
*/
public abstract class AnnotationClient {
public Logger LOG = Logger.getLogger(this.getClass());
private List<String> RES = new ArrayList<String>();
// Create an instance of HttpClient.
private static HttpClient client = new HttpClient();
public List<String> getResu(){
return RES;
}
public String request(HttpMethod method) throws AnnotationException {
String response = null;
// Provide custom retry handler is necessary
method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
new DefaultHttpMethodRetryHandler(3, false));
try {
// Execute the method.
int statusCode = client.executeMethod(method);
if (statusCode != HttpStatus.SC_OK) {
LOG.error("Method failed: " + method.getStatusLine());
}
// Read the response body.
byte[] responseBody = method.getResponseBody(); //TODO Going to buffer response body of large or unknown size. Using getResponseBodyAsStream instead is recommended.
// Deal with the response.
// Use caution: ensure correct character encoding and is not binary data
response = new String(responseBody);
} catch (HttpException e) {
LOG.error("Fatal protocol violation: " + e.getMessage());
throw new AnnotationException("Protocol error executing HTTP request.",e);
} catch (IOException e) {
LOG.error("Fatal transport error: " + e.getMessage());
LOG.error(method.getQueryString());
throw new AnnotationException("Transport error executing HTTP request.",e);
} finally {
// Release the connection.
method.releaseConnection();
}
return response;
}
protected static String readFileAsString(String filePath) throws java.io.IOException{
return readFileAsString(new File(filePath));
}
protected static String readFileAsString(File file) throws IOException {
byte[] buffer = new byte[(int) file.length()];
#SuppressWarnings("resource")
BufferedInputStream f = new BufferedInputStream(new FileInputStream(file));
f.read(buffer);
return new String(buffer);
}
static abstract class LineParser {
public abstract String parse(String s) throws ParseException;
static class ManualDatasetLineParser extends LineParser {
public String parse(String s) throws ParseException {
return s.trim();
}
}
static class OccTSVLineParser extends LineParser {
public String parse(String s) throws ParseException {
String result = s;
try {
result = s.trim().split("\t")[3];
} catch (ArrayIndexOutOfBoundsException e) {
throw new ParseException(e.getMessage(), 3);
}
return result;
}
}
}
public void saveExtractedEntitiesSet(String Question, LineParser parser, int restartFrom) throws Exception {
String text = Question;
int i=0;
//int correct =0 ; int error = 0;int sum = 0;
for (String snippet: text.split("\n")) {
String s = parser.parse(snippet);
if (s!= null && !s.equals("")) {
i++;
if (i<restartFrom) continue;
List<DBpediaResource> entities = new ArrayList<DBpediaResource>();
try {
entities = extract(new Text(snippet.replaceAll("\\s+"," ")));
System.out.println(entities.get(0).getFullUri());
} catch (AnnotationException e) {
// error++;
LOG.error(e);
e.printStackTrace();
}
for (DBpediaResource e: entities) {
RES.add(e.uri());
}
}
}
}
public abstract List<DBpediaResource> extract(Text text) throws AnnotationException;
public void evaluate(String Question) throws Exception {
evaluateManual(Question,0);
}
public void evaluateManual(String Question, int restartFrom) throws Exception {
saveExtractedEntitiesSet(Question,new LineParser.ManualDatasetLineParser(), restartFrom);
}
}
main()
public static void main(String[] args) throws Exception {
String Question ="Is the Amazon river longer than the Nile River?";
db c = new db ();
c.configiration(0.0, 0, "non", "CoOccurrenceBasedSelector", "Default", "yes");
System.out.println("resource : "+c.getResu());
}

I just add one little fix for your answer.
Your code is running, if you add the evaluate method call:
public static void main(String[] args) throws Exception {
String question = "Is the Amazon river longer than the Nile River?";
db c = new db ();
c.configiration(0.0, 0, "non", "CoOccurrenceBasedSelector", "Default", "yes");
c.evaluate(question);
System.out.println("resource : "+c.getResu());
}
Lamine

In the request method of the second class (AnnotationClient) in Adel's answer, the author Pablo Mendes hasn't finished
TODO Going to buffer response body of large or unknown size. Using getResponseBodyAsStream instead is recommended.
which is an annoying warning that needs to be removed by replacing
byte[] responseBody = method.getResponseBody(); //TODO Going to buffer response body of large or unknown size. Using getResponseBodyAsStream instead is recommended.
// Deal with the response.
// Use caution: ensure correct character encoding and is not binary data
response = new String(responseBody);
with
Reader in = new InputStreamReader(method.getResponseBodyAsStream(), "UTF-8");
StringWriter writer = new StringWriter();
org.apache.commons.io.IOUtils.copy(in, writer);
response = writer.toString();