Develop Reference

How can I make a POST request in the same manner as Apache HttpClient lib, using HttpURLConnection

I'm trying to make a POST request to a website. As the response to the POST request, I expect some JSON data.
Using Apache's HttpClient library, I am able to do this without any problems. The response data is JSON so I just parse it.
package com.mydomain.myapp;
import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
public class MyApp {
private static String extract(String patternString, String target) {
Pattern pattern = Pattern.compile(patternString);
Matcher matcher = pattern.matcher(target);
matcher.find();
return matcher.group(1);
}
private String getResponse(InputStream stream) throws Exception {
BufferedReader in = new BufferedReader(new InputStreamReader(stream));
String inputLine;
StringBuffer responseStringBuffer = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
responseStringBuffer.append(inputLine);
}
in.close();
return responseStringBuffer.toString();
}
private final static String BASE_URL = "https://www.volkswagen-car-net.com";
private final static String BASE_GUEST_URL = "/portal/en_GB/web/guest/home";
private void run() throws Exception {
CloseableHttpClient client = HttpClients.createDefault();
HttpGet httpGet = new HttpGet(BASE_URL + BASE_GUEST_URL);
CloseableHttpResponse getResponse = client.execute(httpGet);
HttpEntity responseEntity = getResponse.getEntity();
String data = getResponse(responseEntity.getContent());
EntityUtils.consume(responseEntity);
String csrf = extract("<meta name=\"_csrf\" content=\"(.*)\"/>", data);
System.out.println(csrf);
HttpPost post = new HttpPost(BASE_URL + "/portal/web/guest/home/-/csrftokenhandling/get-login-url");
post.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
post.setHeader("User-Agent'", "Mozilla/5.0 (Linux; Android 6.0.1; D5803 Build/23.5.A.1.291; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/63.0.3239.111 Mobile Safari/537.36");
post.setHeader("Referer", BASE_URL + "/portal");
post.setHeader("X-CSRF-Token", csrf);
CloseableHttpResponse postResponse = client.execute(post);
HttpEntity postResponseEntity = postResponse.getEntity();
String postData = getResponse(postResponseEntity.getContent());
System.out.println(postData);
EntityUtils.consume(postResponseEntity);
postResponse.close();
}
public static void main(String[] args) throws Exception {
MyApp myApp = new MyApp();
myApp.run();
}
}
But I can't use the HttpClient library in my project. I need to be able to do the same thing with "just" HttpURLConnection.
But there is some magic going on with the HttpClient library that I cannot fathom. Because the response to my POST request using HttpURLConnection is just a redirect to a different webpage alltogheter.
Can someone point me in the right direction here?
Here's my current HttpURLConnection attempt:
package com.mydomain.myapp;
import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class MyApp {
private static String extract(String patternString, String target) {
Pattern pattern = Pattern.compile(patternString);
Matcher matcher = pattern.matcher(target);
matcher.find();
return matcher.group(1);
}
private final static String BASE_URL = "https://www.volkswagen-car-net.com";
private final static String BASE_GUEST_URL = "/portal/en_GB/web/guest/home";
private String getResponse(InputStream stream) throws Exception {
BufferedReader in = new BufferedReader(new InputStreamReader(stream));
String inputLine;
StringBuffer responseStringBuffer = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
responseStringBuffer.append(inputLine);
}
in.close();
return responseStringBuffer.toString();
}
private String getResponse(HttpURLConnection connection) throws Exception {
return getResponse(connection.getInputStream());
}
private void run() throws Exception {
HttpURLConnection getConnection1;
URL url = new URL(BASE_URL + BASE_GUEST_URL);
getConnection1 = (HttpURLConnection) url.openConnection();
getConnection1.setRequestMethod("GET");
if (getConnection1.getResponseCode() != HttpURLConnection.HTTP_OK) {
throw new Exception("Request failed");
}
String response = getResponse(getConnection1);
getConnection1.disconnect();
String csrf = extract("<meta name=\"_csrf\" content=\"(.*)\"/>", response);
System.out.println(csrf);
HttpURLConnection postRequest;
URL url2 = new URL(BASE_URL + "/portal/web/guest/home/-/csrftokenhandling/get-login-url");
postRequest = (HttpURLConnection) url2.openConnection();
postRequest.setDoOutput(true);
postRequest.setRequestMethod("POST");
postRequest.setInstanceFollowRedirects(false);
postRequest.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
postRequest.setRequestProperty("User-Agent'", "Mozilla/5.0 (Linux; Android 6.0.1; D5803 Build/23.5.A.1.291; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/63.0.3239.111 Mobile Safari/537.36");
postRequest.setRequestProperty("Referer", BASE_URL + "/portal");
postRequest.setRequestProperty("X-CSRF-Token", csrf);
postRequest.disconnect();
}
public static void main(String[] args) throws Exception {
MyApp myApp = new MyApp();
myApp.run();
}
}

Courtesy of a great programmer resource, e.g. MKYong (you know you've run into his site before ;-)) and I'll go over the gist of it in case the link ever goes down.
Gist:
The HttpURLConnection‘s follow redirect is just an indicator, in fact it won’t help you to do the “real” http redirection, you still need to handle it manually.
If a server is redirected from the original URL to another URL, the response code should be 301: Moved Permanently or 302: Temporary Redirect. And you can get the new redirected url by reading the “Location” header of the HTTP response header.
For example, access to the normal HTTP twitter website – http://www.twitter.com , it will auto redirect to the HTTPS twitter website – https://www.twitter.com.
Sample code
package com.mkyong.http;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
public class HttpRedirectExample {
public static void main(String[] args) {
try {
String url = "http://www.twitter.com";
URL obj = new URL(url);
HttpURLConnection conn = (HttpURLConnection) obj.openConnection();
conn.setReadTimeout(5000);
conn.addRequestProperty("Accept-Language", "en-US,en;q=0.8");
conn.addRequestProperty("User-Agent", "Mozilla");
conn.addRequestProperty("Referer", "google.com");
System.out.println("Request URL ... " + url);
boolean redirect = false;
// normally, 3xx is redirect
int status = conn.getResponseCode();
if (status != HttpURLConnection.HTTP_OK) {
if (status == HttpURLConnection.HTTP_MOVED_TEMP
|| status == HttpURLConnection.HTTP_MOVED_PERM
|| status == HttpURLConnection.HTTP_SEE_OTHER)
redirect = true;
}
System.out.println("Response Code ... " + status);
if (redirect) {
// get redirect url from "location" header field
String newUrl = conn.getHeaderField("Location");
// get the cookie if need, for login
String cookies = conn.getHeaderField("Set-Cookie");
// open the new connnection again
conn = (HttpURLConnection) new URL(newUrl).openConnection();
conn.setRequestProperty("Cookie", cookies);
conn.addRequestProperty("Accept-Language", "en-US,en;q=0.8");
conn.addRequestProperty("User-Agent", "Mozilla");
conn.addRequestProperty("Referer", "google.com");
System.out.println("Redirect to URL : " + newUrl);
}
BufferedReader in = new BufferedReader(
new InputStreamReader(conn.getInputStream()));
String inputLine;
StringBuffer html = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
html.append(inputLine);
}
in.close();
System.out.println("URL Content... \n" + html.toString());
System.out.println("Done");
} catch (Exception e) {
e.printStackTrace();
}
}
}

Login - Filling form and submitting

I an new to java and I need help. I am trying to login to a website using java, and everything seems to be fine until now. I get the response back and everything, but It doesn't attempt to log in, which is kind of weird..
The response I get when I run the code is:
Sending 'POST' request to URL : http://mrpropop.com/login
Post parameters : login=admin&password=admin
Response Code : 200
+website html/css code
Here is my code:
package practice;
import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.CookieHandler;
import java.net.CookieManager;
import java.net.HttpURLConnection;
import java.net.URL;
public class login {
private HttpURLConnection conn;
public login() {
// make sure cookies is turn on
CookieHandler.setDefault(new CookieManager());
}
public String sendPost(String url, String params) throws Exception {
URL obj = new URL(url);
conn = (HttpURLConnection) obj.openConnection();
// Acts like a browser
conn.setUseCaches(false);
conn.setRequestMethod("POST");
conn.setRequestProperty("Host", "mrpropop.com");
conn.setRequestProperty("User-Agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36");
conn.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
conn.setRequestProperty("Accept-Language", "en-US,en;q=0.5");
conn.setRequestProperty("Connection", "keep-alive");
conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
conn.setRequestProperty("Content-Length", Integer.toString(params.length()));
conn.setDoOutput(true);
conn.setDoInput(true);
// Send post request
DataOutputStream wr = new DataOutputStream(conn.getOutputStream());
wr.writeBytes(params);
wr.flush();
wr.close();
int responseCode = conn.getResponseCode();
System.out.println("\nSending 'POST' request to URL : " + url);
System.out.println("Post parameters : " + params);
System.out.println("Response Code : " + responseCode);
return getResponse();
}
private String getResponse() throws IOException {
BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream()));
String line = br.readLine();
StringBuilder sb = new StringBuilder();
while (line != null) {
sb.append(line);
line = br.readLine();
}
return sb.toString();
}
public static void main(String[] argv) throws Exception {
login login = new login();
// 1. login first
String loginUrl = "http://mrpropop.com/login";
String loginParams = "login=admin&password=admin";
login.sendPost(loginUrl, loginParams);
// Post request
String apiUrl = loginUrl;
String apiParams = loginParams;
System.out.println(login.sendPost(apiUrl, apiParams));
}
}
What is wrong here? Thanks in advance!

I suppose you want to do something when you get a code 200.
For example setting a cookie.

The response code of 200 suggests you have successfully logged in. You can display the contents by querying the HTTP body from the response object.

Unable to get response from an url which requires user acceptance before it redirects to target url

I had a job which has to download a file from "https://www.frbservices.org/EPaymentsDirectory/FedACHdir.txt" and place in the folder. Currently we are doing it manually as job runs for once in 15 days but i want to download the file using the program before job process the file.
The problem here is, when we hit the url (https://www.frbservices.org/EPaymentsDirectory/FedACHdir.txt) on the browser, on first time it will be redirected to "https://www.frbservices.org/EPaymentsDirectory/agreement.html", on clicking "Agree" button then it will redirects to "https://www.frbservices.org/EPaymentsDirectory/FedACHdir.txt". On clicking Agree button JSESSIONID is saved as a cookie in the browser. when we click the target url next time it directly opens the required page with out any agree of terms..
I tried below thing to get the response from target url but not able to achieve the response as expected..
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.CookieHandler;
import java.net.CookieManager;
import java.util.ArrayList;
import java.util.List;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.message.BasicNameValuePair;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
public class HttpTest {
static String cookies;
public static void main(String [] args) throws ClientProtocolException, IOException {
String url = "https://www.frbservices.org/EPaymentsDirectory/FedACHdir.txt";
String agreementURL = "https://www.frbservices.org/EPaymentsDirectory/submitAgreement";
String USER_AGENT = "Mozilla/5.0";
CookieHandler.setDefault(new CookieManager());
HttpClient client = HttpClientBuilder.create().build();
String result;
result =doPost(agreementURL, USER_AGENT, client);
System.out.println(result);
result = doGet(url, USER_AGENT, client);
System.out.println("result:"+result);
//String result =doGet(url, USER_AGENT, client);
if (result != null) {
Document doc = Jsoup.parse(result.toString());
//
Element agreeElement = doc.getElementById("agree_terms_use");
}
}
public static String getCookies() {
return cookies;
}
public static void setCookies(String cookies) {
HttpTest.cookies = cookies;
}
public static String doGet(String url , String USER_AGENT, HttpClient client) throws ClientProtocolException, IOException {
HttpGet request = new HttpGet(url);
request.setHeader("User-Agent", USER_AGENT);
request.setHeader("Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
request.setHeader("Accept-Language", "en-US,en;q=0.5");
System.out.println("c"+getCookies());
request.setHeader("Cookie", getCookies());
HttpResponse response = client.execute(request);
int responseCode = response.getStatusLine().getStatusCode();
System.out.println("\nSending 'GET' request to URL : " + url);
System.out.println("Response Code : " + responseCode);
System.out.println(response);
BufferedReader rd = new BufferedReader(
new InputStreamReader(response.getEntity().getContent()));
StringBuffer result = new StringBuffer();
String line = "";
while ((line = rd.readLine()) != null) {
result.append(line);
}
return result.toString();
}
public static String doPost(String url , String USER_AGENT, HttpClient client) throws ClientProtocolException, IOException {
HttpPost post = new HttpPost(url);
post.setHeader("Host", "www.frbservices.org");
post.setHeader("User-Agent", USER_AGENT);
post.setHeader("Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
post.setHeader("Accept-Encoding","gzip, deflate, br");
post.setHeader("Accept-Language", "en-US,en;q=0.5");
post.setHeader("Location", "https://www.frbservices.org/EPaymentsDirectory/FedACHdir.txt");
post.setHeader("Connection", "keep-alive");
post.setHeader("Referer", "https://www.frbservices.org/EPaymentsDirectory/agreement.html");
post.setHeader("Content-Type", "application/x-www-form-urlencoded");
post.setHeader("origin", "https://www.frbservices.org");
post.setHeader("Upgrade-Insecure-Requests","1");
//post.setHeader("agreementValue","Agree");
List<BasicNameValuePair> paramList = new ArrayList<BasicNameValuePair>();
paramList.add(new BasicNameValuePair("agreementValue", "Agree"));
post.setEntity(new UrlEncodedFormEntity(paramList));
HttpResponse response = client.execute(post);
int responseCode = response.getStatusLine().getStatusCode();
System.out.println("\nSending 'GET' request to URL : " + url);
System.out.println("Response Code : " + responseCode);
System.out.println("Response: " + response);
BufferedReader rd = new BufferedReader(
new InputStreamReader(response.getEntity().getContent()));
setCookies(response.getFirstHeader("Set-Cookie") == null ? "" :
response.getFirstHeader("Set-Cookie").toString());
StringBuffer result = new StringBuffer();
String line = "";
while ((line = rd.readLine()) != null) {
result.append(line);
}
return result.toString();
}
}

no response from HttpGet client.execute(request) after initial login was successful

I'm trying to log into this site and collect some data.
I removed the username and password from the code i shared below, but when it takes the first url and password and runs i get a response showing me that the login was successful, however after saving the cookies and trying to run the second link with the query for the site it just hangs. no errors and no response. I played for hours but just cant get it. Its getting stuck at the client.execute(request) thats running with this link (http :// sef.imapp. com/ilinks/property?upin=US120860131120280120&report=comps&distance=0.5)
I copied the code from http://www.mkyong.com/java/apache-httpclient-examples/ (Part 3) and made changes until i got the response html page showing logged in, but I just cant run the query url.
I appreciate any help to be explained in detail.
Thank you.
package connectors;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.CookieHandler;
import java.net.CookieManager;
import java.util.ArrayList;
import java.util.List;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.message.BasicNameValuePair;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
public class HttpCilentExampleToMls {
private String cookies;
private HttpClient client = HttpClientBuilder.create().build();
private final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.36 Safari/537.36";
public static void main(String[] args) throws Exception {
//Login page Changed from google address
String url = "http://sef.imapp.com/ilinks/search";
//Search page Changed from gmail address
String mlsUrl = "http://sef.imapp.com/ilinks/property?upin=US120860131120280120&report=comps&distance=0.5";
// make sure cookies is turn on
CookieHandler.setDefault(new CookieManager());
HttpCilentExampleToMls http = new HttpCilentExampleToMls();
String page = http.GetPageContent(url);
List<NameValuePair> postParams =
// Changed from "Username" and "Password"
http.getFormParams(page, "Username","Password");
http.sendPost(url, postParams);
String result = http.GetPageContent(mlsUrl);
}
private void sendPost(String url, List<NameValuePair> postParams)
throws Exception {
HttpPost post = new HttpPost(url);
// add header
post.setHeader("Host", "sef.imapp.com");
post.setHeader("User-Agent", USER_AGENT);
post.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
post.setHeader("Accept-Language", "en-US,en;q=0.8");
post.setHeader("Cookie", getCookies());
post.setHeader("Connection", "Keep-Alive");
post.setHeader("Referer", "http://sef.imapp.com/ilinks/login?logout=true");
post.setHeader("Content-Type", "application/x-www-form-urlencoded");
post.setEntity(new UrlEncodedFormEntity(postParams));
HttpResponse response = client.execute(post);
int responseCode = response.getStatusLine().getStatusCode();
System.out.println("\nSending 'POST' request to URL : " + url);
System.out.println("Post parameters : " + postParams);
System.out.println("Response Code : " + responseCode);
BufferedReader rd = new BufferedReader(
new InputStreamReader(response.getEntity().getContent()));
StringBuffer result = new StringBuffer();
String line = "";
while ((line = rd.readLine()) != null) {
result.append(line);
System.err.println(line);
}
// System.out.println(result.toString());
}
private String GetPageContent(String url) throws Exception {
HttpGet request = new HttpGet(url);
request.setHeader("User-Agent", USER_AGENT);
request.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
request.setHeader("Accept-Language", "en-US,en;q=0.8");
HttpResponse response = client.execute(request);
int responseCode = response.getStatusLine().getStatusCode();
System.out.println("\nSending 'GET' request to URL : " + url);
System.out.println("Response Code : " + responseCode);
BufferedReader rd = new BufferedReader(
new InputStreamReader(response.getEntity().getContent()));
StringBuffer result = new StringBuffer();
String line = "";
while ((line = rd.readLine()) != null) {
result.append(line);
}
// set cookies
setCookies(response.getFirstHeader("Set-Cookie") == null ? "" :
response.getFirstHeader("Set-Cookie").toString());
return result.toString();
}
public List<NameValuePair> getFormParams(
String html, String username, String password)
throws UnsupportedEncodingException {
System.out.println("Extracting form's data...");
Document doc = Jsoup.parse(html);
// Google form id
Element loginform = doc.getElementById("standardLogin");
Elements inputElements = loginform.getElementsByTag("input");
List<NameValuePair> paramList = new ArrayList<NameValuePair>();
for (Element inputElement : inputElements) {
String key = inputElement.attr("name");
String value = inputElement.attr("value");
if (key.equals("user"))
value = username;
else if (key.equals("passwd"))
value = password;
paramList.add(new BasicNameValuePair(key, value));
}
return paramList;
}
public String getCookies() {
return cookies;
}
public void setCookies(String cookies) {
this.cookies = cookies;
}
}

I had the same problem trying exactly the code at mkyong and a comment from the website solved it: after using HttpGet or HttpPost objects, use the method releaseConnection() in them.
Alex Loginov • 8 months ago Hi, might be helpful for someone: after
you receive the response, please close HttpPost (HttpGet) :
post.releaseConnection(); Else after 2 times you will have no free
connections and will have to wait infinitely for them to release - no
exception will be thrown.

I'm getting the same response html which I used to post parameters in httpclient

I'm getting the same response HTML when I tried to post username and password parameters to login page. I have followed the tutorial that is present here.
http://www.mkyong.com/java/apache-httpclient-examples/
I'm able to successfully do this for gmail page but I'm unable to do this for the IBM Partner World Page. Here is my java program that I used for page login
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.CookieHandler;
import java.net.CookieManager;
import java.util.ArrayList;
import java.util.List;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.message.BasicNameValuePair;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
public class tt {
private String cookies;
private HttpClient client = HttpClientBuilder.create().build();
private final String USER_AGENT = "Mozilla/5.0";
public static void main(String[] args) throws Exception {
String url = "https://www-304.ibm.com/usrsrvc/account/userservices/jsp/login.jsp?persistPage=true&page=/partnerworld/wps/servlet/mem/ContentHandler/partnerworld-public%3Flnk%3Dleft-nav&PD-REFERER=https://www-304.ibm.com/partnerworld/wps/servlet/ContentHandler/partnerworld-public&error=";
String gmail = "https://www.ibm.com/partnerworld/page/X082763O18037G72";
// make sure cookies is turn on
CookieHandler.setDefault(new CookieManager());
tt http = new tt();
String page = http.GetPageContent(url);
List<NameValuePair> postParams =
http.getFormParams(page, "username","password");
http.sendPost(url, postParams);
String result = http.GetPageContent(gmail);
System.out.println(result);
System.out.println("Done");
}
private void sendPost(String url, List<NameValuePair> postParams)
throws Exception {
HttpPost post = new HttpPost(url);
// add header
post.setHeader("Host", "www-304.ibm.com:443");
post.setHeader("User-Agent", USER_AGENT);
post.setHeader("Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
post.setHeader("Accept-Language", "en-US,en;q=0.5");
post.setHeader("Cookie", getCookies());
post.setHeader("Accept-Encoding" ,"gzip,deflate,sdch");
//post.setHeader("Connection", "keep-alive");
post.setHeader("Referer", "https://www-304.ibm.com/partnerworld/wps/servlet/ContentHandler/pw_com_jnw_index");
//post.setHeader("Content-Type", "application/x-www-form-urlencoded");
post.setEntity(new UrlEncodedFormEntity(postParams));
HttpResponse response = client.execute(post);
int responseCode = response.getStatusLine().getStatusCode();
System.out.println("\nSending 'POST' request to URL : " + url);
System.out.println("Post parameters : " + postParams);
System.out.println("Response Code : " + responseCode);
BufferedReader rd = new BufferedReader(
new InputStreamReader(response.getEntity().getContent()));
StringBuffer result = new StringBuffer();
String line = "";
while ((line = rd.readLine()) != null) {
result.append(line);
}
System.out.println(result.toString());
}
private String GetPageContent(String url) throws Exception {
HttpGet request = new HttpGet(url);
request.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.114 Safari/537.36");
request.setHeader("Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
request.setHeader("Accept-Language", "en-US,en;q=0.5");
HttpResponse response = client.execute(request);
int responseCode = response.getStatusLine().getStatusCode();
System.out.println("\nSending 'GET' request to URL : " + url);
System.out.println("Response Code : " + responseCode);
BufferedReader rd = new BufferedReader(
new InputStreamReader(response.getEntity().getContent()));
StringBuffer result = new StringBuffer();
String line = "";
while ((line = rd.readLine()) != null) {
result.append(line);
}
// set cookies
setCookies(response.getFirstHeader("Set-Cookie") == null ? "" :
response.getFirstHeader("Set-Cookie").toString());
return result.toString();
}
public List<NameValuePair> getFormParams(
String html, String username, String password)
throws UnsupportedEncodingException {
System.out.println("Extracting form's data...");
Document doc = Jsoup.parse(html);
// Google form id
Element loginform = doc.getElementById("login");
Elements inputElements = loginform.getElementsByTag("input");
List<NameValuePair> paramList = new ArrayList<NameValuePair>();
for (Element inputElement : inputElements) {
String key = inputElement.attr("name");
String value = inputElement.attr("value");
if (key.equals("username"))
value = username;
else if (key.equals("password"))
value = password;
System.out.println(username+password);
paramList.add(new BasicNameValuePair(key, value));
}
return paramList;
}
public String getCookies() {
return cookies;
}
public void setCookies(String cookies) {
this.cookies = cookies;
}
}
I get the same HTML as response which is used to post parameters. Please help me with this.
I want to know How to login to this website and maintain my session.

I got it. I used chrome browser's developer tool and checked the different urls the browser is trying to get me through GET and POST requests.
Goto Dev Tools> Network > and try accessing the web page normally> You'll see all the information even the cookies and content that is returned from the site.
The forms varies from page to page that you want to mimic. You should check the source of the login page (and different forms in it) that you want to mimic. Use the proper form where the username and password is being sent.
Also when you monitor the webpage requests using dev tools. check all entities. I got my problems fixed by
1. Checking the encoded form of password. UrlEncodedFormEntity is different what my browser is using.
return "login-form-type=pwd&username="+username+"&password="+password;
2. And the exact form url which is actually responsible for sending the parameters.And also the next page after login parameters are sent.
String url = "https://www-304.ibm.com/pkmslogin.form";
String gmail = "https://www.ibm.com/partnerworld/page/X082763O18037G72";
Also make sure that you store the cookies and send them back for the next request. Because this is how you maintain a session.