I have a website which I'm trying to log-on to and fill in forms automatically and then finally sending the data. The first step is achieved with the code below i.e. I can log in and then redirect to the page I'd like to fill with information.
The code gets the form name and value and changes these using GET and POST and I'd like to repeat this process but this time with multiple forms (instead of just username and password). I manage to get the form name and values and change the HTML but the issues occurs when I try to POST the new values back to the server. I don't get any error messages and the code returns the new page but the form data remains unchanged.
I can't seem to find the issue in my code, the second process works just like the first process, the only difference is, I now try to POSTdata to multiple forms.
The problem, I assume, lies either in my fillFormParams or sendPost methods. I appreciate any help!
import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.CookieHandler;
import java.net.CookieManager;
import java.net.URL;
import java.net.URLConnection;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import javax.net.ssl.HttpsURLConnection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.net.HttpURLConnection;
public class HttpUrlConnectionExample {
private List<String> cookies;
private List<String> cookiejar;
private static HttpURLConnection conn;
private final String USER_AGENT = "Mozilla/5.0";
public static void main(String[] args) throws Exception {
String url = "http://www3.bagerisystem.se/webbordersodervidingebagarn/Logon.aspx";
String menuPage = "http://www3.bagerisystem.se/webbordersodervidingebagarn/Meny.aspx";
HttpUrlConnectionExample http = new HttpUrlConnectionExample();
// make sure cookies is turn on
http.createCookieManager();
// 1. Send a "GET" request, so that you can extract the form's data.
String page = http.GetPageContent(url);
String postParams = http.getFormParams(page, "****", "****");
// 2. Construct above post's content and then send a POST request for
// authentication
URL obj = new URL(url);
conn = (HttpURLConnection) obj.openConnection();
http.sendPost(url, postParams,conn);
// 3. success then go to menuPage.
String result = http.GetPageContent(menuPage);
//System.out.println(result);
// 4.Send new GET request
String page2 = http.GetPageContent("http://www3.bagerisystem.se/webbordersodervidingebagarn/Retur.aspx");
//System.out.println(page2);
int[][] returVal = { { 0, 1, 2, 3, 4, 5, 6, },{ 0, 1, 2, 3, 4, 5, 6, },{ 0, 1, 2, 3, 4, 5, 6, },{ 0, 1, 2, 3, 4, 5, 6, },{ 0, 1, 2, 3, 4, 5, 6, },{ 0, 1, 2, 3, 4, 5, 6, }};
String postParams2 = http.fillFormParams(page2, returVal);
// 5. Construct above post's content and then send a POST request for
// authentication
String url1 = "http://www3.bagerisystem.se/webbordersodervidingebagarn/Retur.aspx";
URL obj1 = new URL(url1);
conn = (HttpURLConnection) obj1.openConnection();
http.sendPost(url1, postParams2,conn);
// 6. success then go to menuPage.
String resultz = http.GetPageContent("http://www3.bagerisystem.se/webbordersodervidingebagarn/Retur.aspx");
System.out.println(resultz);
}
private void sendPost(String url, String postParams,HttpURLConnection conn) throws Exception {
// Acts like a browser
conn.setUseCaches(false);
conn.setRequestMethod("POST");
conn.setRequestProperty("Host", "www3.bagerisystem.se");
conn.setRequestProperty("User-Agent", USER_AGENT);
conn.setRequestProperty("Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
conn.setRequestProperty("Accept-Language", "en-US,en;q=0.5");
for (String cookie : this.cookies) {
conn.addRequestProperty("Cookie", cookie.split(";", 1)[0]);
}
conn.setRequestProperty("Connection", "keep-alive");
conn.setRequestProperty("Referer", "http://www3.bagerisystem.se/webbordersodervidingebagarn/Logon.aspx");
conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
conn.setRequestProperty("Content-Length", Integer.toString(postParams.length()));
conn.setDoOutput(true);
conn.setDoInput(true);
// Send post request
DataOutputStream wr = new DataOutputStream(conn.getOutputStream());
wr.writeBytes(postParams);
wr.flush();
wr.close();
int responseCode = conn.getResponseCode();
System.out.println("\nSending 'POST' request to URL : " + url);
System.out.println("Post parameters : " + postParams);
System.out.println("Response Code : " + responseCode);
BufferedReader in =
new BufferedReader(new InputStreamReader(conn.getInputStream()));
String inputLine;
StringBuffer response = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
// System.out.println(response.toString());
}
private String GetPageContent(String url) throws Exception {
URL obj = new URL(url);
conn = (HttpURLConnection) obj.openConnection();
// default is GET
conn.setRequestMethod("GET");
conn.setUseCaches(false);
// act like a browser
conn.setRequestPropert`enter code here`y("User-Agent", USER_AGENT);
conn.setRequestProperty("Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
conn.setRequestProperty("Accept-Language", "en-US,en;q=0.5");
if (cookies != null) {
for (String cookie : this.cookies) {
conn.addRequestProperty("Cookie", cookie.split(";", 1)[0]);
}
}
int responseCode = conn.getResponseCode();
// System.out.println("\nSending 'GET' request to URL : " + url);
// System.out.println("Response Code : " + responseCode);
BufferedReader in =
new BufferedReader(new InputStreamReader(conn.getInputStream()));
String inputLine;
StringBuffer response = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
// Get the response cookies
setCookies(conn.getHeaderFields().get("Set-Cookie"));
return response.toString();
}
public String getFormParams(String html, String username, String password)
throws UnsupportedEncodingException {
//System.out.println("Extracting form's data...");
Document doc = Jsoup.parse(html);
// form id
Element loginform = doc.getElementById("Form1");
Elements inputElements = loginform.getElementsByTag("input");
List<String> paramList = new ArrayList<String>();
for (Element inputElement : inputElements) {
String key = inputElement.attr("name");
String value = inputElement.attr("value");
if (key.equals("UserNameTextBox"))//If corresponding input form is found, enter defined username
value = username;
else if (key.equals("PasswordTextBox"))//If corresponding input form is found, enter defined password
value = password;
paramList.add(key + "=" + URLEncoder.encode(value, "UTF-8"));
}
// build parameters list
StringBuilder result = new StringBuilder();
for (String param : paramList) {
if (result.length() == 0) {
result.append(param);
} else {
result.append("&" + param);
}
}
return result.toString();
}
public List<String> getCookies() {
return cookies;
}
public void setCookies(List<String> cookies) {
this.cookies = cookies;
}
public void createCookieManager(){
if (cookies == null) {
CookieHandler.setDefault(new CookieManager());
}
}
public String fillFormParams(String html, int[][] returVal) throws UnsupportedEncodingException{
Document doc = Jsoup.parse(html);
//Form id
int row = 0;
int column = 0;
StringBuilder result = new StringBuilder();
Element inputForm = doc.getElementById("Form1");
Elements inputElements = inputForm.getElementsByTag("input");
List<String> paramList = new ArrayList<String>();
while(row <5 ){
for (Element inputElement: inputElements){
String key = inputElement.attr("name");
String value = inputElement.attr("value");
String truId = "ReturDataGrid:_ctl" +(row+2);
// "ReturDataGrid%3A_ctl2%3AtxtAntal_1" + "=" + URLEncoder.encode("1", "UTF-8")
String newId = truId + ":txtAntal_" + Integer.toString(column);
if (column == 6) {
row++;
column = (column+1)%7;
}
if(key.equals(newId)){
key = "ReturDataGrid%3A_ctl"+(row+2) +"%3AtxtAntal_1" +column;
value = Integer.toString(returVal[row][column]);
paramList.add(key + "=" + URLEncoder.encode(value, "UTF-8"));
}
}
column++;
}
for (String param : paramList) {
if (result.length() == 0) {
result.append(param);
} else {
result.append("&" + param);
}
}
return result.toString();
}
}
Related
i tried to login gmail via sending post request flowwing the code:
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package testautologingmail;
import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.CookieHandler;
import java.net.CookieManager;
import java.net.HttpCookie;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import javax.net.ssl.HttpsURLConnection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
public class HttpUrlConnectionExample {
private static List<String> cookies = new ArrayList<>();;
private static HttpsURLConnection conn;
private final String USER_AGENT = "Mozilla/5.0";
public HttpUrlConnectionExample(){
}
public static void main(String[] args) throws Exception {
String url = "https://accounts.google.com/ServiceLoginAuth";
String gmail = "https://mail.google.com/mail/";
HttpUrlConnectionExample http = new HttpUrlConnectionExample();
// make sure cookies is turn on
//CookieHandler.setDefault(new CookieManager());
// 1. Send a "GET" request, so that you can extract the form's data.
String page = http.GetPageContent(url);
String postParams = http.getFormParams(page,"kiemtienol1506","vandai1507");
// 2. Construct above post's content and then send a POST request for
// authentication
http.sendPost(url, postParams);
// 3. success then go to gmail.
String result = http.GetPageContent(gmail);
System.out.println(result);
conn.disconnect();
}
private static void getcookies(String url) throws MalformedURLException, IOException{
CookieManager cookieManager = new CookieManager();
CookieHandler.setDefault(cookieManager);
URL obj = new URL(url);
conn = (HttpsURLConnection) obj.openConnection();
conn.getContent();
List<HttpCookie> cookies2 = cookieManager.getCookieStore().getCookies();
System.out.println("get cookies");
for (HttpCookie cookie : cookies2) {
System.out.println(cookie.getDomain());
System.out.println(cookie.toString());
String temp = cookie.toString();
cookies.add(temp);
}
//conn.disconnect();
}
private void sendPost(String url, String postParams) throws Exception {
URL obj = new URL(url);
conn = (HttpsURLConnection) obj.openConnection();
// Acts like a browser
conn.setUseCaches(false);
conn.setRequestMethod("POST");
conn.setRequestProperty("Host", "accounts.google.com");
conn.setRequestProperty("User-Agent", USER_AGENT);
conn.setRequestProperty("Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
conn.setRequestProperty("Accept-Language", "en-US,en;q=0.5");
System.out.println("cookies String");
for (String cookie : cookies) {
//String[] tmp = cookie.split(":");
conn.addRequestProperty("Cookie", cookie.split(";", 1)[0]);
System.out.println(cookie.split(";", 1)[0]);
}
conn.setRequestProperty("Connection", "keep-alive");
conn.setRequestProperty("Referer", "https://accounts.google.com/ServiceLoginAuth");
conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
conn.setRequestProperty("Content-Length", Integer.toString(postParams.length()));
conn.setDoOutput(true);
conn.setDoInput(true);
// Send post request
DataOutputStream wr = new DataOutputStream(conn.getOutputStream());
wr.writeBytes(postParams);
wr.flush();
wr.close();
int responseCode = conn.getResponseCode();
System.out.println("\nSending 'POST' request to URL : " + url);
System.out.println("Post parameters : " + postParams);
System.out.println("Response Code : " + responseCode);
BufferedReader in =
new BufferedReader(new InputStreamReader(conn.getInputStream()));
String inputLine;
StringBuffer response = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
//System.out.println(response.toString());
//conn.disconnect();
}
private String GetPageContent(String url) throws Exception {
URL obj = new URL(url);
conn = (HttpsURLConnection) obj.openConnection();
// default is GET
conn.setRequestMethod("GET");
conn.setUseCaches(false);
// act like a browser
conn.setRequestProperty("User-Agent", USER_AGENT);
conn.setRequestProperty("Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
conn.setRequestProperty("Accept-Language", "en-US,en;q=0.5");
if (cookies != null) {
for (String cookie : cookies) {
conn.addRequestProperty("Cookie", cookie.split(";", 1)[0]);
}
}
int responseCode = conn.getResponseCode();
System.out.println("\nSending 'GET' request to URL : " + url);
System.out.println("Response Code : " + responseCode);
BufferedReader in =
new BufferedReader(new InputStreamReader(conn.getInputStream()));
String inputLine;
StringBuffer response = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
// Get the response cookies
//setCookies(conn.getHeaderFields().get("Set-Cookie"));
getcookies(url);
return response.toString();
}
public String getFormParams(String html, String username, String password)
throws UnsupportedEncodingException {
System.out.println("Extracting form's data...");
Document doc = Jsoup.parse(html);
// Google form id
Element loginform = doc.getElementById("gaia_loginform");
Elements inputElements = loginform.getElementsByTag("input");
List<String> paramList = new ArrayList<String>();
for (Element inputElement : inputElements) {
String key = inputElement.attr("name");
String value = inputElement.attr("value");
String id = inputElement.attr("id");
if (key.equals("Email"))
{
value = username;
paramList.add(key + "=" + URLEncoder.encode(value, "UTF-8"));
}
else if (key.equals("Passwd")){
paramList.add(key + "=" + URLEncoder.encode(value, "UTF-8"));
value = password;
}
else if(id.equals("Passwd-hidden")){
value = password;
paramList.add("Passwd=" + URLEncoder.encode(value, "UTF-8"));
}else
paramList.add(key + "=" + URLEncoder.encode(value, "UTF-8"));
}
// build parameters list
StringBuilder result = new StringBuilder();
for (String param : paramList) {
if (result.length() == 0) {
result.append(param);
} else {
result.append("&" + param);
}
}
return result.toString();
}
public List<String> getCookies() {
return cookies;
}
public void setCookies(List<String> cookies) {
this.cookies = cookies;
}
}
i also try to get cookies flow this line setCookies(conn.getHeaderFields().get("Set-Cookie")); but it's always return null String so i have to make getCookies() function.
i always receive status 200 OK. So is there any support to help me or another code can do this task.
Thanks in Advance.
I'm trying ton connect to a site, according to this page : http://www.mkyong.com/java/how-to-automate-login-a-website-java-example/
But it seems, that the cookies are not set.
Cannot find the problem...
My aim is to authenticate to this sit in order to grab the Html.
If there is another, more simple, solution, don't hesitate.
package htmlmKyongTest;
import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.CookieHandler;
import java.net.CookieManager;
import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import javax.net.ssl.HttpsURLConnection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
public class HttpUrlConnectionExample {
private List<String> cookies;
private HttpsURLConnection conn;
private final String USER_AGENT = "Mozilla/5.0";
public static void main(String[] args) throws Exception {
String url = "https://www.entea.fr/";
String gmail = "https://www.entea.fr/etabs/0671688W/Pages/Accueil.aspx";
HttpUrlConnectionExample http = new HttpUrlConnectionExample();
// make sure cookies is turn on
CookieHandler.setDefault(new CookieManager());
// 1. Send a "GET" request, so that you can extract the form's data.
String page = http.GetPageContent(url);
String postParams = http.getFormParams(page, "myUsername", "myPassword");
// 2. Construct above post's content and then send a POST request for
// authentication
http.sendPost(url, postParams);
// 3. success
String result = http.GetPageContent(gmail);
System.out.println(result);
}
private void sendPost(String url, String postParams) throws Exception {
URL obj = new URL(url);
conn = (HttpsURLConnection) obj.openConnection();
// Acts like a browser
conn.setUseCaches(false);
conn.setRequestMethod("POST");
conn.setRequestProperty("Host", "www.entea.fr");
conn.setRequestProperty("User-Agent", USER_AGENT);
conn.setRequestProperty("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
conn.setRequestProperty("Accept-Language", "fr-FR,fr;q=0.8,en-US;q=0.6,en;q=0.4");
conn.setRequestProperty("Content-Type","application/x-www-form-urlencoded");
for (String cookie : this.cookies) {
conn.addRequestProperty("Set-Cookie", cookie.split(";", 1)[0]);
}
conn.setRequestProperty("Connection", "keep-alive");
conn.setRequestProperty("Referer", "https://www.entea.fr/CookieAuth.dll?GetLogon?curl=Z2F&reason=0&formdir=7");
conn.setRequestProperty("Accept-Encoding","gzip,deflate");
conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
conn.setRequestProperty("Content-Length", Integer.toString(postParams.length()));
conn.setDoOutput(true);
conn.setDoInput(true);
// Send post request
DataOutputStream wr = new DataOutputStream(conn.getOutputStream());
wr.writeBytes(postParams);
wr.flush();
wr.close();
int responseCode = conn.getResponseCode();
System.out.println("\nSending 'POST' request to URL : " + url);
System.out.println("Post parameters : " + postParams);
System.out.println("Response Code : " + responseCode);
BufferedReader in =
new BufferedReader(new InputStreamReader(conn.getInputStream()));
String inputLine;
StringBuffer response = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
}
private String GetPageContent(String url) throws Exception {
URL obj = new URL(url);
conn = (HttpsURLConnection) obj.openConnection();
// default is GET
conn.setRequestMethod("GET");
conn.setUseCaches(false);
// act like a browser
conn.setRequestProperty("User-Agent", USER_AGENT);
conn.setRequestProperty("Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
conn.setRequestProperty("Accept-Language", "en-US,en;q=0.5");
if (cookies != null) {
for (String cookie : this.cookies) {
conn.addRequestProperty("Set-Cookie", cookie.split(";", 1)[0]);
}
}
int responseCode = conn.getResponseCode();
System.out.println("\nSending 'GET' request to URL : " + url);
System.out.println("Response Code : " + responseCode);
BufferedReader in =
new BufferedReader(new InputStreamReader(conn.getInputStream()));
String inputLine;
StringBuffer response = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
// Get the response cookies
setCookies(conn.getHeaderFields().get("Set-Cookie"));
return response.toString();
}
public String getFormParams(String html, String username, String password)
throws UnsupportedEncodingException {
System.out.println("Extracting form's data...");
Document doc = Jsoup.parse(html);
Element loginform = doc.getElementById("logonForm");
Elements inputElements = loginform.getElementsByTag("input");
List<String> paramList = new ArrayList<String>();
for (Element inputElement : inputElements) {
System.out.println("inputElement : " + inputElement.toString());
String key = inputElement.attr("name");
String value = inputElement.attr("value");
if (key.equals("username"))
value = username;
else if (key.equals("password")) {
value = password;
}
if (!(inputElement.attr("style").contains("none"))) paramList.add(key + "=" + URLEncoder.encode(value, "UTF-8"));
}
// build parameters list
StringBuilder result = new StringBuilder();
for (String param : paramList) {
if (result.length() == 0) {
result.append(param);
} else {
result.append("&" + param);
}
}
System.out.println("result : " + result.toString().replace("trusted=4", "SubmitCreds.x=0&SubmitCreds.y=0"));
return result.toString().replace("trusted=4", "SubmitCreds.x=0&SubmitCreds.y=0");
}
public List<String> getCookies() {
return cookies;
}
public void setCookies(List<String> cookies) {
this.cookies = cookies;
}
}
the Output is the following :
Sending 'GET' request to URL : https://www.entea.fr/
Response Code : 200
Extracting form's data...
inputElement : <input type="password" id="autocompleteoff" style="display:none" />
inputElement : <input type="hidden" id="curl" name="curl" value="Z2F" />
inputElement : <input type="hidden" id="flags" name="flags" value="0" />
inputElement : <input type="hidden" id="forcedownlevel" name="forcedownlevel" value="0" />
inputElement : <input type="hidden" id="formdir" name="formdir" value="7" />
inputElement : <input type="text" id="username" name="username" />
inputElement : <input id="password" onfocus="g_fFcs=0" type="password" name="password" autocomplete="off" />
inputElement : <input type="radio" name="trusted" id="rdoPblc" value="0" onclick="clkSec()" checked="checked" />
inputElement : <input type="radio" name="trusted" id="rdoPrvt" value="4" onclick="clkSec()" />
inputElement : <input id="SubmitCreds" onclick="clkLgn()" type="image" value="Ouvrir une session" name="SubmitCreds" src="/CookieAuth.dll?GetPic?formdir=7&image=encart_footer_01.png" alt="Connexion" />
result : curl=Z2F&flags=0&forcedownlevel=0&formdir=7&username=xxxxxxxxx&password=xxxxxxxxxxx*&trusted=0&SubmitCreds.x=0&SubmitCreds.y=0&SubmitCreds=Ouvrir+une+session
Exception in thread "main" java.lang.NullPointerException
at htmlmKyongTest.HttpUrlConnectionExample.sendPost(HttpUrlConnectionExample.java:66)
at htmlmKyongTest.HttpUrlConnectionExample.main(HttpUrlConnectionExample.java:46)
The cookies field isn't initialized for the first sendPost. You should protect your for loop with a if (cookies != null).
EDIT : The actual solution is to split the initial request + redirect into 2 separate GET requests.
Try this :
import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.CookieHandler;
import java.net.CookieManager;
import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import javax.net.ssl.HttpsURLConnection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
public class HttpUrlConnectionExample {
private List<String> cookies;
private HttpsURLConnection conn;
private final String USER_AGENT = "Mozilla/5.0";
public static void main(String[] args) throws Exception {
String url = "https://www.entea.fr/";
String gmail = "https://www.entea.fr/etabs/0671688W/Pages/Accueil.aspx";
HttpUrlConnectionExample http = new HttpUrlConnectionExample();
// make sure cookies is turn on
CookieHandler.setDefault(new CookieManager());
// 1. Send a "GET" request, so that you can extract the form's data.
http.getPageContent(url);
String page = http.getPageContent("https://www.entea.fr/CookieAuth.dll?GetLogon?curl=Z2F&reason=0&formdir=7");
String postParams = http.getFormParams(page, "myUsername", "myPassword");
// 2. Construct above post's content and then send a POST request for
// authentication
http.sendPost(url, postParams);
// 3. success
String result = http.getPageContent(gmail);
System.out.println(result);
}
private void sendPost(String url, String postParams) throws Exception {
URL obj = new URL(url);
conn = (HttpsURLConnection) obj.openConnection();
// Acts like a browser
conn.setUseCaches(false);
conn.setRequestMethod("POST");
conn.setRequestProperty("Host", "www.entea.fr");
conn.setRequestProperty("User-Agent", USER_AGENT);
conn.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
conn.setRequestProperty("Accept-Language", "fr-FR,fr;q=0.8,en-US;q=0.6,en;q=0.4");
conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
for (String cookie : this.cookies) {
conn.addRequestProperty("Set-Cookie", cookie.split(";", 1)[0]);
}
conn.setRequestProperty("Connection", "keep-alive");
conn.setRequestProperty("Referer", "https://www.entea.fr/CookieAuth.dll?GetLogon?curl=Z2F&reason=0&formdir=7");
conn.setRequestProperty("Accept-Encoding", "gzip,deflate");
conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
conn.setRequestProperty("Content-Length", Integer.toString(postParams.length()));
conn.setDoOutput(true);
conn.setDoInput(true);
// Send post request
DataOutputStream wr = new DataOutputStream(conn.getOutputStream());
wr.writeBytes(postParams);
wr.flush();
wr.close();
int responseCode = conn.getResponseCode();
System.out.println("\nSending 'POST' request to URL : " + url);
System.out.println("Post parameters : " + postParams);
System.out.println("Response Code : " + responseCode);
BufferedReader in =
new BufferedReader(new InputStreamReader(conn.getInputStream()));
String inputLine;
StringBuffer response = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
}
private String getPageContent(String url) throws Exception {
URL obj = new URL(url);
conn = (HttpsURLConnection) obj.openConnection();
// default is GET
conn.setRequestMethod("GET");
conn.setUseCaches(false);
// act like a browser
conn.setRequestProperty("User-Agent", USER_AGENT);
conn.setRequestProperty("Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
conn.setRequestProperty("Accept-Language", "en-US,en;q=0.5");
conn.setInstanceFollowRedirects(false);
if (cookies != null) {
for (String cookie : this.cookies) {
conn.addRequestProperty("Set-Cookie", cookie.split(";", 1)[0]);
}
}
int responseCode = conn.getResponseCode();
System.out.println("\nSending 'GET' request to URL : " + url);
System.out.println("Response Code : " + responseCode);
BufferedReader in =
new BufferedReader(new InputStreamReader(conn.getInputStream()));
String inputLine;
StringBuffer response = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
// Get the response cookies
setCookies(conn.getHeaderFields().get("Set-Cookie"));
return response.toString();
}
public String getFormParams(String html, String username, String password)
throws UnsupportedEncodingException {
System.out.println("Extracting form's data...");
Document doc = Jsoup.parse(html);
Element loginform = doc.getElementById("logonForm");
Elements inputElements = loginform.getElementsByTag("input");
List<String> paramList = new ArrayList<String>();
for (Element inputElement : inputElements) {
System.out.println("inputElement : " + inputElement.toString());
String key = inputElement.attr("name");
String value = inputElement.attr("value");
if (key.equals("username"))
value = username;
else if (key.equals("password")) {
value = password;
}
if (!(inputElement.attr("style").contains("none")))
paramList.add(key + "=" + URLEncoder.encode(value, "UTF-8"));
}
// build parameters list
StringBuilder result = new StringBuilder();
for (String param : paramList) {
if (result.length() == 0) {
result.append(param);
} else {
result.append("&" + param);
}
}
System.out.println("result : " + result.toString().replace("trusted=4", "SubmitCreds.x=0&SubmitCreds.y=0"));
return result.toString().replace("trusted=4", "SubmitCreds.x=0&SubmitCreds.y=0");
}
public void setCookies(List<String> cookies) {
if (cookies != null) {
this.cookies = cookies;
}
}
}
Have a good day! I am new to java. I wrote a program in java to login to a HTTP PHP website. When I run the program it doesn't get logged in. It just displays the content of the initial login page itself and not the page that comes after logged in. I don't know what my mistake is. I have searched a lot in the web and in this website as well. But no luck for me. :(
This is my program. Please let me know if there is any error in it.
import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.CookieHandler;
import java.net.CookieManager;
import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import java.net.HttpURLConnection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
public class HttpUrlConnectionExample1 {
private List<String> cookies;
private HttpURLConnection conn;
private final String USER_AGENT = "Mozilla/5.0";
public static void main(String[] args) throws Exception {
String url = "http://www.iwhatgroups.com/log%20in.php";
String email = "http://www.iwhatgroups.com";
HttpUrlConnectionExample1 http = new HttpUrlConnectionExample1();
// make sure cookies is turn on
CookieHandler.setDefault(new CookieManager());
// 1. Send a "GET" request, so that you can extract the form's data.
String page = http.GetPageContent(url);
String postParams = http.getFormParams(page, "MyUserId", "MyPassWord");
// 2. Construct above post's content and then send a POST request for
// authentication
http.sendPost(url, postParams);
// 3. success then go to email.
String result = http.GetPageContent(email);
System.out.println(result);
}
private void sendPost(String url, String postParams) throws Exception {
URL obj = new URL(url);
conn = (HttpURLConnection) obj.openConnection();
// Acts like a browser
conn.setUseCaches(false);
conn.setRequestMethod("POST");
conn.setRequestProperty("Host", "iwhatgroups.com");
conn.setRequestProperty("User-Agent", USER_AGENT);
// conn.setRequestProperty("Accept",
// "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
// conn.setRequestProperty("Accept-Language", "en-US,en;q=0.5");
// for (String cookie : this.cookies) {
// conn.addRequestProperty("Cookie", cookie.split(";", 1)[0]);
// }
// conn.setRequestProperty("Connection", "keep-alive");
conn.setRequestProperty("Referer", "http://iwhatgroups.com");
// conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
// conn.setRequestProperty("Content-Length", Integer.toString(postParams.length()));
conn.setDoOutput(true);
conn.setDoInput(true);
// Send post request
DataOutputStream wr = new DataOutputStream(conn.getOutputStream());
wr.writeBytes(postParams);
wr.flush();
wr.close();
int responseCode = conn.getResponseCode();
System.out.println("\nSending 'POST' request to URL : " + url);
System.out.println("Post parameters : " + postParams);
System.out.println("Response Code : " + responseCode);
BufferedReader in =
new BufferedReader(new InputStreamReader(conn.getInputStream()));
String inputLine;
StringBuffer response = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
// System.out.println(response.toString());
}
private String GetPageContent(String url) throws Exception {
URL obj = new URL(url);
conn = (HttpURLConnection) obj.openConnection();
// default is GET
conn.setRequestMethod("GET");
conn.setUseCaches(false);
// act like a browser
conn.setRequestProperty("User-Agent", USER_AGENT);
conn.setRequestProperty("Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
conn.setRequestProperty("Accept-Language", "en-US,en;q=0.5");
if (cookies != null) {
for (String cookie : this.cookies) {
conn.addRequestProperty("Cookie", cookie.split(";", 1)[0]);
}
}
int responseCode = conn.getResponseCode();
System.out.println("\nSending 'GET' request to URL : " + url);
System.out.println("Response Code : " + responseCode);
BufferedReader in =
new BufferedReader(new InputStreamReader(conn.getInputStream()));
String inputLine;
StringBuffer response = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
System.out.println(inputLine);
}
in.close();
// Get the response cookies
setCookies(conn.getHeaderFields().get("Set-Cookie"));
return response.toString();
}
public String getFormParams(String html, String username, String password)
throws UnsupportedEncodingException {
System.out.println("Extracting form's data...");
Document doc = Jsoup.parse(html);
// Form id
Element loginform = doc.getElementById("mainBody");
Elements inputElements = loginform.getElementsByTag("div");
List<String> paramList = new ArrayList<String>();
for (Element inputElement : inputElements) {
String key = inputElement.attr("name");
String value = inputElement.attr("value");
if (key.equals("inputEmail"))
value = username;
else if (key.equals("inputPassword"))
value = password;
paramList.add(key + "=" + URLEncoder.encode(value, "UTF-8"));
}
// build parameters list
StringBuilder result = new StringBuilder();
for (String param : paramList) {
if (result.length() == 0) {
result.append(param);
} else {
result.append("&" + param);
}
}
return result.toString();
}
public List<String> getCookies() {
return cookies;
}
public void setCookies(List<String> cookies) {
this.cookies = cookies;
}
}
Thank you all for reading or answering this.
NOTE:If I uncomment setRequestProperty it throws NullPointerException. Otherwise (I mean if I comment them) it gives the login page. Please help me out if possible.
Thanks again.
The problems here might be a lot.
- No Redirect handling in GET call
- Wrong URL for POST call
- Missing header parameters.
I would suggest you to use Firebug and debug with a browser a login performed by it.Sometimes the login is handled using JavaScript and the real POST URL might be different from the URL of the login page.
I want to create a java application that automatically logs into a website and does stuff. I'm testing it on my localhost. I'm actually totally new at this and I'm trying to get the concept from http://www.mkyong.com/java/how-to-automate-login-a-website-java-example/ and modifying the code to actually work for my localhost.
package random;
import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
public class InternetAutomationPost {
List<String> cookies;
public void setCookies(List<String> cookies) {
this.cookies = cookies;
}
public List<String> getCookies() {
return cookies;
}
private String requestWebPage(String address) {
try {
URL url = new URL(address);
HttpURLConnection con = (HttpURLConnection)url.openConnection();
// Don't use cache. Get a fresh copy.
con.setUseCaches(false);
// Use post or get.
// And default is get.
con.setRequestMethod("GET");
// Mimic a web browser.
con.addRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
con.addRequestProperty("Accept-Encoding", "gzip,deflate,sdch");
con.addRequestProperty("Accept-Language", "en-US,en;q=0.8");
con.addRequestProperty("Connection", "keep-alive");
con.addRequestProperty("User-Agent", "Mozilla/5.0");
if(cookies != null) {
con.addRequestProperty("Cache-Control", "max-age=0");
for (String cookie : this.cookies) {
System.out.print(cookie.split(";", 1)[0]);
con.addRequestProperty("Cookie", cookie.split(";", 1)[0]);
}
}
int responseCode = con.getResponseCode();
System.out.println("\nSending 'GET' request to URL : " + url);
System.out.println("Response Code : " + responseCode);
BufferedReader br = new BufferedReader(new InputStreamReader(con.getInputStream()));
String inputLine;
StringBuilder response = new StringBuilder();
while( (inputLine = br.readLine()) != null) {
response.append(inputLine);
}
br.close();
// Get the response cookies
setCookies(con.getHeaderFields().get("Set-Cookie"));
return response.toString();
} catch (MalformedURLException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
return "";
}
private String parsePage(String page) {
Document doc;
try {
doc = Jsoup.parse(page);
Elements form = doc.getElementsByAttributeValue("action", "login.php");
List<String> paramList = new ArrayList<String>();
for(Element loginForm : form) {
System.out.println(loginForm.html());
Elements Input = loginForm.getElementsByTag("input");
for(Element input : Input) {
String name = input.attr("name");
String value = input.attr("value");
if(name.equals("email")) {
value = "admin#admin.com";
} else if(name.equals("password")) {
value = "password";
} else if(name.equals("")) {
continue;
}
paramList.add(name + "=" + URLEncoder.encode(value, "UTF-8"));
}
}
StringBuilder params = new StringBuilder();
for(String values : paramList) {
if(params.length() == 0) {
params.append(values);
} else {
params.append("&" + values);
}
}
System.out.println("Params: " + params);
return params.toString();
} catch (IOException e) {
e.printStackTrace();
}
return "";
}
private void sendPostLogin(String location, String params) {
try {
URL url = new URL(location);
HttpURLConnection con = (HttpURLConnection) url.openConnection();
// Don't use cache. Get a fresh copy.
con.setUseCaches(false);
// Use post or get. We use post this time.
con.setRequestMethod("POST");
// Mimic a web browser.
con.addRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
con.addRequestProperty("Accept-Encoding", "gzip,deflate,sdch");
con.addRequestProperty("Accept-Language", "en-US,en;q=0.8");
con.addRequestProperty("Connection", "keep-alive");
con.addRequestProperty("User-Agent", "Mozilla/5.0");
if(cookies != null) {
con.addRequestProperty("Cache-Control", "max-age=0");
for (String cookie : this.cookies) {
con.addRequestProperty("Cookie", cookie.split(";", 1)[0]);
}
}
con.addRequestProperty("Content-Length", Integer.toString(params.length()));
con.addRequestProperty("Content-Type", "application/x-www-form-urlencoded");
con.addRequestProperty("Host", "localhost");
con.addRequestProperty("Origin", "http://localhost");
con.addRequestProperty("Referrer", "http://localhost/social/index.php");
con.setDoOutput(true);
con.setDoInput(true);
// Write the parameters. Send post request.
DataOutputStream wr = new DataOutputStream(con.getOutputStream());
wr.writeBytes(params);
wr.flush();
wr.close();
int responseCode = con.getResponseCode();
System.out.println("\nSending 'POST' request to URL : " + url);
System.out.println("Post parameters : " + params);
System.out.println("Response Code : " + responseCode);
BufferedReader br = new BufferedReader(new InputStreamReader(con.getInputStream()));
String inputLine;
StringBuilder response = new StringBuilder();
while( (inputLine = br.readLine()) != null) {
response.append(inputLine);
}
br.close();
// Get the response cookies
setCookies(con.getHeaderFields().get("Set-Cookie"));
System.out.println(response.toString());
} catch (MalformedURLException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
/**
* #param args
*/
public static void main(String[] args) {
InternetAutomationPost object = new InternetAutomationPost();
String page = object.requestWebPage("http://localhost/social");
String params = object.parsePage(page);
object.sendPostLogin("http://localhost/social/index.php", params);
}
}
EDIT:
Found out why it sent HTTP response code: 413.
con.addRequestProperty("Content-Length:", Integer.toString(params.length()));
should have been:
con.addRequestProperty("Content-Length", Integer.toString(params.length()));
There was a stray ':'. I've fixed it now.
BUT, still my code doesn't actually login and I still need help .
I have put my full program here now.
I might be wrong but I'm thinking that the params aren't actually getting written to the con.getOutputStream() in the code here:
DataOutputStream wr = new DataOutputStream(con.getOutputStream());
wr.writeBytes(params);
wr.flush();
wr.close();
You set the Content-Length to Integer.toString(params.length())
My guess would be, that your params written as bytes are longer than the Content-Length and your server receives more bytes than expected.
Try:
con.addRequestProperty("Content-Length:", Integer.toString(params.getBytes("UTF-8").length()));
This depends on your encoding obviously. Also have a look at this.
As part of a small project i'm doing that will never actually go into 'production' I need to be able to log into a very secure website and retrieve html.
I've been looking into doing this using the apache commons HTTPCLient. However i just wanted to make sure it was even possible as this website is very secure and likely has sso methods to sign in?
If it is possible, what is the best way to do this? I need to be able to navigate through about three pages once i have logged in so will need to store the cookie or session somehow.
Thanks very much!
yes, its possible to do this using apache http components, but for interacting with complex websites nothing (that i know) beats HtmlUnit. to work with httpcomponents you'd need to "script" the whole sequence of http requests, and you'll have a problem if anything in the middle relies on dynamic content/javascript.
HtmlUnit, on the other hand, is an almost complete "bowser in a box" and you can script the interaction at a much higher level - click this, fill those values, submit etc.
You need jsoup.jar file to send the response to the particular website
look at the gmail form data through inspect element for element verfication
package com.kowthal;
import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.CookieHandler;
import java.net.CookieManager;
import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import javax.net.ssl.HttpsURLConnection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
public class HttpUrlConnectionExample {
private List<String> cookies;
private HttpsURLConnection conn;
private final String USER_AGENT = "Mozilla/5.0";
public static void main(String[] args) throws Exception {
String url = "https://accounts.google.com/ServiceLoginAuth";
String gmail = "https://mail.google.com/mail/";
HttpUrlConnectionExample http = new HttpUrlConnectionExample();
// make sure cookies is turn on
CookieHandler.setDefault(new CookieManager());
// 1. Send a "GET" request, so that you can extract the form's data.
String page = http.GetPageContent(url);
String postParams = http.getFormParams(page, "username#gmail.com", "password");
// 2. Construct above post's content and then send a POST request for
// authentication
http.sendPost(url, postParams);
// 3. success then go to gmail.
String result = http.GetPageContent(gmail);
System.out.println(result);
}
private void sendPost(String url, String postParams) throws Exception {
URL obj = new URL(url);
conn = (HttpsURLConnection) obj.openConnection();
// Acts like a browser
conn.setUseCaches(false);
conn.setRequestMethod("POST");
conn.setRequestProperty("Host", "accounts.google.com");
conn.setRequestProperty("User-Agent", USER_AGENT);
conn.setRequestProperty("Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
conn.setRequestProperty("Accept-Language", "en-US,en;q=0.5");
for (String cookie : this.cookies) {
conn.addRequestProperty("Cookie", cookie.split(";", 1)[0]);
}
conn.setRequestProperty("Connection", "keep-alive");
conn.setRequestProperty("Referer", "https://accounts.google.com/ServiceLoginAuth");
conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
conn.setRequestProperty("Content-Length", Integer.toString(postParams.length()));
conn.setDoOutput(true);
conn.setDoInput(true);
// Send post request
DataOutputStream wr = new DataOutputStream(conn.getOutputStream());
wr.writeBytes(postParams);
wr.flush();
wr.close();
int responseCode = conn.getResponseCode();
System.out.println("\nSending 'POST' request to URL : " + url);
System.out.println("Post parameters : " + postParams);
System.out.println("Response Code : " + responseCode);
BufferedReader in =
new BufferedReader(new InputStreamReader(conn.getInputStream()));
String inputLine;
StringBuffer response = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
// System.out.println(response.toString());
}
private String GetPageContent(String url) throws Exception {
URL obj = new URL(url);
conn = (HttpsURLConnection) obj.openConnection();
// default is GET
conn.setRequestMethod("GET");
conn.setUseCaches(false);
// act like a browser
conn.setRequestProperty("User-Agent", USER_AGENT);
conn.setRequestProperty("Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
conn.setRequestProperty("Accept-Language", "en-US,en;q=0.5");
if (cookies != null) {
for (String cookie : this.cookies) {
conn.addRequestProperty("Cookie", cookie.split(";", 1)[0]);
}
}
int responseCode = conn.getResponseCode();
System.out.println("\nSending 'GET' request to URL : " + url);
System.out.println("Response Code : " + responseCode);
BufferedReader in =
new BufferedReader(new InputStreamReader(conn.getInputStream()));
String inputLine;
StringBuffer response = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
// Get the response cookies
setCookies(conn.getHeaderFields().get("Set-Cookie"));
return response.toString();
}
public String getFormParams(String html, String username, String password)
throws UnsupportedEncodingException {
System.out.println("Extracting form's data...");
Document doc = Jsoup.parse(html);
// Google form id
Element loginform = doc.getElementById("gaia_loginform");
Elements inputElements = loginform.getElementsByTag("input");
List<String> paramList = new ArrayList<String>();
for (Element inputElement : inputElements) {
String key = inputElement.attr("name");
String value = inputElement.attr("value");
if (key.equals("Email"))
value = username;
else if (key.equals("Passwd"))
value = password;
paramList.add(key + "=" + URLEncoder.encode(value, "UTF-8"));
}
// build parameters list
StringBuilder result = new StringBuilder();
for (String param : paramList) {
if (result.length() == 0) {
result.append(param);
} else {
result.append("&" + param);
}
}
return result.toString();
}
public List<String> getCookies() {
return cookies;
}
public void setCookies(List<String> cookies) {
this.cookies = cookies;
}
}