Develop Reference

Unable to get full HTML of a page, it stops at a certain point

I am trying to download the HTML of page. After it downloads I try to Log it. Everything goes smoothly but the HTML stops at a certain point every time, even though it has a lot more HTML to show.
I tried using a different page, my page which just has some instructions for my Company and it worked perfectly. Is there a limit maybe? I tried it with urlconnection.connect(), and without it and there is no difference.
public class MainActivity extends AppCompatActivity {
public class DownloadHTML extends AsyncTask<String, Void, String>{
#Override
protected String doInBackground(String... urls) {
URL url;
String result = "";
HttpURLConnection urlConnection = null;
try {
url = new URL(urls[0]);
urlConnection = (HttpURLConnection)url.openConnection();
InputStream in = urlConnection.getInputStream();
InputStreamReader reader = new InputStreamReader(in);
int data = reader.read();
while (data!=-1){
char current = (char) data;
result += current;
data = reader.read();
}
return result;
} catch (Exception e) {
e.printStackTrace();
return "Fail";
}
}
}
#Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
String Result = "";
DownloadHTML task = new DownloadHTML();
try {
Result = task.execute("http://www.posh24.se/kandisar").get();
} catch (Exception e) {
e.printStackTrace();
}
Log.i("URL", Result);
}
}
Here is the splitting and it wont work.
try {
Result = task.execute("http://www.posh24.se/kandisar").get();
String[] splitStrings = Result.split("<div class=\"channelListEntry\">");
Pattern p = Pattern.compile("<img src=\"(.*?)\"");
Matcher m = p.matcher(splitStrings[0]);
while (m.find()){
CelebUrls.add(m.group(1));
}
p = Pattern.compile("alt=\"(.*?)\"");
m = p.matcher(splitStrings[0]);
while (m.find()){
CelebNames.add(m.group(1));
}
} catch (Exception e) {
e.printStackTrace();
}
Log.i("URL", Arrays.toString(CelebUrls.toArray()));
}
}

Modifing your method like this will give you the content of the html page in UTF-8 format.
(In this case its UTF-8 because the page is encoded like that, in doubt you can pass Charset.forName("utf-8") as second paramter to the constructor of InputStreamReader)
When testing you example implementation I only got some output with various unreadable characters.
Ignore the class and the method changes, I only made them to have a standalone example.
public class ParsingTest {
static String doInBackground(String address) {
URL url;
StringBuilder result = new StringBuilder(1000);
HttpURLConnection urlConnection = null;
try {
url = new URL(address);
urlConnection = (HttpURLConnection)url.openConnection();
InputStream in = urlConnection.getInputStream();
BufferedReader reader = new BufferedReader(new InputStreamReader(in));
String line = reader.readLine();
while (line != null){
result.append(line);
result.append("\n");
line = reader.readLine();
}
return result.toString();
} catch (Exception e) {
e.printStackTrace();
return "Fail";
}
}
public static void main(String[] args) {
String result = doInBackground("http://www.posh24.se/kandisar");
System.out.println(result);
}
}
If the only part that interest you are the images of the top100, you can just adjust the while loop to:
String line = reader.readLine();
while (line != null){
if (line.contains("<div class=\"channelListEntry\">")) {
reader.readLine();
reader.readLine();
line = reader.readLine().trim();
// At this points its probably easier to use a List<String> for the result instead
result.append(line);
result.append("\n");
}
line = reader.readLine();
}
This is only a simplied example based on the current design of the page, where the img comes 3 lines after the declaration of the div.
If you want to you can also just extract the url of the image and the alt description directly at this point. Instead of using complicated regex you could rely on the String#indexOf instead.
private static final String SRC = "src=\"";
private static final String ALT = "\" alt=\"";
private static final String END = "\"/>";
public static void extract(String image) {
int index1 = image.indexOf(SRC);
int index2 = image.indexOf(ALT);
int index3 = image.indexOf(END);
System.out.println(image);
System.out.println(image.substring(index1 + SRC.length(), index2));
System.out.println(image.substring(index2 + ALT.length(), index3));
}
Note that if you directly process the content from the page your app does not require the memory to store the full page.

How can i create vertex and edge with HTTP in Orientdb?

Here is my code. I did a "GET" method for have a response of my DB.
Then I read my own file csv. All is ok in this point, but... I have not idea how can i do a "POST" method. i know that i need to use "addRequestProperty"method.
Any idea for create vertex and edge?
public void run() throws MalformedURLException, JSONException, IOException {
String viaURl = "http://xxxxxxxxxxxxxxxxxxxxxxxxxxx/mydb";
URL url = new URL(viaURl);
HttpURLConnection conexion = null;
String texto = null;
String json;
BufferedReader in = null, in2 = null;
int numDump = 5;
String dato;
String csvSplitBy = ";";
int numApps = 0;
OutputStreamWriter out;
try {
Authenticator.setDefault(new Authenticator() {
#Override
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication("xxxxx", "xxxxxxxxxxxxx.".toCharArray());
}
});
conexion = (HttpURLConnection) url.openConnection();
conexion.setRequestMethod("GET");
conexion.connect();
System.out.println("¡¡¡Conectado!!!");
in = new BufferedReader(new InputStreamReader(conexion.getInputStream()));
out = new OutputStreamWriter(conexion.getOutputStream());
json = "";
while ((texto = in.readLine()) != null) {
json += texto;
}
in.close();
System.out.println(json);
conexion.setDoOutput(true);
try {
for (int i = 0; i < numDump; i++) {
String csvFile = "/home/danicroque/dump/dump_" + i;
try {
in2 = new BufferedReader(new FileReader(csvFile));
while ((dato = in2.readLine()) != null) {
numApps++;
String[] datos = dato.split(csvSplitBy, 15);
conexion.setRequestMethod("POST");
conexion.addRequestProperty("_id0" , datos[0]);
}
} catch (FileNotFoundException e) {
e.printStackTrace();
}
}
} catch (IOException ex) {
ex.printStackTrace();
}
} catch (IOException e) {
e.printStackTrace();
} finally {
System.out.println("Fin");
}
}
}
Thank in advance.

You can use this POST methods to create class:
http://your_host:2480/class/mydb/className
to add property to a class
http://your_host:2480/property/mydb/className/propertyName
You can fine more detailed information here.
Hope it helps,
Alex.
UPDATE:
To insert use this POST method:
http://your_host:2480/command/mydb/sql/insert into className(propertyName) values(“yourValue”)

Reading JSON data from internal android storage and populating a ListView

I have a bunch of json files which are saved into the android application internal storage like this:
JSONObject jsonToSave = createJSONObject();
filenames = getFilenames(filepath);
int filenameNumber = filenames.size() + 1;
ContextWrapper contextWrapper = new ContextWrapper(getApplicationContext());
File directory = contextWrapper.getDir(filepath, Context.MODE_PRIVATE);
filename = "newAssessment(" + filenameNumber + ").json";
internalFile = new File(directory , filename);
try {
FileOutputStream fos = new FileOutputStream(internalFile);
fos.write(jsonToSave.toString().getBytes());
}
catch (IOException e){
e.printStackTrace();
}
For context, here are the getFilenames and createJSONObject methods:
private List<String> getFilenames(String path) {
File files = new File(path);
FileFilter filter = new FileFilter() {
private final List<String> exts = Arrays.asList("json");
#Override
public boolean accept(File pathname) {
String ext;
String path = pathname.getPath();
ext = path.substring(path.lastIndexOf(".") + 1);
return exts.contains(ext);
}
};
final File [] filesFound = files.listFiles(filter);
List<String> list = new ArrayList<String>();
if (filesFound != null && filesFound.length > 0){
for (File file : filesFound){
list.add(file.getName());
}
}
return list;
}
private JSONObject createJSONObject(){
String orchard = orchardList.getSelectedItem().toString();
String description = ETAssessmentDescription.getText().toString();
String type = typeList.getSelectedItem().toString();
String assessorStaff = ETAssessorStaff.getText().toString();
String date = ETAssessmentDate.getText().toString();
String additionalAssessorStaff = ETAdditionalAssessorStaff.getText().toString();
String sampleSize = ETSampleSize.getText().toString();
String lineSize = ETLineSize.getText().toString();
boolean fruitCollected;
JSONObject jsonObj = new JSONObject();
if(CBFruitCollected.isChecked()){
fruitCollected = true;
}else{
fruitCollected = false;
}
try{
jsonObj.put("orchard", orchard);
jsonObj.put("description", description);
jsonObj.put("type", type);
jsonObj.put("assessorStaff", assessorStaff);
jsonObj.put("assessmentDate", date);
jsonObj.put("additionalAssessorStaff", additionalAssessorStaff);
jsonObj.put("sampleSize", sampleSize);
jsonObj.put("lineSize", lineSize);
jsonObj.put("fruitCollected", fruitCollected);
}
catch(JSONException ex){
ex.printStackTrace();
}
return jsonObj;
}
What I am trying to do is populate a ListView with a couple of fields from each file. This is what I have so far, using the same getFilenames method:
filenames = getFilenames(filepath);
ContextWrapper contextWrapper = new ContextWrapper(getApplicationContext());
File directory = contextWrapper.getDir(filepath, Context.MODE_PRIVATE);
for (String filename : filenames) {
internalFile = new File(directory, filename);
try {
FileInputStream fis = new FileInputStream(internalFile);
DataInputStream in = new DataInputStream(fis);
BufferedReader br = new BufferedReader(new InputStreamReader(in));
String assessmentDate = "";
String orchard = "";
String strLine;
int dataCounter = 0;
while ((strLine = br.readLine()) != null) {
myData = myData + strLine;
try {
JSONObject object = new JSONObject(myData);
assessmentDate = object.getString("assessmentDate");
orchard = object.getString("orchard");
} catch (JSONException e) {
e.printStackTrace();
}
data.add(dataCounter, assessmentDate + " : " + orchard);
dataCounter++;
}
in.close();
}
catch(IOException e){
e.printStackTrace();
}
}
LVAssessments.setAdapter(new ArrayAdapter<>(this, android.R.layout.simple_list_item_1, data));
I'm sure there is something simple that I'm missing. Thanks for help in advance

Java specific image downloader with multiple URL's

I can't figure out how to reference a specific line of text in a txt file. I need a specific image from a specific list of URL's and I am trying to generate the URL's by concatenating a URL prefix and a search number from a list of numbers in a txt file. I can't figure out how to reference the txt file and get a string from a line number.
package getimages;
public class ExtractAllImages {
public static int url_to_get = 1;
public static String urlupc;
public static String urlpre = "http://urlineedimagefrom/searchfolder/";
public static String url2 = "" + urlpre + urlupc + "";
public static void main(String args[]) throws Exception {
while(url_to_get > 2622){
String line = Files.readAllLines(Paths.get("file_on_my_desktop.txt")).get(url_to_get);
urlupc = line;
url2 = "" + urlpre + urlupc + "";
String webUrl = url2;
URL url = new URL(webUrl);
URLConnection connection = url.openConnection();
InputStream is = connection.getInputStream();
InputStreamReader isr = new InputStreamReader(is);
BufferedReader br = new BufferedReader(isr);
HTMLEditorKit htmlKit = new HTMLEditorKit();
HTMLDocument htmlDoc = (HTMLDocument) htmlKit.createDefaultDocument();
HTMLEditorKit.Parser parser = new ParserDelegator();
HTMLEditorKit.ParserCallback callback = htmlDoc.getReader(0);
parser.parse(br, callback, true);
for (HTMLDocument.Iterator iterator = htmlDoc.getIterator(HTML.Tag.IMG); iterator.isValid(); iterator.next()) {
AttributeSet attributes = iterator.getAttributes();
String imgSrc = (String) attributes.getAttribute(HTML.Attribute.SRC);
if (imgSrc != null && (imgSrc.endsWith(".jpg") || (imgSrc.endsWith(".png")) || (imgSrc.endsWith(".jpeg")) || (imgSrc.endsWith(".bmp")) || (imgSrc.endsWith(".ico")))) {
try {
downloadImage(webUrl, imgSrc);
} catch (IOException ex) {
System.out.println(ex.getMessage());
}
}
}
}
}
public static String right(String value, int length) {
return value.substring(value.length() - length);}
private static void downloadImage(String url, String imgSrc) throws IOException {
BufferedImage image = null;
try {
if (!(imgSrc.startsWith("http"))) {
url = url + imgSrc;
} else {
url = imgSrc;
}
String webUrl = url2;
String imagename = right(webUrl , 12);
imgSrc = imgSrc.substring(imgSrc.lastIndexOf("/") + 1);
String imageFormat = null;
imageFormat = imgSrc.substring(imgSrc.lastIndexOf(".") + 1);
String imgPath = null;
imgPath = "C:/Users/Noah/Desktop/photos/" + urlupc + ".jpg";
URL imageUrl = new URL(url);
image = ImageIO.read(imageUrl);
if (image != null) {
File file = new File(imgPath);
ImageIO.write(image, imageFormat, file);
}
} catch (Exception ex) {
ex.printStackTrace();
}
}
}
My error is in setting the String line, My txt file has 2622 lines and I can't reference the file on my desktop and im not sure how to set the file path to my desktop? Sorry I'm not good at java.
Thanks for any help.

From what I understood you don't know how to access a file that is on your Desktop. Try doing this:
String path = "C:\\Users\\" + System.getProperty("user.name") + "\\Desktop\\" + fileName + ".txt";
Let me explain. We are using
System.getProperty("user.name")
to get the name of the user, if you don't want to use it then you can replace it with the name of your user. Then we are using
"\\Desktop\\"
to acess the Desktop, and finally we are adding the
fileName + ".txt"
to access the file we want that has the extension '.txt'.

How to open a txt file located on the web and read its contents to a string?

I have looked around on how to do this and I keep finding different solutions, none of which has worked fine for me and I don't understand why. Does FileReader only work for local files? I tried a combination of scripts found on the site and it still doesn't quite work, it just throws an exception and leaves me with ERROR for the variable content. Here's the code I've been using unsuccessfully:
public String downloadfile(String link){
String content = "";
try {
URL url = new URL(link);
URLConnection conexion = url.openConnection();
conexion.connect();
InputStream is = url.openStream();
BufferedReader br = new BufferedReader( new InputStreamReader(is));
StringBuilder sb = new StringBuilder();
String line;
while ((line = br.readLine()) != null) {
sb.append(line);
}
content = sb.toString();
br.close();
is.close();
} catch (Exception e) {
content = "ERROR";
Log.e("ERROR DOWNLOADING",
"File not Found" + e.getMessage());
}
return content;
}

Use this as a downloader(provide a path to save your file(along with the extension) and the exact link of the text file)
public static void downloader(String fileName, String url) throws IOException {
File file = new File(fileName);
url = url.replace(" ", "%20");
URL website = new URL(url);
if (file.exists()) {
file.delete();
}
if (!file.exists()) {
ReadableByteChannel rbc = Channels.newChannel(website.openStream());
FileOutputStream fos = new FileOutputStream(fileName);
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
fos.close();
}
}
Then call this function to read the text file
public static String[] read(String fileName) {
String result[] = null;
Vector v = new Vector(10, 2);
BufferedReader br = null;
try {
br = new BufferedReader(new FileReader(fileName));
String tmp = "";
while ((tmp = br.readLine()) != null) {
v.add(tmp);
}
Iterator i = v.iterator();
result = new String[v.toArray().length];
int count = 0;
while (i.hasNext()) {
result[count++] = i.next().toString();
}
} catch (IOException ioe) {
ioe.printStackTrace();
}
return (result);
}
And then finally the main method
public static void main(){
downloader("D:\\file.txt","http://www.abcd.com/textFile.txt");
String data[]=read("D:\\file.txt");
}

try this:
try {
// Create a URL for the desired page
URL url = new URL("mysite.com/thefile.txt");
// Read all the text returned by the server
BufferedReader in = new BufferedReader(new InputStreamReader(url.openStream()));
String str;
StringBuilder sb = new StringBuilder();
while ((str = in.readLine()) != null) {
// str is one line of text; readLine() strips the newline character(s)
sb.append(str );
}
in.close();
String serverTextAsString = sb.toString();
} catch (MalformedURLException e) {
} catch (IOException e) {
}