Closed. This question needs to be more focused. It is not currently accepting answers.
Want to improve this question? Update the question so it focuses on one problem only by editing this post.
Closed 3 years ago.
Improve this question
How to zip a directory which contains sub directories and files and split it like zip1.zip, zip2.zip ... zipN.zip, such that all "zip files size" < "max size". After this is unzipped all the files should appear into a directory with the same directory structure as it was zipped.
For Example -- Check Image
I've similar implementation to your problem. Please check if it works for you.
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import java.util.zip.ZipOutputStream;
public class BasicCompressFileService {
private List<String> fileList; // Stores list of file names.
private static final String OUTPUT_ZIP_FILE = "File.zip"; // Output zip base name
private String SOURCE_FOLDER; // Stores source folder for Utils.
/**
* Funtion to zip directory. Zip files contents will be fetched from inputDir and
* Output will be stored in outputDir. The zip file will be split and each chunck
* will not be more that provided maxSize.
*/
public void compressFolderWithMaxLimit(String inputDir, String outputDir, Long maxSize) {
setupZipUtils();
SOURCE_FOLDER = inputDir;
generateFileList(new File(inputDir));
try {
final int BUFFER = (int) (maxSize * 1048576); // max buffer to create a zip file.
String directoryPath = outputDir;
if (!directoryPath.endsWith("/")) {
directoryPath = directoryPath + "/";
}
String inDirectoryPath = inputDir;
if (!inDirectoryPath.endsWith("/")) {
inDirectoryPath = inDirectoryPath + "/";
}
byte fileRAW[] = new byte[BUFFER];
FileInputStream entryFile;
int count = 1;
for (String aFile : fileList) {
entryFile = new FileInputStream(inDirectoryPath + aFile);
int len;
while ((len = entryFile.read(fileRAW)) > 0) {
FileOutputStream fos = new FileOutputStream(directoryPath
+ OUTPUT_ZIP_FILE.toLowerCase().replace(".zip", (count++) + ".zip").toUpperCase());
ZipOutputStream zos = new ZipOutputStream(fos);
zos.putNextEntry(new ZipEntry(aFile));
zos.write(fileRAW, 0, len);
zos.closeEntry();
zos.close();
}
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* Setup zip Utils.
*/
public void setupZipUtils() {
fileList = new ArrayList<String>();
}
/**
* Generate File list.
* #param node
*/
public void generateFileList(File node) {
// add file only
if (node.isFile()) {
fileList.add(generateZipEntry(node.toString()));
}
if (node.isDirectory()) {
String[] subNote = node.list();
for (String filename : subNote) {
generateFileList(new File(node, filename));
}
}
}
/**
* Generated the Zip entry.
* #param file
* #return
*/
private String generateZipEntry(String file) {
return file.substring(SOURCE_FOLDER.length() + 1, file.length());
}
#Override
public void decompressFolderWith(String inputDir, String outputDir) {
String outputFolder = outputDir;
if (!outputFolder.endsWith("/")) {
outputFolder = outputFolder + "/";
}
String intputFolder = inputDir;
if (!intputFolder.endsWith("/")) {
intputFolder = intputFolder + "/";
}
setupZipUtils();
generateFileList(new File(intputFolder));
Collections.sort(fileList);
for (int x = 0; x < fileList.size(); x++)
try {
unzipFile(new FileInputStream(intputFolder + fileList.get(x)), outputFolder);
} catch (IOException e1) {
e1.printStackTrace();
}
}
/**
* Unzips the file in provided folder.
* #param in
* #param outputFolder
* #throws IOException
*/
private void unzipFile(InputStream in, String outputFolder) throws IOException {
ZipInputStream is = new ZipInputStream(in);
for (ZipEntry entry = null; (entry = is.getNextEntry()) != null;) {
File newFile = new File(outputFolder + entry.getName());
newFile.getParentFile().mkdirs();
OutputStream os = new BufferedOutputStream(new FileOutputStream(newFile, true));
try {
final int bufferSize = 1024;
byte[] buffer = new byte[bufferSize];
for (int readBytes = -1; (readBytes = is.read(buffer, 0, bufferSize)) > -1;) {
os.write(buffer, 0, readBytes);
}
os.flush();
} catch (Exception e1) {
e1.printStackTrace();
} finally {
os.close();
}
}
is.close();
}
}
Related
folder structure is here
console output is here
I'd like to write a test class for the 2 methods below
package jfe;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import org.apache.commons.compress.archivers.sevenz.SevenZArchiveEntry;
import org.apache.commons.compress.archivers.sevenz.SevenZFile;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.utils.IOUtils;
public class JThreadFile {
/**
* uncompresses .tar file
* #param in
* #param out
* #throws IOException
*/
public static void decompressTar(String in, File out) throws IOException {
try (TarArchiveInputStream tin = new TarArchiveInputStream(new FileInputStream(in))){
TarArchiveEntry entry;
while ((entry = tin.getNextTarEntry()) != null) {
if (entry.isDirectory()) {
continue;
}
File curfile = new File(out, entry.getName());
File parent = curfile.getParentFile();
if (!parent.exists()) {
parent.mkdirs();
}
IOUtils.copy(tin, new FileOutputStream(curfile));
}
}
}
/**
* uncompresses .7z file
* #param in
* #param destination
* #throws IOException
*/
public static void decompressSevenz(String in, File destination) throws IOException {
//#SuppressWarnings("resource")
SevenZFile sevenZFile = new SevenZFile(new File(in));
SevenZArchiveEntry entry;
while ((entry = sevenZFile.getNextEntry()) != null){
if (entry.isDirectory()){
continue;
}
File curfile = new File(destination, entry.getName());
File parent = curfile.getParentFile();
if (!parent.exists()) {
parent.mkdirs();
}
FileOutputStream out = new FileOutputStream(curfile);
byte[] content = new byte[(int) entry.getSize()];
sevenZFile.read(content, 0, content.length);
out.write(content);
out.close();
}
sevenZFile.close();
}
}
I use testNG and try to read from a folder, filter the folder for certain extensions (.tar and .7z) feed those files to the uncompress methods and compare the result to the actualOutput folder with AssertEquals. I manage to read the file names from the folder (see console output) but can't feed them to decompressTar(String in, File out). Is this because "result" is a String array and I need a String? I have no clue how DataProvider of TestNG handles data. Any help would be appreciated :) Thank you :)
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import org.testng.Assert;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
public class JThreadFileTest {
protected static final File ACT_INPUT = new File("c:\\Test\\TestInput\\"); //input directory
protected static final File EXP_OUTPUT = new File("c:\\Test\\ExpectedOutput\\"); //expected output directory
protected static final File TEST_OUTPUT = new File("c:\\Test\\TestOutput\\");
#DataProvider(name = "tarJobs")
public Object[] getTarJobs() {
//1
String[] tarFiles = ACT_INPUT.list(new FilenameFilter()
{
public boolean accept(File dir, String name)
{
return name.endsWith(".tar");
}
});
//2
Object[] result = new Object[tarFiles.length];
int i = 0;
for (String filename : tarFiles) {
result[i] = filename;
i++;
}
return result;
}
#Test(dataProvider = "tarJobs")
public void testTar(String result) throws IOException {
System.out.println("Running test" + result);
--> JThreadFile.decompressTar(result, TEST_OUTPUT);
Assert.assertEquals(TEST_OUTPUT, EXP_OUTPUT);
}
}
I wrote java code to extract zip file in alfresco.
But through that code some zip files are not getting extracted properly.
And i have one more observation, that if i manually unzip that file and again manually create zip file then its working fine and its successfully getting extracted in alfresco site.
So i am not getting whats the problem exactly.
Is it problem with file or problem with my code or problem with zip file
Can anyone help me with this scenario...
Please refer below Logs for refrence...
2017-01-25 12:10:12,069 ERROR [tandf.ingestion.TransformExceptionService] [org.springframework.jms.listener.DefaultMessageListenerContainer#22-1] Unhandled Exception occured in ingestion pipeline: org.alfresco.error.AlfrescoRuntimeException: 00250134 Exception in Transaction.
at org.alfresco.repo.transaction.RetryingTransactionHelper.doInTransaction(RetryingTransactionHelper.java:542)
at org.alfresco.repo.transaction.RetryingTransactionHelper.doInTransaction(RetryingTransactionHelper.java:326)
at com.ixxus.tandf.service.XmlZipExtractService.lambda$extract$4(XmlZipExtractService.java:235)
at org.alfresco.repo.security.authentication.AuthenticationUtil.runAs(AuthenticationUtil.java:548)
at
I am Using below code for extracting zip file..
private void unzipToNode(final String zipFilePath, final NodeRef destinationFolder, final String isbn, final String zipFileNoderef, final String assetType, final NodeRef ingestedNodeRef) throws IOException {
LOG.debug("Inside : Start :-> unzipToNode, zipFileNoderef : {}", zipFileNoderef);
String rootDisplayPath = nodeUtils.getDisplayPath(destinationFolder);
List<NodeRef> folderNodes = new ArrayList<>();
try (ZipFile zipFile = new ZipFile(zipFilePath); FileInputStream fis = new FileInputStream(zipFilePath); ZipInputStream zipInput = new ZipInputStream(fis);) {
ZipEntry entry = zipInput.getNextEntry();
int zipFileSize = zipFile.size();
LOG.info("{} : zipFileSize from zip api for zipFileNoderef : {}", zipFileSize, zipFileNoderef);
int zipManualFileCount = 0;
while (entry != null) {
zipManualFileCount++;
LOG.debug("Processing the zip entry : {}, zipFileNoderef : {}", entry.getName(), zipFileNoderef);
InputStream inputStream = zipFile.getInputStream(entry);
/** create or get final folder path for current entry */
NodeRef nodeRef = createOrGetFolderStructure(destinationFolder, entry);
folderNodes.add(nodeRef);
String name;
if (!entry.isDirectory()) {
name = getFileNameFromEntry(entry);
/** if zip entry is file, then create and write the new node in validation site */
createNodeOnValidationSite(nodeRef, name, inputStream, assetType, ingestedNodeRef, rootDisplayPath);
}
/** close current entry and fetch next one */
zipInput.closeEntry();
entry = zipInput.getNextEntry();
}
LOG.info("{} : zipManualFileCount from zip api for zipFileNoderef : {}", zipManualFileCount, zipFileNoderef);
/** Close the last entry */
zipInput.closeEntry();
}
for(NodeRef folderNode : folderNodes){
if(nodeUtils.isNodeEmpty(folderNode)){
LOG.debug("Found empty folder [{}] within .zip [{}]", folderNode, ingestedNodeRef);
/** if the folder is empty we need to copy all properties to it as well to archive it */
copyAllProperties(ingestedNodeRef, folderNode, rootDisplayPath, assetType);
}
}
LOG.debug("Inside : End :-> unzipToNode : zipFileNoderef : {}", zipFileNoderef);
}
For extract a zip file can you please try this code.
package org.alfresco.repo.action.executer;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.Serializable;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.zip.ZipException;
import org.alfresco.model.ContentModel;
import org.alfresco.repo.action.ParameterDefinitionImpl;
import org.alfresco.repo.action.executer.ActionExecuterAbstractBase;
import org.alfresco.repo.content.MimetypeMap;
import org.alfresco.service.cmr.action.Action;
import org.alfresco.service.cmr.action.ParameterDefinition;
import org.alfresco.service.cmr.dictionary.DataTypeDefinition;
import org.alfresco.service.cmr.dictionary.InvalidAspectException;
import org.alfresco.service.cmr.dictionary.InvalidTypeException;
import org.alfresco.service.cmr.model.FileExistsException;
import org.alfresco.service.cmr.model.FileFolderService;
import org.alfresco.service.cmr.model.FileInfo;
import org.alfresco.service.cmr.repository.ChildAssociationRef;
import org.alfresco.service.cmr.repository.ContentIOException;
import org.alfresco.service.cmr.repository.ContentReader;
import org.alfresco.service.cmr.repository.ContentService;
import org.alfresco.service.cmr.repository.ContentWriter;
import org.alfresco.service.cmr.repository.InvalidNodeRefException;
import org.alfresco.service.cmr.repository.MimetypeService;
import org.alfresco.service.cmr.repository.NodeRef;
import org.alfresco.service.cmr.repository.NodeService;
import org.alfresco.service.namespace.InvalidQNameException;
import org.alfresco.service.namespace.NamespaceService;
import org.alfresco.service.namespace.QName;
import org.alfresco.util.TempFileProvider;
import org.alfresco.web.bean.repository.Repository;
import org.apache.tools.zip.ZipEntry;
import org.apache.tools.zip.ZipFile;
/**
* Zip action executor
*
* #author Davide Taibi
*/
public class ZipActionExecuter extends ActionExecuterAbstractBase
{
public static final String NAME = "importzip";
public static final String PARAM_ENCODING = "encoding";
public static final String PARAM_DESTINATION_FOLDER = "destination";
private static final String TEMP_FILE_PREFIX = "alf";
private static final String TEMP_FILE_SUFFIX = ".zip";
/**
* The node service
*/
private NodeService nodeService;
/**
* The content service
*/
private ContentService contentService;
private MimetypeService mimetypeService;
private FileFolderService fileFolderService;
/**
* Sets the NodeService to use
*
* #param nodeService The NodeService
*/
public void setNodeService(NodeService nodeService)
{
this.nodeService = nodeService;
}
/**
* Sets the ContentService to use
*
* #param contentService The ContentService
*/
public void setContentService(ContentService contentService)
{
this.contentService = contentService;
}
public void setFileFolderService(FileFolderService fileFolderService) {
this.fileFolderService = fileFolderService;
}
public void setMimetypeService(MimetypeService mimetypeService) {
this.mimetypeService = mimetypeService;
}
private void extractFile(ZipFile archive,String mExtractToDir){
String fileName;
String destFileName;
byte[] buffer = new byte[16384];
mExtractToDir=mExtractToDir+File.separator;
try {
for ( Enumeration e = archive.getEntries(); e.hasMoreElements(); )
{
ZipEntry entry = (ZipEntry)e.nextElement();
if ( ! entry.isDirectory() )
{
fileName = entry.getName();
fileName = fileName.replace('/', File.separatorChar);
destFileName = mExtractToDir + fileName;
File destFile = new File(destFileName);
String parent = destFile.getParent();
if ( parent != null )
{
File parentFile = new File(parent);
if ( ! parentFile.exists() )
parentFile.mkdirs();
}
InputStream in = archive.getInputStream(entry);
OutputStream out = new FileOutputStream(destFileName);
int count;
while ( (count = in.read(buffer)) != -1 )
out.write(buffer, 0, count );
in.close();
out.close();
}else{
File newdir = new File( mExtractToDir + entry.getName() );
newdir.mkdir();
}
}
} catch (ZipException e) {
e.printStackTrace();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
private void deleteDir(File dir){
File elenco=new File(dir.getPath());
for (File file:elenco.listFiles()){
if (file.isFile())
file.delete();
else
deleteDir(file);
}
dir.delete();
}
public void executeImpl(Action ruleAction, NodeRef actionedUponNodeRef) {
String spaceType;
NodeRef parentNodeRef;
FileInfo fileInfo;
NodeRef nodeRef;
if (this.nodeService.exists(actionedUponNodeRef) == true) {
ContentReader reader = this.contentService.getReader(
actionedUponNodeRef, ContentModel.PROP_CONTENT);
if (reader != null) {
if (MimetypeMap.MIMETYPE_ZIP.equals(reader.getMimetype())) {
File zFile = null;
ZipFile zipFile = null;
try {
spaceType = ContentModel.TYPE_FOLDER.toString();
String dirName = ""+this.nodeService.getProperty(actionedUponNodeRef, ContentModel.PROP_NAME);
dirName=dirName.substring(0,dirName.indexOf(".zip"));
parentNodeRef = (NodeRef)ruleAction.getParameterValue(PARAM_DESTINATION_FOLDER);// this.nodeService.getRootNode(Repository.getStoreRef());
fileInfo = fileFolderService.create(
parentNodeRef,
dirName,
Repository.resolveToQName(spaceType));
nodeRef = fileInfo.getNodeRef();
zFile = TempFileProvider.createTempFile(
TEMP_FILE_PREFIX, TEMP_FILE_SUFFIX);
reader.getContent(zFile);
zipFile = new ZipFile(zFile, "Cp437");
File tempDir=new File(TempFileProvider.getTempDir().getPath()+File.separator+dirName);
if (tempDir.exists()) deleteDir(tempDir);
extractFile(zipFile,tempDir.getPath());
importFile(tempDir.getPath(),nodeRef);
deleteDir(tempDir);
} catch (ContentIOException e) {
e.printStackTrace();
} catch (InvalidNodeRefException e) {
e.printStackTrace();
} catch (FileExistsException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
}
private void importFile(String dir,NodeRef root){
File elenco=new File(dir);
for (File file:elenco.listFiles()){
try {
if (file.isFile()){
InputStream contentStream = new FileInputStream(file);
// assign name
String fileName=file.getName();
Map<QName, Serializable> contentProps = new HashMap<QName, Serializable>();
contentProps.put(ContentModel.PROP_NAME, fileName);
// create content node
ChildAssociationRef association = nodeService
.createNode(
root,
ContentModel.ASSOC_CONTAINS,
QName.createQName(
NamespaceService.CONTENT_MODEL_PREFIX,
fileName),
ContentModel.TYPE_CONTENT,
contentProps);
NodeRef content = association.getChildRef();
// add titled aspect (for Web Client display)
Map<QName, Serializable> titledProps = new HashMap<QName, Serializable>();
titledProps.put(ContentModel.PROP_TITLE, fileName);
titledProps.put(ContentModel.PROP_DESCRIPTION,fileName);
nodeService.addAspect(content,
ContentModel.ASPECT_TITLED,
titledProps);
ContentWriter writer = contentService.getWriter(content,
ContentModel.PROP_CONTENT, true);
writer.setMimetype(mimetypeService.guessMimetype(file.getAbsolutePath()));
writer.setEncoding("Cp437");
writer.putContent(contentStream);
}else{
String spaceType = ContentModel.TYPE_FOLDER.toString();
FileInfo fileInfo = fileFolderService.create(
root,
file.getName(),
Repository.resolveToQName(spaceType));
importFile(file.getPath(),fileInfo.getNodeRef());
}
} catch (InvalidTypeException e) {
e.printStackTrace();
} catch (InvalidAspectException e) {
e.printStackTrace();
} catch (InvalidQNameException e) {
e.printStackTrace();
} catch (ContentIOException e) {
e.printStackTrace();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (InvalidNodeRefException e) {
e.printStackTrace();
} catch (FileExistsException e) {
e.printStackTrace();
}
}
}
protected void addParameterDefinitions(List<ParameterDefinition> paramList)
{
paramList.add(new ParameterDefinitionImpl(PARAM_DESTINATION_FOLDER, DataTypeDefinition.NODE_REF,
true, getParamDisplayLabel(PARAM_DESTINATION_FOLDER)));
paramList.add(new ParameterDefinitionImpl(PARAM_ENCODING, DataTypeDefinition.TEXT,
true, getParamDisplayLabel(PARAM_ENCODING)));
}
}
Moreover the file importzip-action-messages.properties :
importzip.title=Import Zip File
importzip.description=This Action import zip file into alfresco
importzip.aspect-name.display-label=The name of the aspect to apply to the node.
and add the following line to …-context.xml, (you can use action-services-context.xml)
<bean id="importzip" class="org.alfresco.repo.action.executer.ZipActionExecuter" parent="action-executer">
<property name="nodeService">
<ref bean="NodeService"></ref>
</property>
<property name="contentService">
<ref bean="ContentService" />
</property>
<property name="mimetypeService">
<ref bean="MimetypeService"></ref>
</property>
<property name="fileFolderService">
<ref bean="FileFolderService"></ref>
</property>
</bean>
<bean id="importzip-messages" class="org.alfresco.i18n.ResourceBundleBootstrapComponent">
<property name="resourceBundles">
<list>
<value>org.alfresco.repo.action.importzip-action-messages</value>
</list>
</property>
</bean>
and finally add this line to my web-client-config-custom.xml
<config evaluator="string-compare" condition="Action Wizards">
<action-handlers>
<handler name="importzip" class="org.alfresco.web.bean.actions.handlers.ImportHandler" />
</action-handlers>
</config>
How would I programmatically unzip backed up data directly into a /data/data/com.appname folder from the SD Card?
I am able to directly unzip folders in the /data/data/com.appname folder using ES File Explorer, however I need to automate this via the app I'm developing.
I've tried the following code, Unfortunately I am only able to unzip to the SD Card and data folder of my app.
I'm guessing this is due to some form of app/folder security?
MainActivity.java
package fb.ziptester;
import android.provider.Settings;
import android.support.v7.app.AppCompatActivity;
import android.os.Bundle;
import android.view.View;
import android.widget.TextView;
public class MainActivity extends AppCompatActivity {
#Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
}
public void buttonOnClick(View v) {
TextView zipSource = (TextView)findViewById(R.id.zip_source);
TextView zipDest = (TextView)findViewById(R.id.zip_dest);
String sZipSource = "/storage/sdcard1/ACC/BU.zip";
String sZipDest = "/data/data/com.appname/";
zipSource.setText(sZipSource);
zipDest.setText(sZipDest);
UnzipUtility zipUtil = new UnzipUtility();
try {
zipUtil.unzip(sZipSource, sZipDest);
} catch(Exception ex) {
//TODO
}
}
}
package fb.ziptester;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
UnzipUtility.java
public class UnzipUtility {
/**
* Size of the buffer to read/write data
*/
private static final int BUFFER_SIZE = 4096;
/**
* Extracts a zip file specified by the zipFilePath to a directory specified by
* destDirectory (will be created if does not exists)
* #param zipFilePath
* #param destDirectory
* #throws IOException
*/
public void unzip(String zipFilePath, String destDirectory) throws IOException {
File destDir = new File(destDirectory);
if (!destDir.exists()) {
destDir.mkdir();
}
ZipInputStream zipIn = new ZipInputStream(new FileInputStream(zipFilePath));
ZipEntry entry = zipIn.getNextEntry();
// iterates over entries in the zip file
while (entry != null) {
String filePath = destDirectory + File.separator + entry.getName();
if (!entry.isDirectory()) {
// if the entry is a file, extracts it
extractFile(zipIn, filePath);
} else {
// if the entry is a directory, make the directory
File dir = new File(filePath);
dir.mkdir();
}
zipIn.closeEntry();
entry = zipIn.getNextEntry();
}
zipIn.close();
}
/**
* Extracts a zip entry (file entry)
* #param zipIn
* #param filePath
* #throws IOException
*/
private void extractFile(ZipInputStream zipIn, String filePath) throws IOException {
BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(filePath));
byte[] bytesIn = new byte[BUFFER_SIZE];
int read = 0;
while ((read = zipIn.read(bytesIn)) != -1) {
bos.write(bytesIn, 0, read);
}
bos.close();
}
}
Any help would be much appreciated.
Thank you.
I have one html file where I have kept all the URLs(Download link for CSV files).I want a tool/program that has to go through each url one by one and download the file, Then keep the file in the specified folder which will be written in the same html file itself.
html file is a table with 3 columns
File name,File location and download URL
Url will download the CSV file after opening a new window (target=_blank).Also after download it will close the child window automatically if there is no error.
I have tried the automation(Selenium using java)
But there are some challenges as follows.
It should wait until the download completes
Sometimes the url may show error,in that case it should close the child window and return to parent window
I have resolved the 1st case by keeping a watcher which will check whether the file is downloaded or not each second(by counting the number of csv files in the folder)
I can switch to child window and check whether there is any error but if there is no error my driver is got stuck over there.
How to resolve this
Code to check whether error is there in child window
public boolean foundError(FirefoxDriver driver) {
System.out.println(browser.getWindowHandle() + "Parent" + parentHandle);
String child = "";
int numberOfWindows = 0;
//return true;
if (driver.getWindowHandles().size() > 1) {
for (String winHandle : driver.getWindowHandles()) {
numberOfWindows++;
if (!parentHandle.equals(winHandle)) {
child = winHandle;
System.out.println("Child" + winHandle);
}
}
}
if (numberOfWindows > 1) {
System.out.println("tostring1" + driver.toString());
if (!parentHandle.equals(child)) {
driver.switchTo().window(child);
}
System.out.println("Switched to child");
Set set = driver.getWindowHandles();
System.out.println("Number of windows=" + set.size());
// System.out.println("Number of windows="+set.size()+"driver url"+driver.getCurrentUrl());
// System.out.println("tostring2"+driver.toString());
try {
// WebDriverWait wait1 = new WebDriverWait(driver, 5);
System.out.println("Body text" + driver.findElementByTagName("body").getText());/////////////////////////////Here driver will get stuck
//System.out.println("text"+driver.findElementByClassName("body").toString());
// List<WebElement> elements=driver.findElementsByClassName("ErrorBody");elements.size()>0
if (!driver.findElementByTagName("body").getText().equals("")) {
driver.close();
driver.switchTo().window(parentHandle);
return true;
}
System.out.println("No error");
driver.switchTo().window(parentHandle);
System.out.println("Switched to parent");
} catch (Exception e) {
System.out.println("Error Catch block page time out:" + e);
driver.switchTo().window(parentHandle);
return false;
// driver.switchTo().window(parentHandle);
}
}
return false;
}
I used different method
using Jsoup to parse the html file and downloading
import java.io.File;
import java.io.IOException;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
/**
*
* #author nudanesh
*/
public class URLDownload {
private Document doc;
String url = "", folder, file;
private final File sourceFile;
int i = 1;
int r = 1, c = 1;
int anchorCol = 3;
Library lib;
URLDownload() {
lib = new Library();
sourceFile = new File("Download.html");
try {
doc = Jsoup.parse(sourceFile, "UTF-8");
} catch (IOException ex) {
Logger.getLogger(URLDownload.class.getName()).log(Level.SEVERE, null, ex);
}
//Elements links = doc.select("a[href]");
Elements rows = doc.select("tr");
System.out.println("Size=" + rows.size());
for (Element row : rows) {
Elements cols = row.getElementsByTag("td");
c = 1;
for (Element col : cols) {
System.out.println("Row"+r);
if (c == 1) {
file = col.text();//System.out.println("File in main"+file);
} else if (c == 2) {
folder = col.text();//System.out.println("Folder in main"+folder);
} else {
try {
url = col.getElementsByTag("a").attr("href");
} catch (Exception e) {
System.out.print("-");
}
}
c++;
}
if (!url.equals("")) {
lib.setLocation(file,folder);
lib.downloadFile(url);
}
url = "";
i++;
r++;
}
}
public static void main(String arg[]) {
new URLDownload();
}
}
and following is the Library class file
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.nio.file.Files;
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.poi.xssf.usermodel.XSSFCell;
import org.apache.poi.xssf.usermodel.XSSFSheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
/**
*
* #author nudanesh
*/
public class Library {
boolean downloaded = false;
Thread t;
int waitTime = 0;
String baseLoc = "";
int size = 1024, ByteWritten = 0;
URL url;
URLConnection uCon = null;
String folderLoc = "", file = "firstFile.csv";
File loc;
private OutputStream outStream;
private InputStream is=null;
private byte[] buf;
private int ByteRead;
private int FolderInUrl = 4;
private boolean rootFolder = true;
private File resultFile;
private FileOutputStream fileResult;
private XSSFWorkbook workbookResult;
private XSSFSheet sheetResult;
private int updateExcelRowNum = -1;
private int updateExcelColNum = -1;
String date;
private int waitLimit = 900000;
Library() {
/*System.out.print(Calendar.getInstance().toString());
Date d=new Date();
String date=d.toString();
System.out.println(date);*/
//t = new Thread(this);
// t.start();
date = new SimpleDateFormat("yyyy_MM_dd_HH_mm_ss").format(Calendar.getInstance().getTime());
System.out.print(date);
baseLoc = date + "/";
WriteDataToExcel();
baseLoc += "Business Reports/";
createRowExcel(updateExcelRowNum);
updateRowColExcel(updateExcelRowNum, updateExcelColNum, "Report Name");
updateRowColExcel(updateExcelRowNum, updateExcelColNum, "Path");
updateRowColExcel(updateExcelRowNum, updateExcelColNum, "Status");
updateExcel();
}
public void setLocation(String a, String b) {
file = a;
file += ".csv";
folderLoc = baseLoc + getFolderPath(b);
// System.out.println("File Name: "+file);
// System.out.println("Folder loc: "+folderLoc);
}
public String getFolderPath(String b) {
String path = "";
try {
System.out.println("path" + b);
path = b;
// path = java.net.URLDecoder.decode(b, "UTF-8");
String p[] = path.split("/");
path = "";
for (int i = FolderInUrl; i < p.length - 1; i++) {
rootFolder = false;
p[i] = removeSpacesAtEnd(p[i]);
path = path + p[i] + "/";
}
} catch (Exception ex) {
Logger.getLogger(Library.class.getName()).log(Level.SEVERE, null, ex);
}
return path;
}
public void downloadFile(String urlString) {
// System.out.println("Started");
try {
url = new URL(urlString);
} catch (MalformedURLException ex) {
Logger.getLogger(Library.class.getName()).log(Level.SEVERE, null, ex);
}
try {
loc = new File(folderLoc);
if (!loc.exists()) {
loc.mkdirs();
}
outStream = new BufferedOutputStream(new FileOutputStream(folderLoc + file));
uCon = url.openConnection();
uCon.setReadTimeout(waitLimit);
is = uCon.getInputStream();
downloaded=true;
buf = new byte[size];
while ((ByteRead = is.read(buf)) != -1) {
System.out.println("while executing" + ByteRead);
outStream.write(buf, 0, ByteRead);
ByteWritten += ByteRead;
}
//System.out.println("Downloaded" + ByteWritten);
resetCounters();
createRowExcel(updateExcelRowNum);
updateRowColExcel(updateExcelRowNum, updateExcelColNum, file);
updateRowColExcel(updateExcelRowNum, updateExcelColNum, folderLoc);
if (ByteWritten < 1000) {
updateRowColExcel(updateExcelRowNum, updateExcelColNum, "Downloaded ");
} else {
updateRowColExcel(updateExcelRowNum, updateExcelColNum, "Downloaded ");
}
updateExcel();
} catch (Exception e) {
System.out.println("error catch" + e);
resetCounters();
createRowExcel(updateExcelRowNum);
updateRowColExcel(updateExcelRowNum, updateExcelColNum, file);
updateRowColExcel(updateExcelRowNum, updateExcelColNum, folderLoc);
updateRowColExcel(updateExcelRowNum, updateExcelColNum, "Rejected the Download after waiting " + (waitLimit / 60000) + " minutes");
updateExcel();
waitTime = 0;
} finally {
try {
System.out.println("Error in streams");
if(downloaded)
is.close();
outStream.close();
downloaded= false;
} catch (IOException e) {
e.printStackTrace();
}
}
}
public void moveToFolder(String reportName, String path) {
try {
File repo = new File(folderLoc + "/" + reportName + ".csv");
path = folderLoc + "/" + path;
File pathFolder = new File(path);
if (!pathFolder.exists()) {
pathFolder.mkdirs();
}
pathFolder = new File(path + reportName + ".csv");
System.out.println("Path=" + pathFolder.getAbsolutePath() + "\nReport path=" + repo.getAbsolutePath());
System.out.println("Source" + repo.getAbsolutePath());
//System.out.println("Status" + repo.renameTo(new File(pathFolder.getAbsolutePath())));
System.out.println("Status" + Files.move(repo.toPath(), new File(pathFolder.getAbsolutePath()).toPath(), REPLACE_EXISTING));
//Files.
} catch (Exception e) {
System.out.println("error while moving" + e);
}
}
public String changeSpecialCharacters(String report) {
report = report.replaceAll(":", "_");
return report;
}
public String removeSpacesAtEnd(String inputPath) {
for (int i = inputPath.length() - 1; i >= 0; i--) {
if (inputPath.charAt(i) != ' ') {
break;
} else {
System.out.println("Before string is" + inputPath);
inputPath = inputPath.substring(0, i);
System.out.println("AFter string is" + inputPath);
}
}
return inputPath;
}
public void WriteDataToExcel() {
try {
// file = new FileInputStream(new File("config.xlsx"));
// File resultFolder = new File("Results");
// if (resultFolder.exists()) {
// deleteDirectory(resultFolder);
// }
// resultFolder.mkdirs();
if (!new File(baseLoc).exists()) {
new File(baseLoc).mkdirs();
}
resultFile = new File(baseLoc + "Reports info " + date + ".xlsx");
System.out.println("Path" + resultFile.getAbsolutePath());
resultFile.createNewFile();
// rFilePath = resultFile.getAbsolutePath();
fileResult = new FileOutputStream(resultFile);
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
//Get the workbook instance for XLS file
// System.out.println("file success");
XSSFWorkbook workbook = null;
try {
workbookResult = new XSSFWorkbook();
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
System.out.println("Opening the browser");
//Get first sheet from the workbook
sheetResult = workbookResult.createSheet();
//sheetResult.set
//Get iterator to all the rows in current sheet
//Get iterator to all cells of current row
//ar.add(folderLocation);
// ar.add(firefoxProfileLocation);
}
public void updateExcel() {
try {
//fileResult.close();
fileResult = new FileOutputStream(resultFile);
workbookResult.write(fileResult);
fileResult.close();
} catch (Exception e) {
System.out.println(e);
}
}
public void createRowExcel(int num) {
updateExcelRowNum++;
num = updateExcelRowNum;
sheetResult.createRow(num);
}
public void updateRowColExcel(int rnum, int cnum, String value) {
updateExcelColNum++;
cnum = updateExcelColNum;
sheetResult.getRow(rnum).createCell(cnum);
XSSFCell cell = sheetResult.getRow(rnum).getCell(cnum);
cell.setCellValue(value);
}
public void updateColumn(int rnum, int cnum, String value) {
XSSFCell cell = sheetResult.getRow(rnum).getCell(cnum);
cell.setCellValue(value);
}
public void resetCounters() {
updateExcelColNum = -1;
}
/* #Override
public void run() {
while (true) {
if (true) {
waitTime += 1000;
System.out.println(waitTime);
if (waitTime > waitLimit) {
try {
is.close();
outStream.close();
//downloaded=false;
// cancelDownload=true;
} catch (Exception ex) {
Logger.getLogger(Library.class.getName()).log(Level.SEVERE, null, ex);
}
}
}
try {
Thread.sleep(1000);
} catch (Exception e) {
}
}
}*/
}
I need to unzip a zipped directory containing different files' format like .txt, .xml, .xls etc.
I am able to unzip if the directory contains only .txt files but it fails with other files format. Below is the program that I am using and after a bit of googling, all I saw was similar approach -
import java.io.*;
import java.util.Enumeration;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
public class ZipUtils {
public static void extractFile(InputStream inStream, OutputStream outStream) throws IOException {
byte[] buf = new byte[1024];
int l;
while ((l = inStream.read(buf)) >= 0) {
outStream.write(buf, 0, l);
}
inStream.close();
outStream.close();
}
public static void main(String[] args) {
Enumeration enumEntries;
ZipFile zip;
try {
zip = new ZipFile("myzip.zip");
enumEntries = zip.entries();
while (enumEntries.hasMoreElements()) {
ZipEntry zipentry = (ZipEntry) enumEntries.nextElement();
if (zipentry.isDirectory()) {
System.out.println("Name of Extract directory : " + zipentry.getName());
(new File(zipentry.getName())).mkdir();
continue;
}
System.out.println("Name of Extract fille : " + zipentry.getName());
extractFile(zip.getInputStream(zipentry), new FileOutputStream(zipentry.getName()));
}
zip.close();
} catch (IOException ioe) {
System.out.println("There is an IoException Occured :" + ioe);
ioe.printStackTrace();
}
}
}
Throws the below exception -
There is an IoException Occured :java.io.FileNotFoundException: myzip\abc.xml (The system cannot find the path specified)
java.io.FileNotFoundException: myzip\abc.xml (The system cannot find the path specified)
at java.io.FileOutputStream.open(Native Method)
at java.io.FileOutputStream.<init>(FileOutputStream.java:212)
at java.io.FileOutputStream.<init>(FileOutputStream.java:104)
at updaterunresults.ZipUtils.main(ZipUtils.java:43)
When you try to open the file that is going to contain the extracted content, the error occurs.
This is because the myzip folder is not available.
So check if it indeed is not available and create it before extracting the zip:
File outputDirectory = new File("myzip");
if(!outputDirectory.exists()){
outputDirectory.mkdir();
}
As #Perception pointed out in the comments: The output location is relative to the active/working directory. This is probably not very convenient, so you might want to add the extraction location to the location of the extracted files:
File outputLocation = new File(outputDirectory, zipentry.getName());
extractFile(zip.getInputStream(zipentry), new FileOutputStream(outputLocation));
(of course you need also add outputLocation to the directory creation code)
This is a good example in which he showed to unzip all the formats (pdf, txt etc) have look its quite
or you can use this code might work (i haven't tried this)
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
public class ZipUtils
{
private static final int BUFFER_SIZE = 4096;
private static void extractFile(ZipInputStream in, File outdir, String name) throws IOException
{
byte[] buffer = new byte[BUFFER_SIZE];
BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(new File(outdir,name)));
int count = -1;
while ((count = in.read(buffer)) != -1)
out.write(buffer, 0, count);
out.close();
}
private static void mkdirs(File outdir,String path)
{
File d = new File(outdir, path);
if( !d.exists() )
d.mkdirs();
}
private static String dirpart(String name)
{
int s = name.lastIndexOf( File.separatorChar );
return s == -1 ? null : name.substring( 0, s );
}
/***
* Extract zipfile to outdir with complete directory structure
* #param zipfile Input .zip file
* #param outdir Output directory
*/
public static void extract(File zipfile, File outdir)
{
try
{
ZipInputStream zin = new ZipInputStream(new FileInputStream(zipfile));
ZipEntry entry;
String name, dir;
while ((entry = zin.getNextEntry()) != null)
{
name = entry.getName();
if( entry.isDirectory() )
{
mkdirs(outdir,name);
continue;
}
/* this part is necessary because file entry can come before
* directory entry where is file located
* i.e.:
* /foo/foo.txt
* /foo/
*/
dir = dirpart(name);
if( dir != null )
mkdirs(outdir,dir);
extractFile(zin, outdir, name);
}
zin.close();
}
catch (IOException e)
{
e.printStackTrace();
}
}
}
Regards