Develop Reference

How to run Hadoop HDFS command from java code

I'm new in Hadoop! How can I run some hdfs commands from Java code? I've been testing successfully mapreduce with java code and hdfs commands directly from cloudera vm's terminal but now I'd like to learn how to do it with java code.
I've been looking for any materials where to learn but I haven't found yet.
Thanks

I think this may be help to you
I use it execute shell command well .here is the java example
public class JavaRunShell {
public static void main(String[] args){
try {
String shpath=" your command";
Process ps = Runtime.getRuntime().exec(shpath);
ps.waitFor();
}
catch (Exception e) {
e.printStackTrace();
}
}
}

As mentioned by Jagrut, you can use FileSystem API in your java code to interact with hdfs command. Below is the sample code where i am trying to check if a particular directory exists in hdfs or not. If exists, then remove that hdfs directory.
Configuration conf = new Configuration();
Job job = new Job(conf,"HDFS Connect");
FileSystem fs = FileSystem.get(conf);
Path outputPath = new Path("/user/cloudera/hdfsPath");
if(fs.exists(outputPath))
fs.delete(outputPath);
You can also refer to given blogs for further reference -
https://dzone.com/articles/working-with-the-hadoop-file-system-api, https://hadoop.apache.org/docs/r2.8.2/api/org/apache/hadoop/fs/FileSystem.html
https://blog.knoldus.com/2017/04/16/working-with-hadoop-filesystem-api/

You can use the FileSystem API in your Java code to interact with HDFS.

You can use FileSystem API in java code to perform Hdfs commands.
https://hadoop.apache.org/docs/r2.8.2/api/org/apache/hadoop/fs/FileSystem.html
Please find the following sample code.
package com.hadoop.FilesystemClasses;
import java.io.IOException;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.Logger;
import com.hadoop.Constants.Constants;
public class HdfsFileSystemTasks {
public static Logger logger = Logger.getLogger(HdfsFileSystemTasks.class
.getName());
public FileSystem configureFilesystem(String coreSitePath,
String hdfsSitePath) {
FileSystem fileSystem = null;
try {
Configuration conf = new Configuration();
Path hdfsCoreSitePath = new Path(coreSitePath);
Path hdfsHDFSSitePath = new Path(hdfsSitePath);
conf.addResource(hdfsCoreSitePath);
conf.addResource(hdfsHDFSSitePath);
fileSystem = FileSystem.get(conf);
return fileSystem;
} catch (Exception ex) {
ex.printStackTrace();
return fileSystem;
}
}
public String writeToHDFS(FileSystem fileSystem, String sourcePath,
String destinationPath) {
try {
Path inputPath = new Path(sourcePath);
Path outputPath = new Path(destinationPath);
fileSystem.copyFromLocalFile(inputPath, outputPath);
return Constants.SUCCESS;
} catch (IOException ex) {
ex.printStackTrace();
return Constants.FAILURE;
}
}
public String readFileFromHdfs(FileSystem fileSystem, String hdfsStorePath,
String localSystemPath) {
try {
Path hdfsPath = new Path(hdfsStorePath);
Path localPath = new Path(localSystemPath);
fileSystem.copyToLocalFile(hdfsPath, localPath);
return Constants.SUCCESS;
} catch (IOException ex) {
ex.printStackTrace();
return Constants.FAILURE;
}
}
public String deleteHdfsDirectory(FileSystem fileSystem,
String hdfsStorePath) {
try {
Path hdfsPath = new Path(hdfsStorePath);
if (fileSystem.exists(hdfsPath)) {
fileSystem.delete(hdfsPath);
logger.info("Directory{} Deleted Successfully "
+ hdfsPath);
} else {
logger.info("Input Directory{} does not Exists " + hdfsPath);
}
return Constants.SUCCESS;
} catch (Exception ex) {
System.out
.println("Some exception occurred while reading file from hdfs");
ex.printStackTrace();
return Constants.FAILURE;
}
}
public String deleteLocalDirectory(FileSystem fileSystem,
String localStorePath) {
try {
Path localPath = new Path(localStorePath);
if (fileSystem.exists(localPath)) {
fileSystem.delete(localPath);
logger.info("Input Directory{} Deleted Successfully "
+ localPath);
} else {
logger.info("Input Directory{} does not Exists " + localPath);
}
return Constants.SUCCESS;
} catch (Exception ex) {
System.out
.println("Some exception occurred while reading file from hdfs");
ex.printStackTrace();
return Constants.FAILURE;
}
}
public void closeFileSystem(FileSystem fileSystem) {
try {
fileSystem.close();
} catch (Exception ex) {
ex.printStackTrace();
System.out.println("Unable to close Hadoop filesystem : " + ex);
}
}
}
package com.hadoop.FileSystemTasks;
import com.hadoop.Constants.HDFSParameters;
import com.hadoop.Constants.HdfsFilesConstants;
import com.hadoop.Constants.LocalFilesConstants;
import com.hadoop.FilesystemClasses.HdfsFileSystemTasks;
import org.apache.hadoop.fs.FileSystem;
import org.apache.log4j.Logger;
public class ExecuteFileSystemTasks {
public static Logger logger = Logger.getLogger(ExecuteFileSystemTasks.class
.getName());
public static void main(String[] args) {
HdfsFileSystemTasks hdfsFileSystemTasks = new HdfsFileSystemTasks();
FileSystem fileSystem = hdfsFileSystemTasks.configureFilesystem(
HDFSParameters.CORE_SITE_XML_PATH,
HDFSParameters.HDFS_SITE_XML_PATH);
logger.info("File System Object {} " + fileSystem);
String fileWriteStatus = hdfsFileSystemTasks.writeToHDFS(fileSystem,
LocalFilesConstants.SALES_DATA_LOCAL_PATH,
HdfsFilesConstants.HDFS_SOURCE_DATA_PATH);
logger.info("File Write Status{} " + fileWriteStatus);
String filereadStatus = hdfsFileSystemTasks.readFileFromHdfs(
fileSystem, HdfsFilesConstants.HDFS_DESTINATION_DATA_PATH
+ "/MR_Job_Res2/part-r-00000",
LocalFilesConstants.MR_RESULTS_LOCALL_PATH);
logger.info("File Read Status{} " + filereadStatus);
String deleteDirStatus = hdfsFileSystemTasks.deleteHdfsDirectory(
fileSystem, HdfsFilesConstants.HDFS_DESTINATION_DATA_PATH
+ "/MR_Job_Res2");
hdfsFileSystemTasks.closeFileSystem(fileSystem);
}
}

#HbnKing I tried running your code but I kept getting errors. This is the error i got
java.io.IOException: Cannot run program "your": CreateProcess error=2, The system cannot
find the file specified
at java.lang.ProcessBuilder.start(Unknown Source)
at java.lang.Runtime.exec(Unknown Source)
at java.lang.Runtime.exec(Unknown Source)
at java.lang.Runtime.exec(Unknowenter code heren Source)
at jrs.main(jrs.java:5)

Can't delete an existing file in Java [duplicate]

This question already has answers here:
Java 'file.delete()' Is not Deleting Specified File
(8 answers)
Can't delete files in java?
(7 answers)
Closed 5 years ago.
I have this Java class
package com.cf.utils;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.UUID;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import com.cf.CoreFaction;
import com.cf.faction.Faction;
import net.minecraft.util.text.TextFormatting;
import net.minecraftforge.common.DimensionManager;
import net.minecraftforge.fml.common.FMLCommonHandler;
import net.minecraftforge.fml.relauncher.Side;
public class FactionUtils {
public static boolean saveFaction(Faction f) {
if (Utils.isServer()) {
JSONObject obj = new JSONObject();
obj.put("Name", f.getName());
obj.put("Owner", f.getOwner().toString());
JSONArray members = new JSONArray();
for (UUID u : f.getMembers()) {
members.add(u.toString());
}
obj.put("Members", members);
File dir = new File(DimensionManager.getCurrentSaveRootDirectory() + "/factions");
if (!dir.exists())
dir.mkdirs();
try (FileWriter file = new FileWriter(
DimensionManager.getCurrentSaveRootDirectory() + "/factions/" + f.getName() + ".json")) {
file.write(obj.toJSONString());
file.close();
return true;
} catch (IOException e) {
e.printStackTrace();
}
}
return false;
}
public static void deleteFaction(Faction f) {
if (Utils.isServer()) {
File file = new File(DimensionManager.getCurrentSaveRootDirectory() + "/factions/" + f.getName() + ".json");
if (file.exists()) {
if(file.delete()) { //THIS RETURNS FALSE
for (UUID p : f.getMembers()) {
Utils.sendMessage(p, Utils.getTranslation("faction.disbanded", TextFormatting.RED));
}
} else
Utils.sendMessage(f.getOwner(), Utils.getTranslation("faction.disband.error", TextFormatting.RED));
} else
System.out.println("Can't find the file");
}
}
public static Faction getFaction(UUID player) {
for (Faction f : getAllFactions()) {
if (f.getMembers().contains(player))
return f;
}
return null;
}
public static Faction getFaction(String name) {
for (Faction f : getAllFactions()) {
if (f.getName().equalsIgnoreCase(name))
return f;
}
return null;
}
public static ArrayList<Faction> getAllFactions() {
ArrayList<Faction> list = new ArrayList<Faction>();
if (Utils.isServer()) {
File dir = new File(DimensionManager.getCurrentSaveRootDirectory() + "/factions");
if (!dir.exists())
return list;
File[] factions = dir.listFiles();
for (File f : factions) {
JSONParser parser = new JSONParser();
try {
Object obj = parser.parse(new FileReader(f));
JSONObject jsonObject = (JSONObject) obj;
JSONArray members = (JSONArray) jsonObject.get("Members");
Faction faction = new Faction((String) jsonObject.get("Name"),
UUID.fromString((String) jsonObject.get("Owner")));
ArrayList<UUID> ids = new ArrayList<UUID>();
for (int i = 0; i < members.size(); i++) {
ids.add(UUID.fromString(members.get(i).toString()));
}
faction.setMembers(ids);
list.add(faction);
} catch (Exception e) {
e.printStackTrace();
}
}
}
return list;
}
}
When i call the method deleteFaction i want to delete a specific file, but despite the file.exists() returns true, the file.delete() returns false and i can't figure out why. So why the file i'm pointing cannot be deleted?

Be sure you have right permissions for that directory and everything beneath it. Here it depends in your OS, if it is Linux you can run chmod.
Also please make sure to see this it once helped me.

/**
* Deletes the file or directory denoted by this abstract pathname. If
* this pathname denotes a directory, then the directory must be empty in
* order to be deleted.
*
* #return <code>true</code> if and only if the file or directory is
* successfully deleted; <code>false</code> otherwise
*
* #throws SecurityException
* If a security manager exists and its <code>{#link
* java.lang.SecurityManager#checkDelete}</code> method denies
* delete access to the file
*/
are you sure that your process has access rights for removing? Try to catch SecurityException

It's always a good design practice to handle possible errors around any kind of i/o be local or over the network
Try to catch the following exceptions in delete():-
try {
Files.delete(file.toPath());
} catch (NoSuchFileException x) {
System.err.format("%s: no such" + " file or directory%n", file.getAbsolutePath());
} catch (DirectoryNotEmptyException x) {
System.err.format("%s not empty%n", file.getAbsolutePath());
} catch (IOException x) {
// File permission problems are caught here.
System.err.println(x);
}
This is from Java documentation

Is it possible to run a loop when a new file is created in a folder?

So I have to make a program in java that automatically runs in the background and looks for a new .dat file and when it sees the new .dat file it then runs a .bat file to load data into a database. So far I have a program that watches for new file creation, modification, and deletion. I also have a script that runs the .bat file and loads the data into the database now i just need to connect the two but I am not sure how to go about this, If someone could point me in the right direction I would greatly appreciate it.
Below is the code I have so far.
import static java.nio.file.LinkOption.NOFOLLOW_LINKS;
import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE;
import static java.nio.file.StandardWatchEventKinds.OVERFLOW;
import static java.nio.file.StandardWatchEventKinds.ENTRY_DELETE;
import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY;
import java.io.*;
import java.util.*;
import java.io.File;
import java.io.IOException;
import java.nio.file.FileSystem;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.WatchEvent;
import java.nio.file.WatchEvent.Kind;
import java.nio.file.WatchKey;
import java.nio.file.WatchService;
public class Order_Processing {
public static void watchDirectoryPath(Path path)
{
try {
Boolean isFolder = (Boolean) Files.getAttribute(path,
"basic:isDirectory", NOFOLLOW_LINKS);
if (!isFolder)
{
throw new IllegalArgumentException("Path: " + path
+ " is not a folder");
}
}
catch (IOException ioe)
{
ioe.printStackTrace();
}
System.out.println("Watching path: "+ path);
FileSystem fs = path.getFileSystem();
try (WatchService service = fs.newWatchService())
{
path.register(service, ENTRY_CREATE, ENTRY_MODIFY, ENTRY_DELETE);
WatchKey key = null;
while (true)
{
key = service.take();
Kind<?> kind = null;
for (WatchEvent<?> watchEvent : key.pollEvents())
{
kind = watchEvent.kind();
if (OVERFLOW == kind)
{
continue;
}
else if (ENTRY_CREATE == kind)
{
Path newPath = ((WatchEvent<Path>) watchEvent)
.context();
System.out.println("New Path Created: " + newPath);
}
else if (ENTRY_MODIFY == kind)
{
Path newPath = ((WatchEvent<Path>) watchEvent)
.context();
System.out.println("New path modified: "+ newPath);
}
else if (ENTRY_DELETE == kind)
{
Path newPath = ((WatchEvent<Path>) watchEvent)
.context();
System.out.println("New path deleted: "+ newPath);
}
}
if (!key.reset())
{
break;
}
}
}
catch (IOException ioe)
{
ioe.printStackTrace();
}
catch (InterruptedException ie)
{
ie.printStackTrace();
}
}
public static void main(String[] args)
throws FileNotFoundException
{
File dir = new File("C:\\Paradigm");
watchDirectoryPath(dir.toPath());
//below is the script that runs the .bat file and it works if by itself
//with out all the other watch code.
try {
String[] command = {"cmd.exe", "/C", "Start", "C:\\Try.bat"};
Process p = Runtime.getRuntime().exec(command);
}
catch (IOException ex) {
}
}
}

This doesn't work because you have a while (true). This makes sense because you are listening and want the to happen continuously; however, the bat call will never be executed because watchDirectory(...) will never terminate. To solve this, pull the rest of the main out into its own function like so
public static void executeBat() {
try {
String[] command = {"cmd.exe", "/C", "Start", "C:\\Try.bat"};
Process p = Runtime.getRuntime().exec(command);
}
catch (IOException ex) {
// You should do something with this.
// DON'T JUST IGNORE FAILURES
}
so that upon file creation, you can call that bat script
...
else if (ENTRY_CREATE == kind)
{
Path newPath = ((WatchEvent<Path>) watchEvent).context();
executeBat();
}
...

Failed to move file to another directory using renameTo

import java.io.File;
import org.apache.commons.io.FilenameUtils;
public class Tester {
public static void main(String[] args) {
String rootPath = "F:\\Java\\Java_Project";
File fRoot = new File(rootPath);
File[] fsSub = fRoot.listFiles();
for (File file : fsSub) {
if(file.isDirectory()) continue;
String fileNewPath = FilenameUtils.removeExtension(file.getPath()) + "\\" + file.getName();
File fNew = new File(fileNewPath);
try {
file.renameTo(fNew);
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
I am trying to move the file to another directory,for instance,if the File path is
"C:\out.txt"
than I want to move to
"C:\out\out.txt"
If i try to print the original File and the new original information, the work well,But they just can not move successful.

I suggest to try Java 7 NIO2
Files.move(Path source, Path target, CopyOption... options)

URLClassLoader does not read external JAR library of a plugin

I have implemented a simple plugin based application with Java. Main plugin class is derived from and abstract class called "Plugin". The application reads that class from JAR file and runs the plugin by creating an instance of the class. Standard procedure I guess :)
Everything forks fine until now. But the problem occurs when I include a library to my plugin, like MySQL Connector. The exception NoClassDefFoundError and ClassNotFoundException are thrown after execution. I am overcoming the problem by adding MySQL connector library to the main application but what is the point then? :)
I am not a Java expert so I am not sure of any alternative solutions like defining a classpath for libraries etc.
Here is my plugin loader:
http://pastebin.com/90rQ9NfJ
And here is my plugin base class:
http://pastebin.com/Juuicwkm
I am executing from a GUI:
private void jButtonAddActionPerformed(java.awt.event.ActionEvent evt) {
JFileChooser fileChooser = new JFileChooser();
fileChooser.setFileFilter(new FileNameExtensionFilter("JTask Plugin (*.JAR)", "JAR"));
if (fileChooser.showOpenDialog(this) == JFileChooser.APPROVE_OPTION)
{
File pluginFile = fileChooser.getSelectedFile();
PluginLoader pluginLoader = new PluginLoader();
Plugin plugin = pluginLoader.loadPlugin(pluginFile);
if (plugin != null)
jPanelPlugins.add(new PluginControl(jPanelPlugins, plugin));
}
}

You should really include your source code as well.
How are you executing the class i.e. via command line or from a GUI? If from the command line, then the MySQLConnector libraries, along with any other dependent library must be included in the classpath (java -classpath). The top answer to this question should help you- Java: how to import a jar file from command line

if the case, your class is a Mysql Driver you have to exclude (at the time the class is calling) classes that are not available. In the Folder of your .jar file there is one with the name "integration" it contains "jboss" and "c3p0" which are not present at this time.
while (en.hasMoreElements()) {
JarEntry entry = new JarEntry(en.nextElement());
String name = entry.getName();
if (name.contains("/integration/")) {
continue;
} else {
if (!entry.isDirectory() && name.toLowerCase().endsWith(".class"))
{
classList.add(name.replace(".class", ""));
}
}
}
This should load a mysql.xxx.jar file.
Try This
dynamicload.java
package dynamicloading;
import java.io.File;
import java.net.URL;
import java.net.URLClassLoader;
import java.sql.Connection;
import java.sql.Statement;
import java.sql.Driver;
import java.sql.DriverManager;
import java.sql.DriverPropertyInfo;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Enumeration;
import java.util.Properties;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
/**
*
* #author Administrator
*/
class com_mysql_jdbc_Driver implements Driver {
private Driver driver;
com_mysql_jdbc_Driver(Driver cmjd) {
this.driver = cmjd;
}
#Override
public boolean acceptsURL(String aurlS) throws SQLException {
return this.driver.acceptsURL(aurlS);
}
#Override
public Connection connect(String aurlS, Properties pP) throws SQLException {
return this.driver.connect(aurlS, pP);
}
#Override
public int getMajorVersion() {
return this.driver.getMajorVersion();
}
#Override
public int getMinorVersion() {
return this.driver.getMinorVersion();
}
#Override
public DriverPropertyInfo[] getPropertyInfo(String aurlS, Properties pP) throws SQLException {
return this.driver.getPropertyInfo(aurlS, pP);
}
#Override
public boolean jdbcCompliant() {
return this.driver.jdbcCompliant();
}
}
public class DynMain {
/**
* #param args the command line arguments
*/
public static void main(String[] args) throws Exception {
/* please set to your path*/
File file = new File("U:/mozsamples/mysql-connector-java-5.1.19-bin.jar");
Driver cmjdD;
String aktCS;
String urlS = "jdbc:mysql://localhost/db";
String userS = "must-be-set";
String passS = "must-be-set";
Connection con;
Statement stmt;
URLClassLoader clazzLoader = URLClassLoader.newInstance(new URL[]{file.toURI().toURL()});
JarFile jarFile = new JarFile(file);
Enumeration<JarEntry> entries = jarFile.entries();
while (entries.hasMoreElements()) {
JarEntry element = entries.nextElement();
if (element.getName().endsWith(".class")) {
String name = element.getName();
if (name.contains("/integration/")) {
System.out.println( "ignored: " + name );
continue;
} else
{
try {
aktCS = element.getName().replaceAll(".class", "").replaceAll("/", ".");
clazzLoader.loadClass(aktCS);
if (name.contains("com/mysql/jdbc/Driver")) {
cmjdD = (Driver)Class.forName(aktCS, true, clazzLoader).newInstance();
try {
DriverManager.registerDriver(new com_mysql_jdbc_Driver(cmjdD));
System.out.println( "register Class: " + aktCS );
} catch (SQLException e) {
e.printStackTrace();
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
try {
con = DriverManager.getConnection(urlS,userS,passS);
stmt = con.createStatement();
/*ResultSet rs = stmt.executeQuery("select * from idcart where ID=255"); */
stmt.close();
} catch (SQLException esql) {
esql.printStackTrace();
}
int j=0 ;
System.out.println("loaded Driver----------------------------------");
for( Enumeration en = DriverManager.getDrivers() ; en.hasMoreElements() ; j++)
System.out.println( en.nextElement().getClass().getName() );
if (j==0) { System.out.println("Driverlist empty"); }
System.out.println("-----------------------------------------------");
}
}
Output:
register Class: com.mysql.jdbc.Driver
ignored: com/mysql/jdbc/integration/c3p0/MysqlConnectionTester.class
ignored: com/mysql/jdbc/integration/jboss/ExtendedMysqlExceptionSorter.class
ignored: com/mysql/jdbc/integration/jboss/MysqlValidConnectionChecker.class
loaded Driver----------------------------------
sun.jdbc.odbc.JdbcOdbcDriver
dynamicloading.com_mysql_jdbc_Driver
-----------------------------------------------
OK ???