HDFS: How to Operate API (Example)

package test;
import org.apache.hadoop.fs.*;
import org.apache.commons.io.IOUtils;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.text.SimpleDateFormat;

import org.apache.hadoop.conf.Configuration;

public class Test extends FSDataInputStream {
    private static Configuration conf ;
    static{
        URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());
    }
    public static void Config(){
        conf= new Configuration();
        //conf.set("fs.defaultFS","hdfs://hadoop102:8020");
        conf.set("fs.hdfs.impl","org.apache.hadoop.hdfs.DistributedFileSystem");
    }

    public static int ReadLine(String path) throws IOException {
        URI uri= URI.create("hdfs://hadoop102:8020/user/atguigu/txt2");
        FileSystem fs = FileSystem.get(uri,conf);
        Path file = new Path(uri);
        FSDataInputStream getIt = fs.open(file);
        BufferedReader d = new BufferedReader(new InputStreamReader(getIt));

        String content;// = d.readLine(); 
        if((content=d.readLine())!=null){
            System.out.println(content);
        }
        //  System.out.println(content);
        d.close(); 
        fs.close(); 
        return 0;
    }

    public static void PrintFile() throws MalformedURLException, IOException{
        String FilePath="hdfs://hadoop102:8020/user/atguigu/txt2";
        InputStream in=null;
        in=new URL(FilePath).openStream();
        IOUtils.copy(in,System.out);

    }
    public static void lsDir(Configuration conf,String remoteDir){
        URI uri= URI.create("hdfs://hadoop102:8020/user");
        try(FileSystem fs=FileSystem.get(uri,conf)){
            Path dirPath=new Path(uri);
            RemoteIterator<LocatedFileStatus>remoteIterator=fs.listFiles(
                    dirPath,true);
            while(remoteIterator.hasNext()){
                FileStatus s= remoteIterator.next();
                System.out.printf("Path:"+s.getPath().toString());
                System.out.printf("Permissions:"+s.getPermission().toString());
                System.out.println("Size:"+s.getLen());
                Long timeStamp=s.getModificationTime();
                SimpleDateFormat format=new SimpleDateFormat("" +
                        "yyyy-MM-dd HH:mm:ss");
                String date=format.format(timeStamp);
                System.out.printf("Time:"+date);
                System.out.println();
            }
        }catch(IOException e) {
            e.printStackTrace();
        }
    }
    public static void createFile() throws IOException {
        URI uri= URI.create("hdfs://hadoop102:8020/user/atguigu/new");
        FileSystem fs = FileSystem.get(uri,conf);
        fs.mkdirs(new Path(uri));
        fs.close();
    }
    public static void putFile() throws IOException {
        URI uri= URI.create("hdfs://hadoop102:8020/user/atguigu/");
        FileSystem fs = FileSystem.get(uri,conf);
        fs.copyFromLocalFile(false,false,new Path("F:\\file2.txt"),new Path(uri));
        fs.close();
    }
    public static void downloadFile() throws IOException {
        URI uri= URI.create("hdfs://hadoop102:8020/user/atguigu/txt2/");
        FileSystem fs = FileSystem.get(uri,conf);
        fs.copyToLocalFile(false,new Path(uri),new Path("F:\\txt2.txt"),true);
        fs.close();
    }
    public static void deleteFile() throws IOException {
        URI uri= URI.create("hdfs://hadoop102:8020/user/atguigu/file2.txt/");
        FileSystem fs = FileSystem.get(uri,conf);
        fs.delete(new Path(uri),false);
        fs.close();
    }
    public static void main(String[] arg) throws IOException{
        Test.Config();
//        Test.ReadLine("/user/hadoop/txt2");
//        Test.PrintFile();
//        lsDir(conf,"/");
//        putFile();
//        downloadFile();
//        deleteFile();
        createFile();
    }
    public Test(InputStream in) {
        super(in);
    }
}

Similar Posts: