InputStream in = null;
try {
in = new URL("hdfs://host/path").openStream();
//操作输入流in,可以读取到文件的内容
} finally {
IOUtils.closeStream(in);
}import java.io.InputStream;
import java.net.URL;
import org.apache.hadoop.fs.FsUrlStreamHandlerFactory;
import org.apache.hadoop.io.IOUtils;
public class URLCat {
static {
URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());
}
public static void main(String[] args) throws Exception {
InputStream in = null;
try {
in = new URL(args[0]).openStream();
IOUtils.copyBytes(in, System.out, 4096, false);
} finally {
IOUtils.closeStream(in);
}
}
}public static FileSystem get(Configuration conf) throws IOException; public static FileSystem get(URI uri, Configuration conf) throws IOException; public static FileSystem get(final URI uri, final Configuration conf, final String user) throws IOException, InterruptedException;
public static LocalFileSystem getLocal(Configuration conf) throws IOException;
public FSDataInputStream open(Path f) throws IOException; public abstract FSDataInputStream open(Path f, int bufferSize) throws IOException;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
public class FileSystemCat {
public static void main(String[] args) throws Exception {
String uri = args[0];
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(uri), conf);
//System.out.println(fs.getClass().getName()); //这里可以看到得到的实例是DistributedFileSystem,因为core-site.xml里配的是hdfs
FSDataInputStream in = null;
try {
in = fs.open(new Path(uri));
IOUtils.copyBytes(in, System.out, 4096, false);
} finally {
IOUtils.closeStream(in);
}
}
}in.seek(0); IOUtils.copyBytes(in, System.out, 4096, false);
public int read(long position, byte[] buffer, int offset, int length) throws IOException; public void readFully(long position, byte[] buffer, int offset, int length) throws IOException; public void readFully(long position, byte[] buffer) throws IOException;
public FSDataOutputStream create(Path f) throws IOException;
public FSDataOutputStream create(Path f, Progressable progress) throws IOException;
public interface Progressable {
public void progress();
}public FSDataOutputStream append(Path f) throws IOException;
import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
public class FileCopyWithProgress {
public static void main(String[] args) throws Exception {
String localSrc = args[0];
String dst = args[1];
InputStream in = new BufferedInputStream(new FileInputStream(localSrc));
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(dst), conf);
OutputStream out = fs.create(new Path(dst), new Progressable() {
@Override
public void progress() {
System.out.print(".");
// try {
// Thread.sleep(1000);
// } catch (Exception e) {
// e.printStackTrace();
// }
}
});
IOUtils.copyBytes(in, out, 4096, true);
System.out.println();
System.out.println("end.");
}
}public long getPos() throws IOException;
import static org.junit.Assert.*;
import static org.hamcrest.CoreMatchers.*;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
public class ShowFileStatusTest {
private static final String SYSPROP_KEY = "test.build.data";
/** MiniDFSCluster类在hadoop-hdfs-2.4.1-tests.jar中,是一个专门用于测试的in-process HDFS集群 */
private MiniDFSCluster cluster;
private FileSystem fs;
@Before
public void setUp() throws IOException {
Configuration conf = new Configuration();
String sysprop = System.getProperty(SYSPROP_KEY);
if (sysprop == null) {
System.setProperty(SYSPROP_KEY, "/tmp");
}
cluster = new MiniDFSCluster(conf, 1, true, null);
fs = cluster.getFileSystem();
OutputStream out = fs.create(new Path("/dir/file"));
out.write("content".getBytes("UTF-8"));
out.close();
}
@After
public void tearDown() throws IOException {
if (fs != null) {
fs.close();
}
if (cluster != null) {
cluster.shutdown();
}
}
@Test(expected = FileNotFoundException.class)
public void throwsFileNotFoundForNonExistentFile() throws IOException {
fs.getFileStatus(new Path("no-such-file"));
}
@Test
public void fileStatusForFile() throws IOException {
Path file = new Path("/dir/file");
FileStatus stat = fs.getFileStatus(file);
assertThat(stat.getPath().toUri().getPath(), is("/dir/file"));
assertThat(stat.isDirectory(), is(false));
assertThat(stat.getLen(), is(7L));
assertTrue(stat.getModificationTime() <= System.currentTimeMillis());
assertThat(stat.getReplication(), is((short)1));
assertThat(stat.getBlockSize(), is(64 * 1024 * 1024L));
assertThat(stat.getOwner(), is("norris"));
assertThat(stat.getGroup(), is("supergroup"));
assertThat(stat.getPermission().toString(), is("rw-r--r--"));
}
@Test
public void fileStatusForDirectory() throws IOException {
Path dir = new Path("/dir");
FileStatus stat = fs.getFileStatus(dir);
assertThat(stat.getPath().toUri().getPath(), is("/dir"));
assertThat(stat.isDirectory(), is(true));
assertThat(stat.getLen(), is(0L));
assertTrue(stat.getModificationTime() <= System.currentTimeMillis());
assertThat(stat.getReplication(), is((short)0));
assertThat(stat.getBlockSize(), is(0L));
assertThat(stat.getOwner(), is("norris"));
assertThat(stat.getGroup(), is("supergroup"));
assertThat(stat.getPermission().toString(), is("rwxr-xr-x"));
}
}原文:http://blog.csdn.net/norriszhang/article/details/39648857