Java实现DirectIo文件方式操作文件系统
场景
- APIServer接口要求较高的并发，而且还要将数据文件存储到本地备份，在低消耗内存CPU情况下提高程序的运行速度以及稳定性。
- 相关文章
http://man7.org/linux/man-pages/man2/open.2.html
- 实现思路 调用Linux本身的接口 Java采用JNA实现
具体方法调用
1private native int open(String pathname, int flags, int mode);
2private native int fcntl(int id, int cmd, int flags);
3private native int write(int fd, Pointer buf, int count);
4private native int posix_memalign(PointerByReference memptr, int alignment, int size);
5private native int close(int fd);
核心代码
1
2package com.huadi.utils.directio.io;
3
4import java.io.IOException;
5import java.io.OutputStream;
6
7import com.sun.jna.Native;
8import com.sun.jna.Pointer;
9import com.sun.jna.ptr.PointerByReference;
10
11public class JNAOutputStream extends OutputStream {
12
13//just a couple of them are used, here for eventual future uses
14protected static final int O_RDONLY = 00;
15protected static final int O_WRONLY = 01;
16protected static final int O_RDWR = 02;
17protected static final int O_CREAT = 0100;
18protected static final int O_EXCL = 0200;
19protected static final int O_NOCTTY = 0400;
20protected static final int O_TRUNC = 01000;
21protected static final int O_APPEND = 02000;
22protected static final int O_NONBLOCK = 04000;
23protected static final int O_NDELAY = O_NONBLOCK;
24protected static final int O_SYNC = 010000;
25protected static final int O_ASYNC = 020000;
26protected static final int O_DIRECT = 040000;
27protected static final int O_DIRECTORY = 0200000;
28protected static final int O_NOFOLLOW = 0400000;
29protected static final int O_NOATIME = 01000000;
30protected static final int O_CLOEXEC = 02000000;
31
32//fcntl has also other flags, not presented here
33protected static final int F_SETFL = 4;
34
35public static final int BLOCK_SIZE = 512;
36
37static {
38Native.register("c");
39}
40
41private native int open(String pathname, int flags, int mode);
42private native int fcntl(int id, int cmd, int flags);
43private native int write(int fd, Pointer buf, int count);
44private native int posix_memalign(PointerByReference memptr, int alignment, int size);
45private native int close(int fd);
46
47private int fd;
48private Pointer bufPnt;
49private byte[] buffer;
50private int pos;
51
52public JNAOutputStream(String pathname, boolean direct) throws IOException {
53
54if (!direct) {
55fd = open(pathname, O_CREAT|O_TRUNC|O_WRONLY, 00755);
56} else {
57// fd = open(pathname, O_CREAT|O_TRUNC|O_WRONLY|O_DIRECT, 00755);
58fd = open(pathname, O_CREAT|O_APPEND|O_WRONLY|O_DIRECT, 00755);
59}
60
61PointerByReference pntByRef = new PointerByReference();
62posix_memalign(pntByRef, BLOCK_SIZE, BLOCK_SIZE);
63bufPnt = pntByRef.getValue();
64buffer = new byte[BLOCK_SIZE];
65}
66
67@Override
68public void close() throws IOException {
69
70if (pos % BLOCK_SIZE != 0) {
71if (fcntl(fd, F_SETFL, ~O_DIRECT) < 0) {
72throw new IOException("Impossible to modify fd on close()");
73}
74bufPnt.write(0, buffer, 0, pos);
75int rtn = write(fd,bufPnt,pos);
76if (rtn != pos) {
77throw new IOException(rtn+"/"+pos+" bytes written to disk on close()");
78}
79}
80if (close(fd)< 0) {
81throw new IOException("Problems occured while doing close()");
82}
83}
84
85@Override
86public void write(int b) throws IOException {
87if (pos < BLOCK_SIZE) {
88buffer[pos++] = (byte)b;
89}
90if (pos == BLOCK_SIZE) {
91bufPnt.write(0, buffer, 0, BLOCK_SIZE);
92int rtn = write(fd, bufPnt, BLOCK_SIZE);
93if (rtn != BLOCK_SIZE) {
94throw new IOException(rtn+"/"+BLOCK_SIZE+" bytes written to disk");
95}
96pos = 0;
97}
98}
99
100}
101
使用方式
1
2package com.huadi.utils.directio.io;
3
4import java.io.*;
5import java.util.Random;
6
7
8public class Run {
9
10/**
11* @ower 张建新
12* @param args
13* @throws IOException
14*/
15private static final int BLOCK_SIZE = 4 * 1024;
16public static void main(String[] args) throws IOException {
17Random rand = new Random(System.nanoTime());
18int runs = Integer.parseInt(args[0]);
19int integers = Integer.parseInt(args[1]);
20long[] sio = new long[2];
21long[] jnaio = new long[2];
22long[] djnaio = new long[2];
23int[] data = new int[integers];
24for (int i = 0; i < integers; i++) {
25data[i] = rand.nextInt();
26}
27for (int i = 0; i < runs; i++) {
28runStd(data, sio);
29runJNA(data, jnaio, false);
30runJNA(data, djnaio, true);
31}
32System.out.println("JNA I/O (ns): "+jnaio[1]/(double)runs+"/"+jnaio[0]/(double)runs);
33System.out.println("O_DIRECT JNA I/O (ns): "+djnaio[1]/(double)runs+"/"+djnaio[0]/(double)runs);
34System.out.println("Standard I/O (ns): "+sio[1]/(double)runs+"/"+sio[0]/(double)runs);
35}
36
37public static void writeLineToFile2(byte[] data,String filename) throws IOException {
38DataOutputStream out = new DataOutputStream(new JNAOutputStream(filename, true));
39write(out, data);
40out.close();
41}
42
43
44public static void runStd(int data[], long time[]) throws IOException {
45String filename = "/tmp/"+System.nanoTime()+".bin";
46DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(filename), 512));
47time[0] += write(out, data);
48out.close();
49DataInputStream in = new DataInputStream(new BufferedInputStream(new FileInputStream(filename), 512));
50time[1] += read(in, data);
51in.close();
52}
53
54public static void runJNA(int data[], long time[], boolean o_direct) throws IOException {
55String filename = "/tmp/"+System.nanoTime()+".bin";
56DataOutputStream out = new DataOutputStream(new JNAOutputStream(filename, o_direct));
57time[0] += write(out, data);
58out.close();
59DataInputStream in = new DataInputStream(new JNAInputStream(filename, o_direct));
60time[1] += read(in, data);
61in.close();
62}
63
64public static long write(DataOutputStream out, int[] data) throws IOException {
65long t = System.nanoTime();
66for (int i = 0; i < data.length; i++ ) {
67out.writeInt(data[i]);
68}
69out.flush();
70return System.nanoTime() - t;
71}
72
73public static long write(DataOutputStream out, byte[] data) throws IOException {
74long t = System.nanoTime();
75for (int i = 0; i < data.length; i++ ) {
76out.write(data[i]);
77}
78out.flush();
79return System.nanoTime() - t;
80}
81
82
83public static long read(DataInputStream in, int[] data) throws IOException {
84long t = System.nanoTime();
85for (int i = 0; i < data.length; i++) {
86if (data[i] != in.readInt()) {
87throw new IOException("wrong!");
88}
89}
90return System.nanoTime() - t;
91}
92
93}
94