Page History

...

libozfs is a JNI based C API for Ozone File System. It provides with read and write functionality on OzoneFileSystem. It also uses some functions from HDFS(Hadoop Distributed File System) for which it uses libhdfs, which is a JNI based C API for Hadoop’s Distributed File System (HDFS). It provides C APIs to a subset of the HDFS APIs to manipulate HDFS files and the filesystem. libhdfs is part of the Hadoop distribution and comes pre-compiled in $HADOOP_HDFS_HOME/lib/native/libhdfs.so .

The APIs

The libozfs APIs are a subset of the Ozone FileSystem APIs. The header file for libozfs describes each API in detail and is available in $HADOOP_HDFS_HOME/include/ozfs.h.

A Sample Program

#include "ozfs.h"
#include <stdio.h>
#include <stdlib.h>

int main(int argc, char **argv) {
ozfsFS fs;
 const char *rfile = argv[1];
 tSize bufferSize = strtoul(argv[3], NULL, 10);
ozfsFile readFile;
 char* buffer;
 tSize curSize;
if (argc != 4) {
 fprintf(stderr, "Usage: hdfs_read <filename> <filesize> <buffersize>\n");
 exit(-1);
 }
fs=ozfsConnect("host", 9862,"bucket1","vol1");
 if (!fs) {
fprintf(stderr, "Oops! Failed to connect to ozfs!\n");
 exit(-1);
 }

readFile = ozfsOpenFile(fs, rfile, O_RDONLY, bufferSize, 0, 0);
 if (!readFile) {
 fprintf(stderr, "Failed to open %s for writing!\n", rfile);
 exit(-2);
 }
buffer = malloc(sizeof(char) * bufferSize);
 if(buffer == NULL) {
 return -2;
 }
curSize = bufferSize;
 for (; curSize == bufferSize;) {
curSize = ozfsRead(fs, readFile, (void*)buffer, curSize);
 }
free(buffer);
ozfsCloseFile(fs, readFile);
ozfsDisconnect(fs);

 return 0;
}

Requirements:

• Hadoop with compiled libhdfs.so
• Linux kernel > 2.6.9 with fuse, which is the default or Fuse 2.7.x, 2.8.x installed. See: http://fuse.sourceforge.net/ or even easier: http://dag.wieers.com/rpm/packages/fuse
• libozfs.so compiled with Ozone

Compilation

...

           gcc -fPIC -pthread -I {OZONE_HOME}/ozone-native-client/libozone -I {HADOOP_HOME}/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include -g -c ozfs.c

...

               Execute the following command to compile libozfs_read.c and libozfs_write.c:

           gcc gcc -fPIC -pthread -I {OZONE_HOME}/ozone-native-client/libozone -I {HADOOP_HOME}/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include -g -c libozfs_read.c             

...

           gcc gcc -fPIC -pthread -I {OZONE_HOME}/ozone-native-client/libozone -I {HADOOP_HOME}/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include -g -c libozfs_write.c

      2. Generation of libozfs.so

           Execute the following command to generate a .so:

...

           gcc -L {HADOOP_HOME}/hadoop-hdfs-project/hadoop-hdfs-native-client/target/native/srctarget/mainusr/nativelocal/libhdfslib -o ozfs_read libozfs_read.o -lhdfs -pthread -L/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.242.b08-0.el7_7.x86_64/jre/lib/amd64/server 
           -ljvm -L{OZONE_HOME}/ozone-native-client/libozone -lozfs

           Execute the following command to execute ozfs_write:

           gcc -L/home {HADOOP_HOME}/hadoop-hdfs-project/hadoop-hdfs-native-client/target/native/srctarget/mainusr/nativelocal/libhdfslib -o ozfs_write libozfs_write.o -lhdfs -pthread -L/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.242.b08- 0.el7_7.x86_64/jre/lib/amd64/server -ljvm 
           -L{OZONE_HOME}/ozone-native-client/libozone -lozfs

...

           export LD_LIBRARY_PATH={HADOOP_HOME}/hadoop-hdfs-project/hadoop-hdfs-native-client/target/native/target/usr/local/lib:
                  /usr/lib/jvm/java-1.7.0-openjdk-1.7.0.221-2.6.18.0.el7_6.x86_64/jre/lib/amd64/server/:
                  /usr/lib/jvm/java-1.8.0-openjdk-1.8.0.242.b08-0.el7_7.x86_64/jre/lib/amd64/server:
                  {OZONE_HOME}/ozone-native-client/libozone

...

    <configuration>
    <properties>
     <property><name>ozone>ozone.scm.datanode.id.dir</name><value>/tmp/ozone/data</value></property>
     <property><name>ozone.replication</name><value>1</value></property>
     <property><name>ozone.metadata.dirs</name><value>/tmp/ozone/data/metadata</value></property>
     <property><name>ozone.scm.names</name><value>localhost</value></property>
     <property><name>ozone>ozone.om.address</name><value>localhost</value></property>
    </properties>
    </configuration>

Thread Safe      

     libdhfs is thread safe.

...