Here My git, where I am maintaining the actual updates for hadoop compilation
github://sanjeevtripurari/hadoop-configs
## On Local Directory
cd Projects/hadoop/
my git config on local
cat .git/config
# [remote "origin"]
# url = ssh://git@github.com/sanjeevtripurari/hadoop.git
# [remote "upstream"]
# url = ssh://git@github.com/apache/hadoop.git
## Keeping the source updated
git pull upstream trunk
git push -u
### Development env setup
sudo add-apt-repository ppa:webupd8team/java
sudo apt-get update
sudo apt-get upgrade
sudo apt-get install build-essential git maven subversion
sudo apt-get install g++ autoconf automake libtool cmake zlib1g-dev pkg-config libssl-dev protobuf-compiler
### Java Install
## For oracle java auto install we need
echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections
sudo apt-get install oracle-java8-installer
## Set JAVA_HOME, and java8 as default
sudo apt-get install oracle-java8-set-default
sudo update-java-alternatives -s java-8-oracle
### We need Protobuf 2.5.0, required by hadoop source to compile
wget https://github.com/google/protobuf/archive/v2.5.0.tar.gz
tar -tvzf v2.5.0.tar.gz
cd protobuf-2.5.0/
./autogen.sh
./configure --prefix=/opt/protobuf-2.5.0
make
sudo make install
cd /opt/protobuf-2.5.0/
## Add to ldconfig, so hadoop compilation can pick the .so files
sudo ldconfig -v /opt/protobuf-2.5.0/lib |grep -i pro
## export PATH
export PATH=/opt/protobuf-2.5.0/bin:$PATH
## Here you go for hadoop compilation, skip doc, as java8 has issues parsing doc, but works with java7
mvn clean package -Pdist,native -Dmaven.javadoc.skip=true -DskipTests -Dtar
Check the target in
# target: hadoop-dist/target/
Pull my cluster config, if you want to do quick setup.
## For me i got the hadoop source compiled and this is the o/p it
main:
[exec] $ tar cf hadoop-3.0.0-SNAPSHOT.tar hadoop-3.0.0-SNAPSHOT
[exec] $ gzip -f hadoop-3.0.0-SNAPSHOT.tar
[exec]
[exec] Hadoop dist tar available at: /home/sanjeevt/Projects/hadoop/hadoop-dist/target/hadoop-3.0.0-SNAPSHOT.tar.gz
[exec]
[INFO] Executed tasks
[INFO]
[INFO] --- maven-javadoc-plugin:2.8.1:jar (module-javadocs) @ hadoop-dist ---
[INFO] Skipping javadoc generation
[INFO] ------------------------------------------------------------------------
[INFO] Reactor Summary:
[INFO]
[INFO] Apache Hadoop Main ................................ SUCCESS [4.008s]
[INFO] Apache Hadoop Build Tools ......................... SUCCESS [1.185s]
[INFO] Apache Hadoop Project POM ......................... SUCCESS [2.567s]
[INFO] Apache Hadoop Annotations ......................... SUCCESS [4.056s]
[INFO] Apache Hadoop Assemblies .......................... SUCCESS [1.178s]
[INFO] Apache Hadoop Project Dist POM .................... SUCCESS [6.116s]
[INFO] Apache Hadoop Maven Plugins ....................... SUCCESS [6.408s]
[INFO] Apache Hadoop MiniKDC ............................. SUCCESS [6.141s]
[INFO] Apache Hadoop Auth ................................ SUCCESS [14.335s]
[INFO] Apache Hadoop Auth Examples ....................... SUCCESS [7.447s]
[INFO] Apache Hadoop Common .............................. SUCCESS [8:23.207s]
[INFO] Apache Hadoop NFS ................................. SUCCESS [21.162s]
[INFO] Apache Hadoop KMS ................................. SUCCESS [57.151s]
[INFO] Apache Hadoop Common Project ...................... SUCCESS [0.573s]
[INFO] Apache Hadoop HDFS Client ......................... SUCCESS [3:50.178s]
[INFO] Apache Hadoop HDFS ................................ SUCCESS [9:44.831s]
[INFO] Apache Hadoop HttpFS .............................. SUCCESS [2:04.535s]
[INFO] Apache Hadoop HDFS BookKeeper Journal ............. SUCCESS [13.240s]
[INFO] Apache Hadoop HDFS-NFS ............................ SUCCESS [15.292s]
[INFO] Apache Hadoop HDFS Project ........................ SUCCESS [0.635s]
[INFO] Apache Hadoop YARN ................................ SUCCESS [0.539s]
[INFO] Apache Hadoop YARN API ............................ SUCCESS [1:41.401s]
[INFO] Apache Hadoop YARN Common ......................... SUCCESS [2:11.975s]
[INFO] Apache Hadoop YARN Server ......................... SUCCESS [0.768s]
[INFO] Apache Hadoop YARN Server Common .................. SUCCESS [34.561s]
[INFO] Apache Hadoop YARN NodeManager .................... SUCCESS [1:34.676s]
[INFO] Apache Hadoop YARN Web Proxy ...................... SUCCESS [11.593s]
[INFO] Apache Hadoop YARN ApplicationHistoryService ...... SUCCESS [25.655s]
[INFO] Apache Hadoop YARN ResourceManager ................ SUCCESS [1:50.939s]
[INFO] Apache Hadoop YARN Server Tests ................... SUCCESS [8.727s]
[INFO] Apache Hadoop YARN Client ......................... SUCCESS [17.878s]
[INFO] Apache Hadoop YARN SharedCacheManager ............. SUCCESS [10.647s]
[INFO] Apache Hadoop YARN Applications ................... SUCCESS [0.436s]
[INFO] Apache Hadoop YARN DistributedShell ............... SUCCESS [8.340s]
[INFO] Apache Hadoop YARN Unmanaged Am Launcher .......... SUCCESS [5.777s]
[INFO] Apache Hadoop YARN Site ........................... SUCCESS [0.515s]
[INFO] Apache Hadoop YARN Registry ....................... SUCCESS [19.661s]
[INFO] Apache Hadoop YARN Project ........................ SUCCESS [4:34.361s]
[INFO] Apache Hadoop MapReduce Client .................... SUCCESS [1.314s]
[INFO] Apache Hadoop MapReduce Core ...................... SUCCESS [1:23.951s]
[INFO] Apache Hadoop MapReduce Common .................... SUCCESS [34.491s]
[INFO] Apache Hadoop MapReduce Shuffle ................... SUCCESS [6.693s]
[INFO] Apache Hadoop MapReduce App ....................... SUCCESS [54.634s]
[INFO] Apache Hadoop MapReduce HistoryServer ............. SUCCESS [18.979s]
[INFO] Apache Hadoop MapReduce JobClient ................. SUCCESS [53.345s]
[INFO] Apache Hadoop MapReduce HistoryServer Plugins ..... SUCCESS [4.575s]
[INFO] Apache Hadoop MapReduce NativeTask ................ SUCCESS [2:53.471s]
[INFO] Apache Hadoop MapReduce Examples .................. SUCCESS [13.553s]
[INFO] Apache Hadoop MapReduce ........................... SUCCESS [2:31.103s]
[INFO] Apache Hadoop MapReduce Streaming ................. SUCCESS [15.238s]
[INFO] Apache Hadoop Distributed Copy .................... SUCCESS [22.616s]
[INFO] Apache Hadoop Archives ............................ SUCCESS [8.276s]
[INFO] Apache Hadoop Rumen ............................... SUCCESS [24.058s]
[INFO] Apache Hadoop Gridmix ............................. SUCCESS [17.028s]
[INFO] Apache Hadoop Data Join ........................... SUCCESS [7.206s]
[INFO] Apache Hadoop Ant Tasks ........................... SUCCESS [3.241s]
[INFO] Apache Hadoop Extras .............................. SUCCESS [6.438s]
[INFO] Apache Hadoop Pipes ............................... SUCCESS [12.511s]
[INFO] Apache Hadoop OpenStack support ................... SUCCESS [10.768s]
[INFO] Apache Hadoop Amazon Web Services support ......... SUCCESS [8.633s]
[INFO] Apache Hadoop Azure support ....................... SUCCESS [13.133s]
[INFO] Apache Hadoop Client .............................. SUCCESS [15.168s]
[INFO] Apache Hadoop Mini-Cluster ........................ SUCCESS [0.840s]
[INFO] Apache Hadoop Scheduler Load Simulator ............ SUCCESS [16.242s]
[INFO] Apache Hadoop Tools Dist .......................... SUCCESS [35.833s]
[INFO] Apache Hadoop Tools ............................... SUCCESS [0.465s]
[INFO] Apache Hadoop Distribution ........................ SUCCESS [1:01.293s]
[INFO] ------------------------------------------------------------------------
[INFO] BUILD SUCCESS
[INFO] ------------------------------------------------------------------------
[INFO] Total time: 55:44.365s
[INFO] Finished at: Tue Sep 01 07:29:57 UTC 2015
[INFO] Final Memory: 139M/468M
[INFO] ------------------------------------------------------------------------
No comments:
Post a Comment