├── Chapter08 ├── Memcached-Java │ ├── ContactMemcachedClient.java │ └── ContactMemcachedExample.java ├── Memcached-Perl │ └── perl-example.pl ├── Memcached-Python │ ├── add-contacts.py │ ├── delete-contact.py │ └── fetch-contact.py ├── NDB-C++ │ └── example.c++ ├── NDB-Java │ ├── ClusterJ.java │ ├── Contact.java │ └── clusterj.properties └── NDB-NodeJS │ └── NodeJS.js ├── Chapter09 ├── INSTALL.txt └── hadoop_bigdata.sql ├── LICENSE └── README.md /Chapter08/Memcached-Java/ContactMemcachedClient.java: -------------------------------------------------------------------------------- 1 | import com.whalin.MemCached.MemCachedClient; 2 | import com.whalin.MemCached.SockIOPool; 3 | import java.util.Map; 4 | 5 | 6 | public class ContactMemcachedClient { 7 | 8 | private static String[] servers = {"localhost:11211"}; 9 | private static Integer[] weights = { 1 }; 10 | private MemCachedClient memcachedClient; 11 | public static final String NAMESPACE = "@@contacts_table"; 12 | 13 | /** 14 | * Initialize connection 15 | */ 16 | public ContactMemcachedClient() { 17 | 18 | SockIOPool pool = SockIOPool.getInstance(); 19 | pool.setServers(servers); 20 | pool.setWeights( weights ); 21 | if (!pool.isInitialized()) { 22 | pool.setInitConn(5); 23 | pool.setMinConn(5); 24 | pool.setMaxConn(250); 25 | pool.initialize(); 26 | } 27 | MemCachedClient memCachedClient = new MemCachedClient(); 28 | this.memcachedClient = memCachedClient; 29 | } 30 | 31 | /** 32 | * Add/Update value of specified key 33 | * @param key 34 | * @param value 35 | */ 36 | public void set(String key, Object value) { 37 | memcachedClient.set(key, value); 38 | } 39 | 40 | /** 41 | * Fetch value by key 42 | * @param key 43 | * @return 44 | */ 45 | public Object get(String key) { 46 | return memcachedClient.get(key); 47 | } 48 | 49 | /** 50 | * Delete value by key 51 | * @param key 52 | * @return 53 | */ 54 | public Object delete(String key) { 55 | return memcachedClient.delete(key); 56 | } 57 | 58 | /** 59 | * Fetch values as map from the multiple keys 60 | * @param keys 61 | * @return 62 | */ 63 | public Map getAllByKey(String[] keys) { 64 | return memcachedClient.getMulti(keys); 65 | } 66 | 67 | /** 68 | * Check weather value exists against key or not 69 | * @param key 70 | * @return 71 | */ 72 | public boolean isKeyExists(String key) { 73 | return memcachedClient.keyExists(key); 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /Chapter08/Memcached-Java/ContactMemcachedExample.java: -------------------------------------------------------------------------------- 1 | 2 | 3 | public class ContactMemcachedExample { 4 | 5 | public static void main(String[] args) { 6 | 7 | ContactMemcachedClient contactMemcachedClient = new ContactMemcachedClient(); 8 | 9 | // Store contact details 10 | String valueTobeStored = "jaydip,lakhatariya,jaydip@knowarth.com,9111111111,knowarth"; 11 | contactMemcachedClient.set(ContactMemcachedClient.NAMESPACE + "101",valueTobeStored); 12 | 13 | // Fetch contact detail 14 | contactMemcachedClient.get(ContactMemcachedClient.NAMESPACE + "101"); 15 | 16 | // Delete contact detail 17 | contactMemcachedClient.delete(ContactMemcachedClient.NAMESPACE + "101"); 18 | } 19 | } -------------------------------------------------------------------------------- /Chapter08/Memcached-Perl/perl-example.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | use Cache::Memcached; 3 | use Data::Dumper; 4 | 5 | # Configure the memcached server 6 | my $memcache = new Cache::Memcached { 7 | 'servers' => [ 'localhost:11211'] 8 | }; 9 | 10 | my $contactId=$ARGV[0]; 11 | my $firstName=$ARGV[1]; 12 | my $lastName=$ARGV[2]; 13 | my $emailAddress=$ARGV[3]; 14 | my $mobileNumber=$ARGV[4]; 15 | my $companyName=$ARGV[5]; 16 | 17 | #Store contact Information 18 | $contactData = $firstName . ",". $lastName . "," . $emailAddress . "," . $mobileNumber . "," . $companyName; 19 | $memcache->set($contactId,$contactData)) 20 | 21 | #Fetch contact information to check if it stored or not 22 | my $contact = $cache->get($contactId); 23 | if (defined($contact)) 24 | { 25 | print "$contact\n"; 26 | } 27 | else 28 | { 29 | print STDERR "Contact Information does not stored"; 30 | } -------------------------------------------------------------------------------- /Chapter08/Memcached-Python/add-contacts.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import memcache 3 | memcached = memcache.Client(['127.0.0.1:11211'], debug=1); 4 | 5 | # Insert Contact Detail 6 | def addContact(contactId, firstName, lastName, emailAddress, mobileNumber, companyName): 7 | 8 | valueTobeStore = firstName + "," 9 | valueTobeStore = lastName + "," 10 | valueTobeStore = emailAddress + "," 11 | valueTobeStore = mobileNumber + "," 12 | valueTobeStore = companyName 13 | 14 | memcached.set(contactId,valueTobeStore) 15 | print "Contact information has stored successfully" 16 | 17 | 18 | # Get contact information pass from the command line 19 | contactId= sys.argv[0] 20 | firstName= sys.argv[1] 21 | lastName= sys.argv[2] 22 | emailAddress = sys.argv[3] 23 | mobileNumber = sys.argv[4] 24 | companyName = sys.argv[5] 25 | 26 | # Call addContact method to store contact information 27 | addContact(contactId, firstName, lastName, emailAddress, mobileNumber, companyName) 28 | 29 | 30 | -------------------------------------------------------------------------------- /Chapter08/Memcached-Python/delete-contact.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import memcache 3 | memcached = memcache.Client(['127.0.0.1:11211'], debug=1); 4 | 5 | def deleteContact(contactId) 6 | memcached.delete(contactId) 7 | print "Contact information has been deleted successfully" 8 | 9 | 10 | # Get contact information pass from the command line 11 | contactId= sys.argv[0] 12 | deleteContact(contactId) -------------------------------------------------------------------------------- /Chapter08/Memcached-Python/fetch-contact.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import memcache 3 | memcached = memcache.Client(['127.0.0.1:11211'], debug=1); 4 | 5 | # Fetch contact information by id 6 | def getContactById(contactId): 7 | contact = memcached.get(contactId) 8 | print(contact) 9 | 10 | # Get contact id from the command line 11 | contactId= sys.argv[0] 12 | getContactById(contactId) -------------------------------------------------------------------------------- /Chapter08/NDB-C++/example.c++: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | static void run_application(Ndb_cluster_connection &); 6 | 7 | #define PRINT_ERROR(code,msg) 8 | std::cout << "Error occurred: " << code << ", Message: " << message << std::endl 9 | 10 | #define APIERROR(error) { 11 | PRINT_ERROR(error.code,error.message); 12 | exit(-1); 13 | } 14 | 15 | int main(int argc, char** argv) 16 | { 17 | // Initialize ndb 18 | ndb_init(); 19 | 20 | // Connect MySQL Cluster 21 | { 22 | const char *connection_string = "localhost:1186"; 23 | Ndb_cluster_connection cluster_connection(connection_string); 24 | 25 | // Connect to cluster management server (ndb_mgmd) 26 | if (cluster_connection.connect(4, 5, 1)) 27 | { 28 | std::cout << "A management server is not ready yet.\n"; 29 | exit(-1); 30 | } 31 | 32 | // Connects to the storage nodes (ndbd's) 33 | if (cluster_connection.wait_until_ready(30,0) < 0) 34 | { 35 | std::cout << "Data nodes are not ready yet.\n"; 36 | exit(-1); 37 | } 38 | 39 | // Run the application code 40 | run_application(cluster_connection); 41 | } 42 | 43 | ndb_end(0); 44 | 45 | return 0; 46 | } 47 | 48 | static void create_table(MYSQL &); 49 | static void do_insert(Ndb &); 50 | static void do_update(Ndb &); 51 | static void do_delete(Ndb &); 52 | static void do_read(Ndb &); 53 | 54 | static void run_application(Ndb_cluster_connection &cluster_connection) 55 | { 56 | // Connect to database via NDB API 57 | Ndb myNdb( &cluster_connection, "demodb" ); 58 | if (myNdb.init()) APIERROR(myNdb.getNdbError()); 59 | 60 | //Do different operations on database 61 | do_insert(myNdb); 62 | do_update(myNdb); 63 | do_delete(myNdb); 64 | do_read(myNdb); 65 | } 66 | 67 | static void do_insert(Ndb &myNdb) 68 | { 69 | const NdbDictionary::Dictionary* myDict= myNdb.getDictionary(); 70 | const NdbDictionary::Table *myTable= myDict->getTable("Contacts"); 71 | if (myTable == NULL) APIERROR(myDict->getNdbError()); 72 | 73 | NdbTransaction *myTransaction= myNdb.startTransaction(); 74 | if (myTransaction == NULL) APIERROR(myNdb.getNdbError()); 75 | 76 | NdbOperation *myOperation= myTransaction->getNdbOperation(myTable); 77 | if (myOperation == NULL) APIERROR(myTransaction->getNdbError()); 78 | 79 | myOperation->insertTuple(); 80 | myOperation->equal("id", 123); 81 | myOperation->setValue("firstName", "Jaydip"); 82 | myOperation->setValue("lastName", "Lakhatariya"); 83 | myOperation->setValue("emailAddress", "jaydip.lakh@gmail.com"); 84 | myOperation->setValue("mobileNumber", "9724371151"); 85 | myOperation->setValue("companyName", "Knowarth"); 86 | 87 | if (myTransaction->execute( NdbTransaction::Commit ) == -1) 88 | APIERROR(myTransaction->getNdbError()); 89 | 90 | myNdb.closeTransaction(myTransaction); 91 | } 92 | 93 | static void do_update(Ndb &myNdb) 94 | { 95 | const NdbDictionary::Dictionary* myDict= myNdb.getDictionary(); 96 | const NdbDictionary::Table *myTable= myDict->getTable("Contacts"); 97 | if (myTable == NULL) APIERROR(myDict->getNdbError()); 98 | 99 | NdbTransaction *myTransaction= myNdb.startTransaction(); 100 | if (myTransaction == NULL) APIERROR(myNdb.getNdbError()); 101 | 102 | NdbOperation *myOperation= myTransaction->getNdbOperation(myTable); 103 | if (myOperation == NULL) APIERROR(myTransaction->getNdbError()); 104 | 105 | myOperation->updateTuple(); 106 | myOperation->equal( "id", 123); 107 | myOperation->setValue("firstName", "Keyur"); 108 | myOperation->setValue("lastName", "Lakhatariya"); 109 | myOperation->setValue("emailAddress", "Keyur.lakh@gmail.com"); 110 | myOperation->setValue("mobileNumber", "9998887771"); 111 | myOperation->setValue("companyName", "Knowarth"); 112 | 113 | if( myTransaction->execute( NdbTransaction::Commit ) == -1 ) 114 | APIERROR(myTransaction->getNdbError()); 115 | 116 | myNdb.closeTransaction(myTransaction); 117 | } 118 | 119 | static void do_delete(Ndb &myNdb) 120 | { 121 | const NdbDictionary::Dictionary* myDict= myNdb.getDictionary(); 122 | const NdbDictionary::Table *myTable= myDict->getTable("Contacts"); 123 | 124 | if (myTable == NULL) 125 | APIERROR(myDict->getNdbError()); 126 | 127 | NdbTransaction *myTransaction= myNdb.startTransaction(); 128 | if (myTransaction == NULL) APIERROR(myNdb.getNdbError()); 129 | 130 | NdbOperation *myOperation= myTransaction->getNdbOperation(myTable); 131 | if (myOperation == NULL) APIERROR(myTransaction->getNdbError()); 132 | 133 | myOperation->deleteTuple(); 134 | myOperation->equal( "id", 123); 135 | 136 | if (myTransaction->execute(NdbTransaction::Commit) == -1) 137 | APIERROR(myTransaction->getNdbError()); 138 | 139 | myNdb.closeTransaction(myTransaction); 140 | } 141 | 142 | static void do_read(Ndb &myNdb) 143 | { 144 | const NdbDictionary::Dictionary* myDict= myNdb.getDictionary(); 145 | const NdbDictionary::Table *myTable= myDict->getTable("Contacts"); 146 | 147 | if (myTable == NULL) 148 | APIERROR(myDict->getNdbError()); 149 | 150 | NdbTransaction *myTransaction= myNdb.startTransaction(); 151 | if (myTransaction == NULL) APIERROR(myNdb.getNdbError()); 152 | 153 | NdbOperation *myOperation= myTransaction->getNdbOperation(myTable); 154 | if (myOperation == NULL) APIERROR(myTransaction->getNdbError()); 155 | 156 | myOperation->readTuple(NdbOperation::LM_Read); 157 | myOperation->equal("id", 123); 158 | 159 | NdbRecAttr *emailRecAttr= myOperation->getValue("emailAddress", NULL); 160 | NdbRecAttr *mobileRecAttr= myOperation->getValue("mobileNumber", NULL); 161 | 162 | if (emailRecAttr == NULL) APIERROR(myTransaction->getNdbError()); 163 | if (mobileRecAttr == NULL) APIERROR(myTransaction->getNdbError()); 164 | 165 | if(myTransaction->execute( NdbTransaction::Commit ) == -1) 166 | APIERROR(myTransaction->getNdbError()); 167 | 168 | printf("FirstName is: \n", emailRecAttr->aRef()); 169 | printf("Mobile Number is: \n", mobileRecAttr->aRef()); 170 | 171 | myNdb.closeTransaction(myTransaction); 172 | } -------------------------------------------------------------------------------- /Chapter08/NDB-Java/ClusterJ.java: -------------------------------------------------------------------------------- 1 | import com.mysql.clusterj.ClusterJHelper; 2 | import com.mysql.clusterj.SessionFactory; 3 | import com.mysql.clusterj.Session; 4 | import com.mysql.clusterj.Query; 5 | import com.mysql.clusterj.query.QueryBuilder; 6 | import com.mysql.clusterj.query.QueryDomainType; 7 | import java.io.File; 8 | import java.io.InputStream; 9 | import java.io.FileInputStream; 10 | import java.io.*; 11 | import java.util.Properties; 12 | import java.util.List; 13 | 14 | public class ClusterJ { 15 | 16 | private static Session session; 17 | 18 | public static void main (String[] args) { 19 | 20 | // Load the properties from the clusterj.properties file 21 | File propsFile = null; 22 | InputStream inStream = null; 23 | Properties props = null; 24 | props.load(inStream); 25 | try { 26 | propsFile = new File("clusterj.properties"); 27 | inStream = new FileInputStream(propsFile); 28 | props = new Properties(); 29 | props.load(inStream); 30 | } catch(Exception e) { 31 | System.err.println(e); 32 | return; 33 | } 34 | 35 | // Connect to database and create session 36 | SessionFactory factory = ClusterJHelper.getSessionFactory(props); 37 | session = factory.getSession(); 38 | if(session != null) { 39 | System.out.println("Connection Established Successfully"); 40 | } 41 | saveContact(123, "Jaydip", "Lakhatariya", "jaydip.lakh@gmail.com", "9724371151", "Knowarth"); 42 | findByContactId(123); 43 | deleteContactById(123); 44 | } 45 | } 46 | 47 | /* Save Contact Details */ 48 | private static void saveContact(int contactId, String firstName, String lastName, String emailAddress, String mobileNumber, String companyName) { 49 | 50 | // Create and initialise Contact information 51 | Contact newContact = session.newInstance(Contact.class); 52 | newContact.setId(contactId); 53 | newContact.setFirstName(firstName); 54 | newContact.setLastName(lastName); 55 | newContact.setEmailAddress(emailAddress); 56 | newContact.setMobileNumber(mobileNumber); 57 | newContact.setCompanyName(companyName); 58 | 59 | // Store contact information to the database 60 | session.persist(newContact); 61 | System.out.println("Contact information has been saved successfully"); 62 | } 63 | 64 | /* Fetch the contact detail by id */ 65 | private static void findByContactId(int contactId) { 66 | 67 | Contact contact = session.find(Contact.class, contactId); 68 | if(contact != null) { 69 | System.out.println("Contact found with name: " + contact.getFirstName + " " + contact.getLastName()); 70 | } else { 71 | System.out.println("No contact found with the ID " + contactId); 72 | } 73 | } 74 | 75 | /* Delete contact by id */ 76 | private static void deleteContactById(int contactId) { 77 | session.deletePersistent(Contact.class, contactId); 78 | System.out.prinln("Deleted contact detail of id: " + contactId); 79 | } -------------------------------------------------------------------------------- /Chapter08/NDB-Java/Contact.java: -------------------------------------------------------------------------------- 1 | import com.mysql.clusterj.annotation.Column; 2 | import com.mysql.clusterj.annotation.PersistenceCapable; 3 | import com.mysql.clusterj.annotation.PrimaryKey; 4 | 5 | @PersistenceCapable(table="Contacts") 6 | public interface Contact { 7 | 8 | @PrimaryKey 9 | public int getId(); 10 | public void setId(int id); 11 | 12 | @Column(name="firstName") 13 | public String getFirstName(); 14 | public String setFirstName(String firstName); 15 | 16 | @Column(name="lastName") 17 | public String getLastName(); 18 | public String setLastName(String lastName); 19 | 20 | @Column(name="emailAddress") 21 | public String getEmailAddress(); 22 | public String setEmailAddress(String emailAddress); 23 | 24 | @Column(name="mobileNumber") 25 | public String getMobileNumber(); 26 | public String setMobileNumber(String mobileNumber); 27 | 28 | @Column(name="companyName") 29 | public String getCompanyName(); 30 | public String setCompanyName(String companyName); 31 | } -------------------------------------------------------------------------------- /Chapter08/NDB-Java/clusterj.properties: -------------------------------------------------------------------------------- 1 | com.mysql.clusterj.connectstring=localhost:1186 2 | com.mysql.clusterj.database=demodb 3 | com.mysql.clusterj.connect.retries=4 4 | com.mysql.clusterj.connect.delay=5 5 | com.mysql.clusterj.connect.timeout.before=30 6 | com.mysql.clusterj.connect.timeout.after=20 7 | com.mysql.clusterj.max.transactions=1024 8 | -------------------------------------------------------------------------------- /Chapter08/NDB-NodeJS/NodeJS.js: -------------------------------------------------------------------------------- 1 | var nosql = require('mysql-js'); 2 | 3 | var Contact = function(contactId, firstName, lastName, emailAddress, mobileNumber, companyName) { 4 | if (contactId) this.contactId = contactId; 5 | if (firstName) this.firstName = firstName; 6 | if (lastName) this.lastName = lastName; 7 | if (emailAddress) this.emailAddress = emailAddress; 8 | if (mobileNumber) this.mobileNumber = mobileNumber; 9 | if (companyName) this.companyName = companyName; 10 | }; 11 | 12 | // Map class with the Table 13 | var annotations = new nosql.TableMapping('Contacts').applyToClass(Contact); 14 | 15 | // This function will be automatically called whenever connection successfully established. 16 | var onOpenSession = function(err, session) { 17 | 18 | if (err) { 19 | console.log('Error while opening session'); 20 | console.log(err); 21 | process.exit(0); 22 | } else { 23 | console.log('Connection established successfully'); 24 | 25 | // Store contact information 26 | var contact = new Contact(123, 'Jaydip', 'Lakhatariya', 'jaydip.lakh@gmail.com', '9111111111', 'Knowarth Tech'); 27 | session.persist(contact, onInsert, contact, session); 28 | } 29 | }; 30 | 31 | // This function will be automatically called whenever a new record has been inserted 32 | var onInsert = function(err, object, session) { 33 | 34 | if (err) { 35 | console.log("Error occurred while inserting new contact") 36 | console.log(err); 37 | } else { 38 | console.log('Contact Inserted Successfully: ' + JSON.stringify(object)); 39 | 40 | // Find the contact from the database 41 | session.find(Contact, 123, onFindContact); 42 | } 43 | }; 44 | 45 | var onFindContact = function(err, contact) { 46 | 47 | if (err) { 48 | console.log("Error while fetching contact") 49 | console.log(err); 50 | } else { 51 | console.log('Contact information is: ' + JSON.stringify(contact)); 52 | 53 | // Delete the respective contact 54 | session.remove(contact, onDeleteContact, contact.contactId); 55 | } 56 | }; 57 | 58 | // Call whenever deletion happens on record 59 | var onDeleteContact = function(err, object) { 60 | 61 | if (err) { 62 | console.log("Error while deleting contact"); 63 | console.log(err); 64 | } else { 65 | console.log('Successfully deleted contact: ' + JSON.stringify(object)); 66 | } 67 | process.exit(0); 68 | }; 69 | 70 | // Initialize the database properties 71 | var dbProperties = nosql.ConnectionProperties('ndb'); 72 | 73 | // Connect to the database 74 | nosql.openSession(dbProperties, Contact, onOpenSession); -------------------------------------------------------------------------------- /Chapter09/INSTALL.txt: -------------------------------------------------------------------------------- 1 | follow below steps to install and configure Java, Hadoop and Sqoop on the LINUX server. 2 | 3 | Step 1 - Download java from https://www.java.com/en/download/help/linux_x64_install.xml location 4 | 5 | Step 2 - unzip java and move to /usr/local/java location. 6 | 7 | Step 3 - Setup environment variables for java run time 8 | export JAVA_HOME=/usr/local/java 9 | export PATH=$PATH:$JAVA_HOME/bin 10 | 11 | 12 | Installing and Configuring Hadoop on the server 13 | Step 4 - Download Hadoop from below URL 14 | wget http://www-us.apache.org/dist/hadoop/common/hadoop-2.8.1/hadoop-2.8.1.tar.gz 15 | 16 | Step 5 - extract the Hadoop tar and rename it 17 | cd /usr/local 18 | tar -xvz http://www-us.apache.org/dist/hadoop/common/hadoop-2.8.1/hadoop-2.8.1.tar.gz 19 | mv hadoop-2.8.1 hadoop 20 | 21 | Step 6 - Set variables for the Hadoop 22 | export HADOOP_HOME=/usr/local/hadoop 23 | export HADOOP_MAPRED_HOME=$HADOOP_HOME 24 | export HADOOP_COMMON_HOME=$HADOOP_HOME 25 | export HADOOP_HDFS_HOME=$HADOOP_HOME 26 | export YARN_HOME=$HADOOP_HOME 27 | export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native 28 | export PATH=$PATH:$HADOOP_HOME/sbin:$HADOOP_HOME/bin 29 | 30 | Step 7 - Set Java Path inside hadoop's configuration 31 | cd $HADOOP_HOME/etc/hadoop 32 | export JAVA_HOME=/usr/local/java 33 | 34 | Step 8 - Go to hadoop's configuration directory 35 | cd hadoop/etc/hadoop/ 36 | 37 | Step 9 - vim core-site.xml and copy below configuration between configuration tag 38 | 39 | fs.default.name 40 | hdfs://localhost:9000 41 | 42 | 43 | Step 10 - vim hdfs-site.xml and copy below property between the configuration tag 44 | 45 | dfs.replication 46 | 1 47 | 48 | 49 | 50 | dfs.name.dir 51 | file:///home/hadoop/hadoopinfra/hdfs/namenode 52 | 53 | 54 | 55 | dfs.data.dir 56 | file:///home/hadoop/hadoopinfra/hdfs/datanode 57 | 58 | 59 | Step 11 - vim yarn-site.xml and copy below property between the configuration tag 60 | 61 | 62 | 63 | yarn.nodemanager.aux-services 64 | mapreduce_shuffle 65 | 66 | 67 | 68 | Step 12 - cp mapred-site.xml.template mapred-site.xml 69 | vim mapred-site.xml and copy below property between the configuration tag 70 | 71 | mapreduce.framework.name 72 | yarn 73 | 74 | 75 | Step 13 - Start dfs service by running below command 76 | start-dfs.sh 77 | start-yarn.sh 78 | 79 | Step 14 - Access Hadoop's application UI using browser 80 | http://localhost:8088/cluster 81 | 82 | Step 15 - Download Sqoop using below url 83 | wget http://www-us.apache.org/dist/sqoop/1.4.6/sqoop-1.4.6.bin__hadoop-2.0.4-alpha.tar.gz 84 | 85 | Step 16 - extract tar file and move to sqoop directory 86 | tar -xvz sqoop-1.4.6.bin__hadoop-2.0.4-alpha.tar.gz 87 | mv sqoop-1.4.6.bin__hadoop-2.0.4-alpha to sqoop 88 | export SQOOP_HOME=/usr/local/sqoop 89 | export PATH=$PATH:$SQOOP_HOME/bin 90 | cd $SQOOP_HOME/conf 91 | mv sqoop-env-template.sh sqoop-env.sh 92 | 93 | 94 | Step 17 - Export the Hadoop's configuration in Sqoop 95 | export HADOOP_COMMON_HOME=/usr/local/hadoop 96 | export HADOOP_MAPRED_HOME=/usr/local/hadoop 97 | 98 | Step 18 - Download MySQL connector 99 | wget http://ftp.ntu.edu.tw/MySQL/Downloads/Connector-J/mysql-connector-java-5.1.19.zip 100 | unzip mysql-connector-java-5.1.19.zip 101 | 102 | Step 19 - copy the jar file from mysql connector's directory to sqoop 103 | cd mysql-connector-java-5.1.19 104 | mv mysql-connector-java-5.1.19-bin.jar /usr/local/sqoop/lib 105 | 106 | Now Sqoop is installed and service can be started 107 | 108 | cd $SQOOP_HOME/bin 109 | sqoop-version 110 | 111 | Above command will give you current version of sqoop running 112 | 113 | 114 | 115 | -------------------------------------------------------------------------------- /Chapter09/hadoop_bigdata.sql: -------------------------------------------------------------------------------- 1 | CREATE DATABASE IF NOT EXISTS hadoop_bigdata; 2 | CREATE TABLE 3 | `hadoop_bigdata`.`users` ( 4 | `user_id` INT NOT NULL AUTO_INCREMENT , 5 | `email` VARCHAR(200) NOT NULL , 6 | `date_of_joining` DATE NOT NULL , 7 | `date_of_birth` DATE NOT NULL , 8 | `first_name` VARCHAR(200) NOT NULL , 9 | PRIMARY KEY (`user_id`) 10 | ) ENGINE = InnoDB; -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Packt 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | # MySQL 8 for Big Data 5 | This is the code repository for [MySQL 8 for Big Data](https://www.packtpub.com/big-data-and-business-intelligence/mysql-8-big-data?utm_source=github&utm_medium=repository&utm_campaign=9781788397186), published by [Packt](https://www.packtpub.com/?utm_source=github). It contains all the supporting project files necessary to work through the book from start to finish. 6 | ## About the Book 7 | MySQL is one of the most popular relational databases in the world today, and has become a popular choice of tool to handle vast amounts of structured data - that is, structured Big Data. This book will demonstrate how you can dabble with large amounts of data using MySQL 8. It also highlights topics such as integrating MySQL 8 and a Big Data solution like Apache Hadoop using different tools like Apache Sqoop and MySQL Applier. With practical examples and use-cases, you will get a better clarity on how you can leverage the offerings of MySQL 8 to build a robust Big Data solution. 8 | ## Instructions and Navigation 9 | All of the code is organized into folders. Each folder starts with a number followed by the application name. For example, Chapter02. 10 | 11 | 12 | 13 | The code will look like the following: 14 | ``` 15 | CREATE TABLE access_log ( 16 | log_id INT NOT NULL, 17 | type VARCHAR(100), 18 | access_url VARCHAR(100), 19 | access_date TIMESTAMP NOT NULL, 20 | response_time INT NOT NULL, 21 | access_by INT NOT NULL 22 | ) 23 | PARTITION BY RANGE (UNIX_TIMESTAMP(access_date)) ( 24 | PARTITION p0 VALUES LESS THAN (UNIX_TIMESTAMP('2017-05-01 00:00:00')), 25 | PARTITION p1 VALUES LESS THAN (UNIX_TIMESTAMP('2017-09-01 00:00:00')), 26 | PARTITION p2 VALUES LESS THAN (UNIX_TIMESTAMP('2018-01-01 00:00:00')), 27 | PARTITION p3 VALUES LESS THAN (UNIX_TIMESTAMP('2018-05-01 00:00:00')), 28 | PARTITION p4 VALUES LESS THAN (UNIX_TIMESTAMP('2018-09-01 00:00:00')), 29 | PARTITION p5 VALUES LESS THAN (UNIX_TIMESTAMP('2019-01-01 00:00:00')), 30 | ); 31 | ``` 32 | 33 | This book will guide you through the installation of all the tools that you need to follow the examples. You will need to install the following software to effectively run the code samples present in this book: 34 | 35 | MySQL 8.0.3 36 | Hadoop 2.8.1 37 | Apache Sqoop 1.4.6 38 | 39 | ## Related Products 40 | * [MySQL 8 Cookbook](https://www.packtpub.com/big-data-and-business-intelligence/mysql-8-cookbook?utm_source=github&utm_medium=repository&utm_campaign=9781788395809) 41 | 42 | * [MySQL for Python](https://www.packtpub.com/big-data-and-business-intelligence/mysql-python?utm_source=github&utm_medium=repository&utm_campaign=9781849510189) 43 | 44 | * [Mastering phpMyAdmin for Effective MySQL Management ](https://www.packtpub.com/big-data-and-business-intelligence/mastering-phpmyadmin-effective-mysql-management?utm_source=github&utm_medium=repository&utm_campaign=9781904811039) 45 | 46 | ### Download a free PDF 47 | 48 | If you have already purchased a print or Kindle version of this book, you can get a DRM-free PDF version at no cost.
Simply click on the link to claim your free PDF. 49 |

https://packt.link/free-ebook/9781788397186

--------------------------------------------------------------------------------