Td mazab

DBProject/bin/.gitignore

@@ -1,2 +1 @@
 /edu/
-/ExtractFeatures.class

DBProject/files/tmp.feat

@@ -1 +1 @@
-0	0	0	0	0	0	1	0	1	2	0	0	0	0	0	0	0	3	0	0	1	5	0	0	1	0	
+0	0	0	0	0	0	5	0	1	2	0	0	1	0	0	0	0	1	0	0	1	1	1	0	1	17	

DBProject/files/tmp.sql

@@ -1,11 +1,24 @@
-select
-	100.00 * sum(case
-		when p_type not like 'PROMO%'
-			then cast( l_extendedprice * (1 - l_discount) as decimal (18,2))
-		else 0
-	end) / sum( cast (l_extendedprice * (1 - l_discount) as decimal(18,2) )) as promo_revenue
+  select
+	sum(  cast (l_extendedprice* (1 - l_discount) as decimal(18,2)) ) as revenue
 from
 	lineitem,
 	part
 where
-	l_partkey = p_partkey;
+	(
+		p_partkey = l_partkey
+		and p_container in ('SM CASE' 'SM BOX' 'SM PACK' 'SM PKG')
+	)
+	or
+	(
+		p_partkey = l_partkey
+		and p_brand = 'Brand#45'
+		and p_size between 1 and 10
+		and l_shipmode in ('AIR' 'AIR REG')
+	)
+	or
+	(
+		p_partkey = l_partkey
+		and p_brand = 'Brand#33'
+		and p_container in ('LG CASE' 'LG BOX' 'LG PACK' 'LG PKG')
+		and l_quantity >= 23 and l_quantity <= 23 + 10
+	);

DBProject/src/edu/umich/td/controller/ConcurrentController.java

@@ -0,0 +1,236 @@
+/*Author: Mahmoud Azab
+ * Last Updated: November 2014
+ * Controller.java
+ */
+
+package edu.umich.td.controller;
+
+import java.awt.image.SampleModel;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import edu.umich.td.database.*;
+import edu.umich.td.fextractors.Feature;
+import edu.umich.td.fextractors.FeatureCategory;
+import edu.umich.td.fextractors.QPlanFeatureExtractor;
+import edu.umich.td.fextractors.QTextFeatureExtractor;
+import edu.umich.td.fextractors.WKLDFeatureExtractor;
+import edu.umich.td.workload.DataInstance;
+import edu.umich.td.workload.QueriesPool;
+import edu.umich.td.workload.Query;
+import edu.umich.td.workload.WorkLoadGenerator;
+
+public class ConcurrentController {
+
+	public static HashMap<Long, DataInstance> timeStampsToQueryFeatures = new HashMap<Long, DataInstance>();
+
+	static int currentConcurrentWorkLoad = 0;
+
+	public static void main(String[] args) {
+		// 1- Load queries into the query pool from the files
+		// 5- Read different query batches
+		// 6- Extract query/workload features
+		// 7- Get query/workload execution plans
+		// 8- Extract features from the execution plans
+		// 9- Execute the returned list of queries from the WL creator
+		// 10- Collect CPU and I/O usage
+
+		StatsCollector.tables = StatsCollector.GetAllTablesAndCounts("TPCH");
+
+		int Temp = 0;
+		int limiter = 0;
+		int batch_size = 5;
+		WorkLoadGenerator workloads = new WorkLoadGenerator();
+		workloads.ReadWorkLoadFromDirectory("/home/mahmoud/Documents/DB/Data/Workloads/TestWorkloads", batch_size);
+
+		Connection con = TdDatabase.OpenConnection();
+
+		//ConnectionPool pool = new ConnectionPool(batch_size);
+
+		int K = 0;
+		for (ArrayList<Query> queries : WorkLoadGenerator.wlQueries) {
+			System.out.println((K++) + ".");
+
+			if (queries.size() == batch_size) {
+				for (Query q : queries) {
+					QTextFeatureExtractor qtFeatExtractor = new QTextFeatureExtractor();
+					QPlanFeatureExtractor qtPlanFeatExtractor = new QPlanFeatureExtractor();
+					q.featureVector.clear();
+					q.featureVector.addAll(qtFeatExtractor.ExtractFeatures(q));
+					try {
+						q.featureVector.addAll(qtPlanFeatExtractor.getFeaturesFromPlan(con, q.qText));
+					} catch (Exception e) {
+						System.err.println(q.qText);
+						System.err.println(e.getMessage());
+					}
+				}
+			}
+
+		}
+
+		for (ArrayList<Query> queries : WorkLoadGenerator.wlQueries) {
+			if (queries.size() == batch_size) {
+				try {
+					Thread.sleep(15000);
+				} catch (InterruptedException e1) {
+					// TODO Auto-generated catch block
+					e1.printStackTrace();
+				}
+				for (Query q : queries) {
+					System.out.println("=========\tProcessing Query #" + K + "\t=========>  " + q.parentFolder + "/"
+							+ q.fileName + "\t features: " + q.featureVector.size());
+
+					for (int M = 0; M < batch_size; M++) {
+						Thread t = new Thread(new Runnable() {
+							public void run() {
+								// Connection connect =
+								// ConnectionPool.GetConnection();
+								Connection connect;
+								try {
+									connect = DriverManager.getConnection(TdDatabase.url, TdDatabase.sUser,
+											TdDatabase.sPassword);
+
+									for (Query q : WorkLoadGenerator.wlQueries.get(currentConcurrentWorkLoad)) {
+										if (!q.blocked) {
+											q.blocked = true;
+											String query = q.qText;
+											long timeStamp = System.currentTimeMillis();
+											query = query.replaceFirst("\n", "/*Q" + timeStamp + "*/\n");
+											boolean status = TdDatabase.ExecuteQuery(connect, query);
+											DataInstance data = new DataInstance();
+											data.features = q.CloneFeatures();
+											timeStampsToQueryFeatures.put(timeStamp, data);
+										}
+									}
+									connect.close();
+								} catch (SQLException e) {
+									// TODO Auto-generated catch block
+									e.printStackTrace();
+								}
+							}
+
+						});
+						t.start();
+
+						try {
+							Thread.sleep(100);
+						} catch (InterruptedException e) {
+							// TODO Auto-generated catch block
+							e.printStackTrace();
+						}
+					}
+				}
+			} else
+				limiter++;
+		}
+
+		try {
+			Thread.sleep(15000);
+		} catch (InterruptedException e) {
+			e.printStackTrace();
+		}
+		System.err.println("Writing Back");
+
+		WriteDataIntoFile(batch_size, "test");
+
+		TdDatabase.CloseConnection(con);
+
+	}
+
+	public static void WriteToFile(String fileName, String text, boolean append) {
+		try {
+			BufferedWriter bw = new BufferedWriter(new FileWriter(new File(fileName), append));
+			bw.write(text);
+			bw.close();
+		} catch (Exception e) {
+			System.out.println(e.getMessage());
+		}
+	}
+
+	static public String GetPrintableInstance(long C, ArrayList<Feature> f, int samplesSize, FeatureCategory category) {
+
+		StringBuilder sbuild = new StringBuilder();
+		sbuild.append(samplesSize + ",");
+		for (Feature feature : f) {
+			if (feature.category != category)
+				sbuild.append(feature.featureValue + ",");
+		}
+
+		sbuild.append(C);
+		return sbuild.toString();
+
+	}
+
+	public static void WriteDataIntoFile(int samplesSize, String type) {
+		StringBuilder cpuSbuild_ALL = new StringBuilder();
+		StringBuilder ioSbuild_ALL = new StringBuilder();
+
+		StringBuilder cpuSbuild_TEXT = new StringBuilder();
+		StringBuilder ioSbuild_TEXT = new StringBuilder();
+
+		StringBuilder cpuSbuild_PLAN = new StringBuilder();
+		StringBuilder ioSbuild_PLAN = new StringBuilder();
+
+		StringBuilder cpuSbuild_WL = new StringBuilder();
+		StringBuilder ioSbuild_WL = new StringBuilder();
+
+		Connection statsCon = StatsCollector.OpenConnection();
+
+		Iterator it = timeStampsToQueryFeatures.entrySet().iterator();
+		while (it.hasNext()) {
+			int[] CpuIo = { 0, 0 };
+			Map.Entry pairs = (Map.Entry) it.next();
+			CpuIo = StatsCollector.CollectCpuIO(statsCon, "Q" + pairs.getKey());
+			DataInstance instance = (DataInstance) pairs.getValue();
+			instance.CPU = CpuIo[0];
+			instance.IO = CpuIo[1];
+
+			cpuSbuild_ALL.append(GetPrintableInstance(instance.CPU, instance.features, samplesSize,
+					FeatureCategory.NULL) + "\n");
+			ioSbuild_ALL.append(GetPrintableInstance(instance.IO, instance.features, samplesSize, FeatureCategory.NULL)
+					+ "\n");
+
+			cpuSbuild_TEXT.append(GetPrintableInstance(instance.CPU, instance.features, samplesSize,
+					FeatureCategory.QUERYTEXT) + "\n");
+			ioSbuild_TEXT.append(GetPrintableInstance(instance.IO, instance.features, samplesSize,
+					FeatureCategory.QUERYTEXT) + "\n");
+
+			cpuSbuild_PLAN.append(GetPrintableInstance(instance.CPU, instance.features, samplesSize,
+					FeatureCategory.QUERYPLAN) + "\n");
+			ioSbuild_PLAN.append(GetPrintableInstance(instance.IO, instance.features, samplesSize,
+					FeatureCategory.QUERYPLAN) + "\n");
+
+			cpuSbuild_WL.append(GetPrintableInstance(instance.CPU, instance.features, samplesSize,
+					FeatureCategory.WORKLOAD) + "\n");
+			ioSbuild_WL.append(GetPrintableInstance(instance.IO, instance.features, samplesSize,
+					FeatureCategory.WORKLOAD) + "\n");
+
+			it.remove();
+		}
+
+		WriteToFile("files/Test/IO_" + samplesSize + "_ALL." + type, cpuSbuild_ALL.toString(), true);
+		WriteToFile("files/Test/CPU_" + samplesSize + "_ALL." + type, ioSbuild_ALL.toString(), true);
+
+		WriteToFile("files/Test/IO_" + samplesSize + "_TEXt." + type, cpuSbuild_ALL.toString(), true);
+		WriteToFile("files/Test/CPU_" + samplesSize + "_TEXT." + type, ioSbuild_ALL.toString(), true);
+
+		WriteToFile("files/Test/IO_" + samplesSize + "_PLAN." + type, cpuSbuild_ALL.toString(), true);
+		WriteToFile("files/Test/CPU_" + samplesSize + "_PLAN." + type, ioSbuild_ALL.toString(), true);
+
+		WriteToFile("files/Test/IO_" + samplesSize + "_WL." + type, cpuSbuild_ALL.toString(), true);
+		WriteToFile("files/Test/CPU_" + samplesSize + "_WL." + type, ioSbuild_ALL.toString(), true);
+
+		StatsCollector.CloseConnection(statsCon);
+		timeStampsToQueryFeatures.clear();
+
+	}
+
+}

DBProject/src/edu/umich/td/controller/ConnectionPool.java

@@ -0,0 +1,88 @@
+package edu.umich.td.controller;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+
+public class ConnectionPool {
+
+	static Connection[] connections;
+	static boolean[] inUse;
+
+	// Credentials
+	public static String sUser = "mazab";
+	public static String sPassword = "eecs584";
+
+	public static String url = "jdbc:teradata://ec2-54-164-159-30.compute-1.amazonaws.com";
+
+	public static Connection Open_Connection(int i) {
+		try {
+			try {
+				Class.forName("com.teradata.jdbc.TeraDriver");
+			} catch (ClassNotFoundException e1) {
+				e1.printStackTrace();
+			}
+			// Attempting to connect to Teradata
+			connections[i] = DriverManager.getConnection(url, sUser, sPassword);
+
+		} catch (Exception e) {
+			System.err.println(e.getMessage());
+		}
+		return null;
+	}
+
+	public ConnectionPool(int poolSize) {
+		connections = new Connection[poolSize];
+
+		for(int i=0; i < connections.length; i++){
+
+			connections[i] = Open_Connection(i);
+
+		}
+
+		inUse = new boolean[poolSize];
+	}
+
+	public static boolean OpenConnection(int i) {
+		try {
+			if (!connections[i].isClosed()) {
+				Open_Connection(i);
+			}
+		} catch (SQLException e) {
+			return false;
+		}
+		return true;
+
+	}
+
+	public static Connection GetConnection() {
+
+		while (true) {
+			for (int i = 0; i < inUse.length; i++) {
+				if (!inUse[i]) {
+					try {
+						if (!connections[i].isClosed()) {
+							return connections[i];
+						}else{
+							if(OpenConnection(i))
+								return connections[i];
+
+						}
+					} catch (SQLException e) {
+						// TODO Auto-generated catch block
+						e.printStackTrace();
+					}
+				}
+			}
+			try {
+				Thread.sleep(100);
+			} catch (InterruptedException e) {
+
+				e.getMessage();
+				return null;
+			}
+
+		}
+	}
+
+}