diff --git a/automaton.py b/automaton.py index e5e84b5..07cc72c 100755 --- a/automaton.py +++ b/automaton.py @@ -48,7 +48,7 @@ def run(self): cluster.launch() if self.config.options.terminate_cluster: cluster.connect() - if self.config.options.terminate_cluster == "all": + if self.config.options.terminate_cluster=="all": cluster.terminate_all() else: cluster.terminate(self.config.options.terminate_cluster) @@ -60,13 +60,13 @@ def run(self): cluster.deploy_software() if self.config.options.excute_benchmarks: cluster.connect() - cluster.excute_benchmarks() + cluster.excute_benchmarks(self.config.options.excute_benchmarks) def clean_exit(signum, frame): global SIGEXIT SIGEXIT = True - LOG.critical("Exit signal received. Exiting at the next sane time. " + LOG.critical("Exit signal received. Exiting at the next sane time." "Please stand by.") diff --git a/etc/benchmarking.conf b/etc/benchmarking.conf index 9570634..eac3a86 100755 --- a/etc/benchmarking.conf +++ b/etc/benchmarking.conf @@ -1,15 +1,10 @@ [Benchmark-01] sierra = 0 -hotel = 2 +hotel = 1 log_files = ~/BioPerf/Outputs/log url = http://www.bioperf.org/BioPerf.zip remote_location = ~/BioPerf [Benchmark-02] -sierra = 0 -hotel = 1 -log_files = ~/BioPerf/Outputs/log -url = http://www.bioperf.org/BioPerf.zip -remote_location = ~/BioPerf [Benchmark-03] \ No newline at end of file diff --git a/etc/clouds.conf b/etc/clouds.conf index 1c199ea..6391b1e 100755 --- a/etc/clouds.conf +++ b/etc/clouds.conf @@ -1,7 +1,7 @@ [hotel] cloud_uri = svc.uc.futuregrid.org cloud_port = 8444 -image_id = automaton.gz +image_id = debian-lenny.gz cloud_type = nimbus availability_zone = us-east-1 instance_type = m1.large diff --git a/etc/global.conf b/etc/global.conf index 3578ec7..80a7b9d 100755 --- a/etc/global.conf +++ b/etc/global.conf @@ -1,6 +1,6 @@ [DEFAULT] key_name = automaton -key_path = /Users/voran/.ssh/id_rsa.pub +key_path = /Users/voran/.ssh/id_rsa1.pub ssh_priv_key = /Users/suiyuan0226/.ssh/automaton.pem git_repo_home = /home/staged-deployment-scripts git_repo_location = https://github.com/alal3177/staged-deployment-scripts.git diff --git a/graphing/graphing.py b/graphing/graphing.py index 259bbad..29a0740 100644 --- a/graphing/graphing.py +++ b/graphing/graphing.py @@ -8,6 +8,8 @@ def __init__(self, config): self.config = config self.parser = Parser(self.config) self.graph_path = self.config.globals.graph_path + if not os.path.exists(self.graph_path): + os.makedirs(self.graph_path) self.attributes = list() self.attributes = self.parser.instance_types diff --git a/lib/util.py b/lib/util.py index 224359e..2235e84 100755 --- a/lib/util.py +++ b/lib/util.py @@ -102,55 +102,41 @@ def parse_options(): parser = OptionParser() + parser = OptionParser() + parser.add_option("-d", "--debug", action="store_true", dest="debug", help="Enable debugging log level.") parser.set_defaults(debug=False) - parser.add_option("-g", "--global_file", action="store", - dest="global_file", - help="Location of the file with global parameters " - "(default: etc/global.conf).") + parser.add_option("-g", "--global_file", action="store", dest="global_file", + help="Location of the file with global parameters (default: etc/global.conf).") parser.set_defaults(global_file="etc/global.conf") - parser.add_option("-c", "--clouds_file", action="store", - dest="clouds_file", - help="Location of the file with cloud parameters " - "(default: etc/clouds.conf).") + parser.add_option("-c", "--clouds_file", action="store", dest="clouds_file", + help="Location of the file with cloud parameters (default: etc/clouds.conf).") parser.set_defaults(clouds_file="etc/clouds.conf") - parser.add_option("-b", "--benchmarking_file", action="store", - dest="benchmarking_file", - help="Location of the file with benchmarking " - "parameters (default: etc/benchmarking.conf).") + parser.add_option("-b", "--benchmarking_file", action="store", dest="benchmarking_file", + help="Location of the file with benchmarking parameters (default: etc/benchmarking.conf).") parser.set_defaults(benchmarking_file="etc/benchmarking.conf") - parser.add_option("-l", "--launch_cluster", action="store_true", - dest="launch_cluster", - help="Launch desired number of clusters") + parser.add_option("-l", "--launch_cluster", action="store_true",dest="launch_cluster",help="Launch clusters") - parser.add_option("-t", "--terminate_cluster", action="store", - dest="terminate_cluster", - help="Terminate specific instance, argument: " - "all/instance_id", - default=False) + parser.add_option("-t", "--terminate_cluster", action="store",dest="terminate_cluster",help="Terminate specific instance, arguement: all/instance_id. To check the instance id: automaton.py -i") + parser.set_defaults(terminate_cluster="all") - parser.add_option("-s", "--deploy_software", action="store_true", - dest="deploy_software", help="Deploy Software") + parser.add_option("-s", "--deploy_software", action="store_true",dest="deploy_software",help="Deploy Softwares on instances.") - parser.add_option("-e", "--excute_benchmarks", action="store_true", - dest="excute_benchmarks", help="excute benchmarks") + parser.add_option("-e", "--excute_benchmarks", action="store",dest="excute_benchmarks",help="Excute benchmarks with different size dataset, arguement: small/medium/large.") + parser.set_defaults(excute_benchmarks="small") - parser.add_option("-o", "--gather_logs", action="store_true", - dest="gather_logs", help="Gather logs") + parser.add_option("-o", "--gather_logs", action="store_true",dest="gather_logs",help="Gather logs from instances.") - parser.add_option("-p", "--generate_graphs", action="store_true", - dest="generate_graphs", - help="Generate graphs that based on the collected logs") + parser.add_option("-p", "--generate_graphs", action="store_true",dest="generate_graphs",help="Generate graphs that based on the collected logs") - parser.add_option("-i", "--show_id", action="store_true", dest="show_id", - help="show the id of all running instances") + parser.add_option("-i", "--show_id", action="store_true",dest="show_id",help="Show the instance id of all running/terminated instances") (options, args) = parser.parse_args() - + return (options, args) diff --git a/resources/cluster/clusters.py b/resources/cluster/clusters.py index 8d7c2d0..08d8f28 100755 --- a/resources/cluster/clusters.py +++ b/resources/cluster/clusters.py @@ -170,13 +170,13 @@ def download_logs(self): if command_return != 0: LOG.error("Download logs: " + command.stdout) LOG.error("Download logs error: " + command.stderr) - + def deploy_software(self): ssh_priv_key = self.config.globals.ssh_priv_key ssh_username = self.config.globals.ssh_username ssh_timeout = int(self.config.globals.ssh_timeout) - reservations = list() - not_available = 0 + reservations = list() + not_available = 0; if self.reservations: reservations = self.reservations else: @@ -184,34 +184,46 @@ def deploy_software(self): reservations = cloud.conn.get_all_instances() for reservation in reservations: for instance in reservation.instances: - if self.database.check_benchmark(self.benchmark.name, - instance.id): - if not check_port_status(instance.ip_address, 22, - ssh_timeout): - LOG.error("Deploy_software: the port 22 is not " - "available right now. please try it later") - continue - cmds = list() + if self.database.check_benchmark(self.benchmark.name, instance.id): + if not check_port_status(instance.ip_address, 22, ssh_timeout): + LOG.error("Deploy_software: the port 22 is not available right now. please try it later") + continue + cmds = list() + cmds.append("rm -rf ~/*") cmds.append("wget %s" % (self.url)) - #cmds.append("apt-get update") - #cmds.append("apt-get install unzip") + cmds.append("apt-get update") + cmds.append("apt-get install unzip") cmds.append("unzip BioPerf.zip") - cmds.append("sed -i 's/read BIOPERF/#read " - "BIOPERF/g' install-BioPerf.sh") + cmds.append("sed -i 's/read BIOPERF/#read BIOPERF/g' install-BioPerf.sh") cmds.append("./install-BioPerf.sh") + cmds.append("wget ftp://ftp.cc.gatech.edu/pub/people/bader/BioPerf/swissprot.tar.gz") + cmds.append("tar -xvf swissprot.tar.gz") + cmds.append("mv Swissprot/* .") + cmds.append("wget ftp://ftp.cc.gatech.edu/pub/people/bader/BioPerf/Pfam") + cmds.append("wget ftp://ftp.cc.gatech.edu/pub/people/bader/BioPerf/nr") + cmds.append("sed -i '10 i\DATABASES=~/' ~/.profile") + cmds.append("sed -i '10 i\export DATABASES' ~/.profile") + cmds.append("sed -i '5c input='y'' ~/BioPerf/Scripts/Run-scripts/CleanOutputs.sh") + cmds.append("sed -i '21c #' ~/BioPerf/Scripts/Run-scripts/run-bioperf.sh") + cmds.append("sed -i '26c #' ~/BioPerf/Scripts/Run-scripts/run-bioperf.sh") + cmds.append("sed -i '10c arch='X'' ~/BioPerf/Scripts/Run-scripts/run-bioperf.sh") + cmds.append("sed -i '71c input3='A'' ~/BioPerf/Scripts/Run-scripts/run-bioperf.sh") + cmds.append('''sed -i "659c ch='Y';" ./BioPerf/Source-codes/Phylip/src/promlk.c''') + cmds.append("sed -i '/scanf/c //' ./BioPerf/Source-codes/Phylip/src/promlk.c") + cmds.append("cd ./BioPerf/Source-codes/Phylip/src;make promlk;cd ~") + cmds.append("mv ./BioPerf/Source-codes/Phylip/src/promlk ./BioPerf/Binaries/x86-Binaries/Phylip") + for c in cmds: - command = RemoteCommand(instance.public_dns_name, - ssh_priv_key, c) + command = RemoteCommand(instance.public_dns_name, ssh_priv_key, c) command_return = command.execute() - if command_return != 0: - LOG.error("Deploy_software: " + command.stdout) - LOG.error("Deploy_software error: " + - command.stderr) + if command_return !=0: + LOG.error("Deploy_software: "+command.stdout) + LOG.error("Deploy_software error: "+command.stderr) - def excute_benchmarks(self): + def excute_benchmarks(self,dataset_size): ssh_priv_key = self.config.globals.ssh_priv_key ssh_username = self.config.globals.ssh_username - reservations = list() + reservations = list() if self.reservations: reservations = self.reservations else: @@ -219,39 +231,30 @@ def excute_benchmarks(self): reservations = cloud.conn.get_all_instances() for reservation in reservations: for instance in reservation.instances: - if self.database.check_benchmark(self.benchmark.name, - instance.id): + if self.database.check_benchmark(self.benchmark.name, instance.id): cmds = list() - cmds.append("sed -i '5c input='y'' ~/BioPerf/Scripts/" - "Run-scripts/CleanOutputs.sh") - cmds.append("sed -i '13c rm -f $BIOPERF/Outputs/log' " - "~/BioPerf/Scripts/Run-scripts/" - "CleanOutputs.sh") - cmds.append("sed -i '21c #' " - "~/BioPerf/Scripts/Run-scripts/run-bioperf.sh") - cmds.append("sed -i '26c #' " - "~/BioPerf/Scripts/Run-scripts/run-bioperf.sh") - cmds.append("sed -i '10c arch='X'' " - "~/BioPerf/Scripts/Run-scripts/run-bioperf.sh") - cmds.append("sed -i '71c input3='A'' " - "~/BioPerf/Scripts/Run-scripts/run-bioperf.sh") - cmds.append("sed -i '134c input='A'' " - "~/BioPerf/Scripts/Run-scripts/run-bioperf.sh") - cmds.append("sed -i '145c user1='y'' " - "~/BioPerf/Scripts/Run-scripts/run-bioperf.sh") - cmds.append("./BioPerf/Scripts/Run-scripts/" - "CleanOutputs.sh") - cmds.append("echo 'Y' 'Y'|" - "./BioPerf/Scripts/Run-scripts/run-bioperf.sh" - " > ~/BioPerf/Outputs/log") - + cmds.append("sed -i '13c rm -f $BIOPERF/Outputs/log' ~/BioPerf/Scripts/Run-scripts/CleanOutputs.sh") + cmds.append("sed -i '60c FASTA=0' ~/BioPerf/Scripts/Run-scripts/run-bioperf.sh") + cmds.append("sed -i '62c GRAPPA=0' ~/BioPerf/Scripts/Run-scripts/run-bioperf.sh") + cmds.append("./BioPerf/Scripts/Run-scripts/CleanOutputs.sh") + if dataset_size=="large": + #cmds.append("sed -i '134c input='A'' ~/BioPerf/Scripts/Run-scripts/run-bioperf.sh") + cmds.append("echo C>c;echo H>>c;echo '1'>>c;echo Y>>c") + + elif dataset_size=="medium": + #cmds.append("sed -i '134c input='B'' ~/BioPerf/Scripts/Run-scripts/run-bioperf.sh") + cmds.append("echo B>c;echo H>>c;echo '1'>>c;echo Y>>c") + else: + #cmds.append("sed -i '134c input='C'' ~/BioPerf/Scripts/Run-scripts/run-bioperf.sh") + cmds.append("echo A>c;echo H>>c;echo '1'>>c;echo Y>>c") + cmds.append("cat c|./BioPerf/Scripts/Run-scripts/run-bioperf.sh > ~/BioPerf/Outputs/log") + for c in cmds: - command = RemoteCommand(instance.public_dns_name, - ssh_priv_key, c) + command = RemoteCommand(instance.public_dns_name, ssh_priv_key, c) command_return = command.execute() - if command_return != 0: - LOG.error("Excute_benchmarks: " + command.stdout) - LOG.error("Excute_benchmarks: " + command.stderr) + if command_return !=0: + LOG.error("Excute_benchmarks: "+command.stdout) + LOG.error("Excute_benchmarks: "+command.stderr) class Clusters(object): diff --git a/self.db b/self.db index 35c61d5..b1fbefb 100644 Binary files a/self.db and b/self.db differ