diff --git a/#README# b/#README# deleted file mode 100644 index 8a96db9..0000000 --- a/#README# +++ /dev/null @@ -1,58 +0,0 @@ -Installation -============ -To install BatchQ type: - -$ easy_install batchq - -Documentation -============= -The documentation can be found at http://batchq.rtfd.org - - -For version 0.3 -=============== -x = Brutus() -jobs = [] -with x: - using_lfs() - jobs += submit("command") - end_lfs() - -We have following objects -========================== -Connection type: -Terminal: -Submission system: - - -### Following would be easy to implement: -machine = Brutus() - -LSF(machine) \ - working_directory( ) - input_directory ( ) - jobs += run_job( " " ) - - -with FileCommander( ): - lcd( ) - rcd( ) - send_files( ) - sync( ) - -with Terminal([start options]): - cd(working_directory) - - use( Subshell () ) ## LSF(), default = Shell() - - for i in range(0,10): - jobs += submit( command ) # Submit returns a collection object - - enduse() - - -### -connection = Brutus() - -connection.send_files(["file1","file2"]) -connection.call("xx") diff --git a/MANIFEST b/MANIFEST deleted file mode 100644 index f18de64..0000000 --- a/MANIFEST +++ /dev/null @@ -1,94 +0,0 @@ -# file GENERATED by distutils, do NOT edit -AUTHORS -LICENSE -README -setup.py -batchq/__init__.py -batchq/bin/q -batchq/contrib/__init__.py -batchq/contrib/alps/__init__.py -batchq/contrib/alps/test.py -batchq/contrib/alps/tools.py -batchq/contrib/alps/examples/example1_dict.py -batchq/contrib/alps/examples/example1_dict.py~ -batchq/contrib/alps/examples/example2_configuration.py -batchq/contrib/alps/examples/example3_class.py -batchq/contrib/alps/examples/example3_class.py~ -batchq/contrib/alps/examples/original.py -batchq/contrib/examples/00_basics.py -batchq/contrib/examples/00f_wildcards.py -batchq/contrib/examples/01_create_file.py -batchq/contrib/examples/core_basepipe.output -batchq/contrib/examples/core_basepipe.py -batchq/contrib/examples/core_process.output -batchq/contrib/examples/core_process.py -batchq/contrib/examples/hello.txt -batchq/contrib/examples/replacement.py -batchq/contrib/examples/test.py -batchq/contrib/examples/tutorial2_basestructure.py -batchq/contrib/examples/tutorial2_call.py -batchq/contrib/examples/tutorial2_create_file2.py -batchq/contrib/examples/tutorial2_create_file3.py -batchq/contrib/examples/tutorial2_create_file4.py -batchq/contrib/examples/tutorial2_error1.py -batchq/contrib/examples/tutorial2_error2.py -batchq/contrib/examples/tutorial2_hello_param.output -batchq/contrib/examples/tutorial2_hello_param.py -batchq/contrib/examples/tutorial2_hello_world.output -batchq/contrib/examples/tutorial2_hello_world.py -batchq/contrib/examples/tutorial2_inherit.py -batchq/contrib/examples/tutorial2_python_implementation.py -batchq/contrib/examples/tutorial2_python_implementation_SSH.py -batchq/contrib/examples/tutorial2_replacement.output -batchq/contrib/examples/tutorial2_replacement.py -batchq/contrib/examples/tutorial3_nohup1.py -batchq/contrib/examples/tutorial3_nohup2.py -batchq/contrib/examples2/nohup/__init__.py -batchq/contrib/examples2/nohup/nohup.py -batchq/core/__init__.py -batchq/core/batch.py -batchq/core/communication.py -batchq/core/errors.py -batchq/core/library.py -batchq/core/memory.py -batchq/core/process.py -batchq/core/terminal.py -batchq/core/utils.py -batchq/core/terminal/__init__.py -batchq/core/terminal/base.py -batchq/core/terminal/patternregister.py -batchq/core/terminal/vt100.py -batchq/core/terminal/xterm.py -batchq/pipelines/__init__.py -batchq/pipelines/interpreters/__init__.py -batchq/pipelines/interpreters/python.py -batchq/pipelines/math/__init__.py -batchq/pipelines/math/maple.py -batchq/pipelines/shell/__init__.py -batchq/pipelines/shell/bash.py -batchq/pipelines/shell/sftp.py -batchq/pipelines/shell/ssh.py -batchq/pipelines/shell/utils.py -batchq/queues/__init__.py -batchq/queues/functions.py -batchq/queues/lsf.py -batchq/queues/nohup.py -batchq/shortcuts/__init__.py -batchq/shortcuts/queue.py -batchq/shortcuts/shell.py -batchq/templates/__init__.py -docs/api.rst -docs/index.rst -docs/introduction.rst -docs/tutorial.rst -docs/tutorial_bsub.rst -docs/tutorial_nhup.rst -docs/vistrails.rst -docs/webinterface.rst -docs/api/batch.rst -docs/api/math_terminals.rst -docs/api/shell_terminals.rst -docs/api/terminal.rst -docs/contrib/django_webserver.rst -docs/contrib/kivy_terminal.rst -docs/user/intro.rst diff --git a/batchq/__init__.py b/batchq/__init__.py index 4118f1c..033b584 100644 --- a/batchq/__init__.py +++ b/batchq/__init__.py @@ -1,2 +1,2 @@ def get_version(): - return "0.1-3-pre-alpha" + return "0.1-4-pre-alpha" diff --git a/batchq/bin/q b/batchq/bin/q new file mode 100755 index 0000000..0ba7777 --- /dev/null +++ b/batchq/bin/q @@ -0,0 +1,213 @@ +#! /opt/local/bin/python +from batchq.queues import * +from batchq.core.batch import load_settings +import sys +import inspect +import json +import copy + +class Main(object): + """ + The command line tool can be used for queuing and templating. The + general syntax: + + $ q name [task] [arg1] [arg2] ... [argN] [-u] [-t] [-i] [--parameter1=value1] [--parameter2=value2] ... [--parameterM=valueM] + + $ q name [arg1] [arg2] ... [argN] [-t] [--parameter1=value1] [--parameter2=value2] ... [--parameterM=valueM] + + Example: + $ q nohup-ssh submitjob --host=localhost --username=tfr --indir=. --outdir=/Users/tfr/Downloads/ + + + If the name is both a queue name and a template name, the queue will be + executed as default. For queues the parameters are set in an instance of + the object, where as for templates these are passed on as context + variables. If the flag -t is on, the name is looked up as a template. + + """ + + def __call__(self, name,task=None, *args, **kwargs): + """ + + """ + from batchq.core.library import Library + which = -1 + u,t,i,q,f = False,False,False,False,True + + if not task is None and "@" in task: + task, conffile = task.split("@") + + newargs, newkwargs, switches = load_settings(conffile) + u,t,i,q,f = switches + if len(args)< len(newargs): + for n in range(0,len(args)): + newargs[n] = args[n] + args = tuple(newargs) + + newkwargs.update(kwargs) + kwargs = newkwargs + + + + if 'u' in kwargs: + u = kwargs['u'] + del kwargs['u'] + self.update() + if 'i' in kwargs: + i = kwargs['i'] + del kwargs['i'] + if 't' in kwargs: + t = kwargs['t'] + del kwargs['t'] + if 'q' in kwargs: + q = kwargs['q'] + del kwargs['q'] + if t: + raise BaseException("Either of -t or -q must be set, not both.") + if t or q: + f = False + + if t and name in Library.templates.dict: + which = 2 + elif q and name in Library.queues.dict: + which = 1 + elif name in Library.functions.dict: + which = 0 + elif name in Library.queues.dict: + which = 1 + elif name in Library.templates.dict: + which = 2 + + + if which == -1: + print inspect.getdoc(self.update) + print + print "Available functions:", ", ".join([a for a in Library.functions.dict.iterkeys()]) + print "Available queues:", ", ".join([a for a in Library.queues.dict.iterkeys()]) + print "Available templates:", ", ".join([a for a in Library.templates.dict.iterkeys()]) + print + return -1 + + + switches = (u,t,i,q,f) + if which == 0: + fnc = Library.functions.dict[name] + fnc(task, args, kwargs, switches) + + if which == 1: + + + cls = Library.queues.dict[name] + try: + if i: + kwargs[u'q_interact'] = True + instance = cls(*args,**kwargs) + except: + print + print "Class: ", cls + print "Arguments: ", args + print "Keyword arguments: ", kwargs + print "Error:", sys.exc_info()[0] + print + + if inspect.getdoc(cls): + print inspect.getdoc(cls) + print + + raise + + + if hasattr(instance, task): + + fnc = getattr(instance,task) + r = fnc() + ret = r.val() + if type(ret) is not int: + print ret + try: + ret = int(ret) + except: + ret = 0 + return ret + else: + if inspect.getdoc(cls): + print inspect.getdoc(cls) + print + raise BaseException("Task '%s' is not defined for %s." %(task,name)) + return 0 + + if which == 2: + outfile = args[0] + + f = open(task, "r+") + input = f.read() + f.close() + eng = Library.templates.dict[name] + + ins = eng(input) + ins.set_filename(task) + output = ins.render({}) + + f = open(outfile, "w") + f.write(output) + f.close() + + + return 0 + + + def update(self): + """ + If a listed module is not available, call q with the update + option -u: + + $ q.py name [task] -u [-i] [--parameter1=value1] [--parameter2=value2] ... [--parameterN=valueN] + + This will update the cache and locate new modules. NOTE: At the + moment no cache is installed so you really don't need to worry + about this. + """ + pass + +if __name__ == "__main__": + from batchq import get_version + main = Main() + ret = -1 +# try: + lst = sys.argv + + indices = [i for i in range(0, len(lst)) if len(lst[i])==0 or lst[i][0] !="-"] + indices.reverse() + noparam = [lst.pop(i) for i in indices] + noparam.reverse() + + try: + file, name, task = noparam[0:3] + args = noparam[3:] + except: + try: + task = None + file, name = noparam[0:2] + args = () + except: + print "BatchQ ", get_version() + print + print "Type 'q help' for help." + print "Type 'q list' for a list of available commands." + sys.exit(0) + + ext = lambda x: x if len(x) == 2 else x+[True] + fnc = lambda x: ext(x[2:].split("=",)) if len(x) > 2 and x[0:2] == "--" else (x[1:], True) + kwargs = dict([fnc(val) for val in lst ]) + + ret = main(name,task,*args, **kwargs) +# except: +# print "BatchQ" +# print "by Troels F. Roennow, 2011-2012 ETH Zurich" +# print +# print inspect.getdoc(main) +# print + sys.exit(ret) + + + diff --git a/batchq/contrib/alps/examples/example1_dict.py~ b/batchq/contrib/alps/examples/example1_dict.py~ deleted file mode 100644 index 65fa96e..0000000 --- a/batchq/contrib/alps/examples/example1_dict.py~ +++ /dev/null @@ -1,58 +0,0 @@ -from batchq.contrib.alps import runApplicationBackground, NoHUPSSH, LSFBSub, load_queue,DescriptorQ -import pyalps -import matplotlib.pyplot as plt -import pyalps.plot -import sys - -parms = [] -for t in [1.5,2,2.5]: - parms.append( - { - 'LATTICE' : "square lattice", - 'T' : t, - 'J' : 1 , - 'THERMALIZATION' : 1000, - 'SWEEPS' : 100000, - 'UPDATE' : "cluster", - 'MODEL' : "Ising", - 'L' : 8 - } - ) - -input_file = pyalps.writeInputFiles('parm1',parms) - -settings = {'command': ".", 'username':"tfr", 'server':"satis.ethz.ch", 'port':22, 'working_directory':"/Users/tfr/RemoteJobs/Submission", 'input_directory':"/Users/tfr/RemoteJobs/TestInDir", 'output_directory':"/Users/tfr/RemoteJobs/TestOutDir2", 'q_interact':True, "options": "", "prior":"", "post":""} -q = load_queue(NoHUPSSH, settings) -desc = runApplicationBackground('spinmc',input_file,Tmin=5,writexml=True, queue = q, force_resubmit = False ) - -if not desc.finished(): - print "Your simulations has not yet ended, please run this command again later." -else: - if desc.failed(): - print "Your submission has failed" - sys.exit(-1) - result_files = pyalps.getResultFiles(prefix='parm1') - print result_files - print pyalps.loadObservableList(result_files) - data = pyalps.loadMeasurements(result_files,['|Magnetization|','Magnetization^2']) - print data - plotdata = pyalps.collectXY(data,'T','|Magnetization|') - plt.figure() - pyalps.plot.plot(plotdata) - plt.xlim(0,3) - plt.ylim(0,1) - plt.title('Ising model') - plt.show() - print pyalps.plot.convertToText(plotdata) - print pyalps.plot.makeGracePlot(plotdata) - print pyalps.plot.makeGnuplotPlot(plotdata) - binder = pyalps.DataSet() - binder.props = pyalps.dict_intersect([d[0].props for d in data]) - binder.x = [d[0].props['T'] for d in data] - binder.y = [d[1].y[0]/(d[0].y[0]*d[0].y[0]) for d in data] - print binder - plt.figure() - pyalps.plot.plot(binder) - plt.xlabel('T') - plt.ylabel('Binder cumulant') - plt.show() diff --git a/batchq/contrib/alps/examples/example3_class.py~ b/batchq/contrib/alps/examples/example3_class.py~ deleted file mode 100644 index e0ed392..0000000 --- a/batchq/contrib/alps/examples/example3_class.py~ +++ /dev/null @@ -1,67 +0,0 @@ -from batchq.contrib.alps import runApplicationBackground, NoHUPSSH, LSFBSub, load_queue,DescriptorQ -import pyalps -import matplotlib.pyplot as plt -import pyalps.plot -import sys - -parms = [] -for t in [1.5,2,2.5]: - parms.append( - { - 'LATTICE' : "square lattice", - 'T' : t, - 'J' : 1 , - 'THERMALIZATION' : 1000, - 'SWEEPS' : 100000, - 'UPDATE' : "cluster", - 'MODEL' : "Ising", - 'L' : 8 - } - ) - -input_file = pyalps.writeInputFiles('parm1',parms) - -class Brutus(DescriptorQ): - queue = LSFBSub - username = "tronnow" - server="brutus.ethz.ch" - port=22 - options = "" - prior = "module load open_mpi goto2 python hdf5 cmake mkl\nexport PATH=$PATH:$HOME/opt/alps/bin" - post = "" - working_directory = "Submission" - -desc = runApplicationBackground('spinmc',input_file,Tmin=5,writexml=True, descriptor = Brutus(), force_resubmit = False ) - - -if not desc.finished(): - print "Your simulations has not yet ended, please run this command again later." -else: - if desc.failed(): - print "Your submission has failed" - sys.exit(-1) - result_files = pyalps.getResultFiles(prefix='parm1') - print result_files - print pyalps.loadObservableList(result_files) - data = pyalps.loadMeasurements(result_files,['|Magnetization|','Magnetization^2']) - print data - plotdata = pyalps.collectXY(data,'T','|Magnetization|') - plt.figure() - pyalps.plot.plot(plotdata) - plt.xlim(0,3) - plt.ylim(0,1) - plt.title('Ising model') - plt.show() - print pyalps.plot.convertToText(plotdata) - print pyalps.plot.makeGracePlot(plotdata) - print pyalps.plot.makeGnuplotPlot(plotdata) - binder = pyalps.DataSet() - binder.props = pyalps.dict_intersect([d[0].props for d in data]) - binder.x = [d[0].props['T'] for d in data] - binder.y = [d[1].y[0]/(d[0].y[0]*d[0].y[0]) for d in data] - print binder - plt.figure() - pyalps.plot.plot(binder) - plt.xlabel('T') - plt.ylabel('Binder cumulant') - plt.show() diff --git a/batchq/core/index.html b/batchq/core/index.html deleted file mode 100644 index fca785c..0000000 --- a/batchq/core/index.html +++ /dev/null @@ -1,547 +0,0 @@ - - - - - - - - - - - - -
- -
-

Treatments

Naive method

We made use of Machine1 in the following section - see Appendix: Machine1 details.

The source code (referred to as sub2) that form the basis the simulation can be downloaded here.

Next the code (sub2) was executed by submitting following command(s):

./script
-
-

The raw data from the simulation can be downloaded here.

We applied the following procedure:

#!/usr/bin/env python
-
-if __name__ == "__main__":
-  print "Hello script"
-
-

to get:

Hello script
-
-

Alternative method

We made use of Machine1 in the following section - see Appendix: Machine1 details.

Using the raw data from earlier (sub2 in Naive method) we proceeded:

We applied the following procedure:

        def plot_figure(self,directory):
-            print "Treating data"
-            return "Treated by function"
-
-

to get:

Treated by function
-
-

Smart method

We made use of Machine1 in the following section - see Appendix: Machine1 details.

Using the raw data from earlier (sub2 in Naive method) we proceeded:

We applied the following procedure:

        def plot_figure(self,directory):
-            print "Treating data"
-            return "Treated by function"
-
-

to get:

Treated by function
-
-

Machine1

==================== CPU ====================processor	: 0
-vendor_id	: AuthenticAMD
-cpu family	: 16
-model		: 4
-model name	: Quad-Core AMD Opteron(tm) Processor 8380
-stepping	: 2
-cpu MHz		: 2500.104
-cache size	: 512 KB
-physical id	: 1
-siblings	: 4
-core id		: 0
-cpu cores	: 4
-apicid		: 4
-fpu		: yes
-fpu_exception	: yes
-cpuid level	: 5
-wp		: yes
-flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm 3dnowext 3dnow constant_tsc nonstop_tsc pni cx16 popcnt lahf_lm cmp_legacy svm extapic cr8_legacy altmovcr8 abm sse4a misalignsse 3dnowprefetch osvw
-bogomips	: 5000.20
-TLB size	: 1024 4K pages
-clflush size	: 64
-cache_alignment	: 64
-address sizes	: 48 bits physical, 48 bits virtual
-power management: ts ttp tm stc 100mhzsteps hwpstate [8]
-
-processor	: 1
-vendor_id	: AuthenticAMD
-cpu family	: 16
-model		: 4
-model name	: Quad-Core AMD Opteron(tm) Processor 8380
-stepping	: 2
-cpu MHz		: 2500.104
-cache size	: 512 KB
-physical id	: 1
-siblings	: 4
-core id		: 1
-cpu cores	: 4
-apicid		: 5
-fpu		: yes
-fpu_exception	: yes
-cpuid level	: 5
-wp		: yes
-flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm 3dnowext 3dnow constant_tsc nonstop_tsc pni cx16 popcnt lahf_lm cmp_legacy svm extapic cr8_legacy altmovcr8 abm sse4a misalignsse 3dnowprefetch osvw
-bogomips	: 5000.27
-TLB size	: 1024 4K pages
-clflush size	: 64
-cache_alignment	: 64
-address sizes	: 48 bits physical, 48 bits virtual
-power management: ts ttp tm stc 100mhzsteps hwpstate [8]
-
-processor	: 2
-vendor_id	: AuthenticAMD
-cpu family	: 16
-model		: 4
-model name	: Quad-Core AMD Opteron(tm) Processor 8380
-stepping	: 2
-cpu MHz		: 2500.104
-cache size	: 512 KB
-physical id	: 1
-siblings	: 4
-core id		: 2
-cpu cores	: 4
-apicid		: 6
-fpu		: yes
-fpu_exception	: yes
-cpuid level	: 5
-wp		: yes
-flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm 3dnowext 3dnow constant_tsc nonstop_tsc pni cx16 popcnt lahf_lm cmp_legacy svm extapic cr8_legacy altmovcr8 abm sse4a misalignsse 3dnowprefetch osvw
-bogomips	: 5000.31
-TLB size	: 1024 4K pages
-clflush size	: 64
-cache_alignment	: 64
-address sizes	: 48 bits physical, 48 bits virtual
-power management: ts ttp tm stc 100mhzsteps hwpstate [8]
-
-processor	: 3
-vendor_id	: AuthenticAMD
-cpu family	: 16
-model		: 4
-model name	: Quad-Core AMD Opteron(tm) Processor 8380
-stepping	: 2
-cpu MHz		: 2500.104
-cache size	: 512 KB
-physical id	: 1
-siblings	: 4
-core id		: 3
-cpu cores	: 4
-apicid		: 7
-fpu		: yes
-fpu_exception	: yes
-cpuid level	: 5
-wp		: yes
-flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm 3dnowext 3dnow constant_tsc nonstop_tsc pni cx16 popcnt lahf_lm cmp_legacy svm extapic cr8_legacy altmovcr8 abm sse4a misalignsse 3dnowprefetch osvw
-bogomips	: 5000.32
-TLB size	: 1024 4K pages
-clflush size	: 64
-cache_alignment	: 64
-address sizes	: 48 bits physical, 48 bits virtual
-power management: ts ttp tm stc 100mhzsteps hwpstate [8]
-
-processor	: 4
-vendor_id	: AuthenticAMD
-cpu family	: 16
-model		: 4
-model name	: Quad-Core AMD Opteron(tm) Processor 8380
-stepping	: 2
-cpu MHz		: 2500.104
-cache size	: 512 KB
-physical id	: 2
-siblings	: 4
-core id		: 0
-cpu cores	: 4
-apicid		: 8
-fpu		: yes
-fpu_exception	: yes
-cpuid level	: 5
-wp		: yes
-flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm 3dnowext 3dnow constant_tsc nonstop_tsc pni cx16 popcnt lahf_lm cmp_legacy svm extapic cr8_legacy altmovcr8 abm sse4a misalignsse 3dnowprefetch osvw
-bogomips	: 5002.05
-TLB size	: 1024 4K pages
-clflush size	: 64
-cache_alignment	: 64
-address sizes	: 48 bits physical, 48 bits virtual
-power management: ts ttp tm stc 100mhzsteps hwpstate [8]
-
-processor	: 5
-vendor_id	: AuthenticAMD
-cpu family	: 16
-model		: 4
-model name	: Quad-Core AMD Opteron(tm) Processor 8380
-stepping	: 2
-cpu MHz		: 2500.104
-cache size	: 512 KB
-physical id	: 2
-siblings	: 4
-core id		: 1
-cpu cores	: 4
-apicid		: 9
-fpu		: yes
-fpu_exception	: yes
-cpuid level	: 5
-wp		: yes
-flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm 3dnowext 3dnow constant_tsc nonstop_tsc pni cx16 popcnt lahf_lm cmp_legacy svm extapic cr8_legacy altmovcr8 abm sse4a misalignsse 3dnowprefetch osvw
-bogomips	: 5000.92
-TLB size	: 1024 4K pages
-clflush size	: 64
-cache_alignment	: 64
-address sizes	: 48 bits physical, 48 bits virtual
-power management: ts ttp tm stc 100mhzsteps hwpstate [8]
-
-processor	: 6
-vendor_id	: AuthenticAMD
-cpu family	: 16
-model		: 4
-model name	: Quad-Core AMD Opteron(tm) Processor 8380
-stepping	: 2
-cpu MHz		: 2500.104
-cache size	: 512 KB
-physical id	: 2
-siblings	: 4
-core id		: 2
-cpu cores	: 4
-apicid		: 10
-fpu		: yes
-fpu_exception	: yes
-cpuid level	: 5
-wp		: yes
-flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm 3dnowext 3dnow constant_tsc nonstop_tsc pni cx16 popcnt lahf_lm cmp_legacy svm extapic cr8_legacy altmovcr8 abm sse4a misalignsse 3dnowprefetch osvw
-bogomips	: 5001.62
-TLB size	: 1024 4K pages
-clflush size	: 64
-cache_alignment	: 64
-address sizes	: 48 bits physical, 48 bits virtual
-power management: ts ttp tm stc 100mhzsteps hwpstate [8]
-
-processor	: 7
-vendor_id	: AuthenticAMD
-cpu family	: 16
-model		: 4
-model name	: Quad-Core AMD Opteron(tm) Processor 8380
-stepping	: 2
-cpu MHz		: 2500.104
-cache size	: 512 KB
-physical id	: 2
-siblings	: 4
-core id		: 3
-cpu cores	: 4
-apicid		: 11
-fpu		: yes
-fpu_exception	: yes
-cpuid level	: 5
-wp		: yes
-flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm 3dnowext 3dnow constant_tsc nonstop_tsc pni cx16 popcnt lahf_lm cmp_legacy svm extapic cr8_legacy altmovcr8 abm sse4a misalignsse 3dnowprefetch osvw
-bogomips	: 5001.75
-TLB size	: 1024 4K pages
-clflush size	: 64
-cache_alignment	: 64
-address sizes	: 48 bits physical, 48 bits virtual
-power management: ts ttp tm stc 100mhzsteps hwpstate [8]
-
-processor	: 8
-vendor_id	: AuthenticAMD
-cpu family	: 16
-model		: 4
-model name	: Quad-Core AMD Opteron(tm) Processor 8380
-stepping	: 2
-cpu MHz		: 2500.104
-cache size	: 512 KB
-physical id	: 3
-siblings	: 4
-core id		: 0
-cpu cores	: 4
-apicid		: 12
-fpu		: yes
-fpu_exception	: yes
-cpuid level	: 5
-wp		: yes
-flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm 3dnowext 3dnow constant_tsc nonstop_tsc pni cx16 popcnt lahf_lm cmp_legacy svm extapic cr8_legacy altmovcr8 abm sse4a misalignsse 3dnowprefetch osvw
-bogomips	: 5004.06
-TLB size	: 1024 4K pages
-clflush size	: 64
-cache_alignment	: 64
-address sizes	: 48 bits physical, 48 bits virtual
-power management: ts ttp tm stc 100mhzsteps hwpstate [8]
-
-processor	: 9
-vendor_id	: AuthenticAMD
-cpu family	: 16
-model		: 4
-model name	: Quad-Core AMD Opteron(tm) Processor 8380
-stepping	: 2
-cpu MHz		: 2500.104
-cache size	: 512 KB
-physical id	: 3
-siblings	: 4
-core id		: 1
-cpu cores	: 4
-apicid		: 13
-fpu		: yes
-fpu_exception	: yes
-cpuid level	: 5
-wp		: yes
-flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm 3dnowext 3dnow constant_tsc nonstop_tsc pni cx16 popcnt lahf_lm cmp_legacy svm extapic cr8_legacy altmovcr8 abm sse4a misalignsse 3dnowprefetch osvw
-bogomips	: 5002.52
-TLB size	: 1024 4K pages
-clflush size	: 64
-cache_alignment	: 64
-address sizes	: 48 bits physical, 48 bits virtual
-power management: ts ttp tm stc 100mhzsteps hwpstate [8]
-
-processor	: 10
-vendor_id	: AuthenticAMD
-cpu family	: 16
-model		: 4
-model name	: Quad-Core AMD Opteron(tm) Processor 8380
-stepping	: 2
-cpu MHz		: 2500.104
-cache size	: 512 KB
-physical id	: 3
-siblings	: 4
-core id		: 2
-cpu cores	: 4
-apicid		: 14
-fpu		: yes
-fpu_exception	: yes
-cpuid level	: 5
-wp		: yes
-flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm 3dnowext 3dnow constant_tsc nonstop_tsc pni cx16 popcnt lahf_lm cmp_legacy svm extapic cr8_legacy altmovcr8 abm sse4a misalignsse 3dnowprefetch osvw
-bogomips	: 4999.71
-TLB size	: 1024 4K pages
-clflush size	: 64
-cache_alignment	: 64
-address sizes	: 48 bits physical, 48 bits virtual
-power management: ts ttp tm stc 100mhzsteps hwpstate [8]
-
-processor	: 11
-vendor_id	: AuthenticAMD
-cpu family	: 16
-model		: 4
-model name	: Quad-Core AMD Opteron(tm) Processor 8380
-stepping	: 2
-cpu MHz		: 2500.104
-cache size	: 512 KB
-physical id	: 3
-siblings	: 4
-core id		: 3
-cpu cores	: 4
-apicid		: 15
-fpu		: yes
-fpu_exception	: yes
-cpuid level	: 5
-wp		: yes
-flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm 3dnowext 3dnow constant_tsc nonstop_tsc pni cx16 popcnt lahf_lm cmp_legacy svm extapic cr8_legacy altmovcr8 abm sse4a misalignsse 3dnowprefetch osvw
-bogomips	: 5000.34
-TLB size	: 1024 4K pages
-clflush size	: 64
-cache_alignment	: 64
-address sizes	: 48 bits physical, 48 bits virtual
-power management: ts ttp tm stc 100mhzsteps hwpstate [8]
-
-processor	: 12
-vendor_id	: AuthenticAMD
-cpu family	: 16
-model		: 4
-model name	: Quad-Core AMD Opteron(tm) Processor 8380
-stepping	: 2
-cpu MHz		: 2500.104
-cache size	: 512 KB
-physical id	: 4
-siblings	: 4
-core id		: 0
-cpu cores	: 4
-apicid		: 16
-fpu		: yes
-fpu_exception	: yes
-cpuid level	: 5
-wp		: yes
-flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm 3dnowext 3dnow constant_tsc nonstop_tsc pni cx16 popcnt lahf_lm cmp_legacy svm extapic cr8_legacy altmovcr8 abm sse4a misalignsse 3dnowprefetch osvw
-bogomips	: 5002.05
-TLB size	: 1024 4K pages
-clflush size	: 64
-cache_alignment	: 64
-address sizes	: 48 bits physical, 48 bits virtual
-power management: ts ttp tm stc 100mhzsteps hwpstate [8]
-
-processor	: 13
-vendor_id	: AuthenticAMD
-cpu family	: 16
-model		: 4
-model name	: Quad-Core AMD Opteron(tm) Processor 8380
-stepping	: 2
-cpu MHz		: 2500.104
-cache size	: 512 KB
-physical id	: 4
-siblings	: 4
-core id		: 1
-cpu cores	: 4
-apicid		: 17
-fpu		: yes
-fpu_exception	: yes
-cpuid level	: 5
-wp		: yes
-flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm 3dnowext 3dnow constant_tsc nonstop_tsc pni cx16 popcnt lahf_lm cmp_legacy svm extapic cr8_legacy altmovcr8 abm sse4a misalignsse 3dnowprefetch osvw
-bogomips	: 5000.77
-TLB size	: 1024 4K pages
-clflush size	: 64
-cache_alignment	: 64
-address sizes	: 48 bits physical, 48 bits virtual
-power management: ts ttp tm stc 100mhzsteps hwpstate [8]
-
-processor	: 14
-vendor_id	: AuthenticAMD
-cpu family	: 16
-model		: 4
-model name	: Quad-Core AMD Opteron(tm) Processor 8380
-stepping	: 2
-cpu MHz		: 2500.104
-cache size	: 512 KB
-physical id	: 4
-siblings	: 4
-core id		: 2
-cpu cores	: 4
-apicid		: 18
-fpu		: yes
-fpu_exception	: yes
-cpuid level	: 5
-wp		: yes
-flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm 3dnowext 3dnow constant_tsc nonstop_tsc pni cx16 popcnt lahf_lm cmp_legacy svm extapic cr8_legacy altmovcr8 abm sse4a misalignsse 3dnowprefetch osvw
-bogomips	: 5000.84
-TLB size	: 1024 4K pages
-clflush size	: 64
-cache_alignment	: 64
-address sizes	: 48 bits physical, 48 bits virtual
-power management: ts ttp tm stc 100mhzsteps hwpstate [8]
-
-processor	: 15
-vendor_id	: AuthenticAMD
-cpu family	: 16
-model		: 4
-model name	: Quad-Core AMD Opteron(tm) Processor 8380
-stepping	: 2
-cpu MHz		: 2500.104
-cache size	: 512 KB
-physical id	: 4
-siblings	: 4
-core id		: 3
-cpu cores	: 4
-apicid		: 19
-fpu		: yes
-fpu_exception	: yes
-cpuid level	: 5
-wp		: yes
-flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm 3dnowext 3dnow constant_tsc nonstop_tsc pni cx16 popcnt lahf_lm cmp_legacy svm extapic cr8_legacy altmovcr8 abm sse4a misalignsse 3dnowprefetch osvw
-bogomips	: 5000.36
-TLB size	: 1024 4K pages
-clflush size	: 64
-cache_alignment	: 64
-address sizes	: 48 bits physical, 48 bits virtual
-power management: ts ttp tm stc 100mhzsteps hwpstate [8]
-
-
-==================== MEM ====================MemTotal:     66006512 kB
-MemFree:       6554688 kB
-Buffers:         82128 kB
-Cached:       45534236 kB
-SwapCached:      18168 kB
-Active:       10948144 kB
-Inactive:     45197176 kB
-HighTotal:           0 kB
-HighFree:            0 kB
-LowTotal:     66006512 kB
-LowFree:       6554688 kB
-SwapTotal:    67111528 kB
-SwapFree:     64682268 kB
-Dirty:             244 kB
-Writeback:           0 kB
-AnonPages:    10525092 kB
-Mapped:          84140 kB
-Slab:          2906788 kB
-PageTables:      91172 kB
-NFS_Unstable:       12 kB
-Bounce:              0 kB
-CommitLimit:  100114784 kB
-Committed_AS: 14159436 kB
-VmallocTotal: 34359738367 kB
-VmallocUsed:    322908 kB
-VmallocChunk: 34359391351 kB
-HugePages_Total:     0
-HugePages_Free:      0
-HugePages_Rsvd:      0
-Hugepagesize:     2048 kB
-
-

-
-
- - - diff --git a/batchq/core/terminal/#vt100.py# b/batchq/core/terminal/#vt100.py# deleted file mode 100644 index d5ed6ce..0000000 --- a/batchq/core/terminal/#vt100.py# +++ /dev/null @@ -1,383 +0,0 @@ -#################################################################################### -# Copyright (C) 2011-2012 -# Troels F. Roennow, ETH Zurich -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of -# this software and associated documentation files (the "Software"), to deal in -# the Software without restriction, including without limitation the rights to -# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -# of the Software, and to permit persons to whom the Software is furnished to do -# so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -#################################################################################### - -# Note the VT100 intepreter is still incomplete. Please consult -# http://ascii-table.com/ansi-escape-sequences-vt-100.php -# http://www.termsys.demon.co.uk/vtansi.htm - -### TODO: Figure out what is meant by [?0h ... [1;2;0h ? [120h - -from batchq.core.terminal.patternregister import PatternRegister -from batchq.core.terminal.base import BaseInterpreter - -VT100Register = PatternRegister() -KeyRegister = PatternRegister() - -class VT100Interpreter(BaseInterpreter): - """ - This object implements VT100 interpretation. - """ - def __init__(self): - super(VT100Interpreter, self).__init__(VT100Register, KeyRegister) -################### -### KEYS - @KeyRegister.register(r"\x00") - def null(self): - pass - - @KeyRegister.register(r"\x01") - def start_of_heading(self): - pass - - @KeyRegister.register(r"\x02") - def start_of_text(self): - pass - - @KeyRegister.register(r"\x03") - def end_of_text(self): - pass - - @KeyRegister.register(r"\x04") - def end_of_transmission(self): - pass - - @KeyRegister.register(r"\x05") - def enquiry(self): - pass - - @KeyRegister.register(r"\x06") - def acknowledge(self): - pass - - - @KeyRegister.register(r"\x07") - def bell(self): - pass - - @KeyRegister.register(r"\x08") - def backspace(self): - self.cursor_back() - - @KeyRegister.register(r"\x09") - def horisontal_tab(self): - self.put("\t") - - @KeyRegister.register(r"\x0A") - def newline(self): - self._curline+=1 - self.fix_buffer() - - @KeyRegister.register(r"\x0B") - def vertical_tab(self): - pass - - @KeyRegister.register(r"\x0C") - def newpage(self): - self._curline+=1 - self.fix_buffer() - - - @KeyRegister.register(r"\x0D") - def carriage_return(self): - self._curchar = 0 - self.fix_buffer() - - @KeyRegister.register(r"\x0E") - def shift_out(self): - pass - - @KeyRegister.register(r"\x0F") - def shift_in(self): - pass - - -############### -## Escape sequences - @VT100Register.register(r"\x1b\[\??(?P\d*)(?P(h|l))") - def set_mode (self,mode = None, property =None): # [? ... h or [? ... m or [? ...l - - if property == "m": - if mode is None or mode == 0: - self._attributes = {'bold':False, 'underline': False, 'blinking': False, 'low_intensity':False, 'invisible': False, 'reverse_video': False} - elif mode == 1: - self._attributes['bold'] = True - elif mode == 2: - self._attributes['low_intensity'] = True - elif mode == 4: - self._attributes['underline'] = True - elif mode == 5: - self._attributes['blinking'] = True - elif mode == 7: - self._attributes['reverse_video'] = True - elif mode == 8: - self._attributes['invisible'] = True - - - @VT100Register.register(r"\x1b\[(?P(\d+;)*\d*)m") - def set_display_attr (self,attributes=None): - pass -# print "XXXXXXXX::: Setting display attr", attributes - - - - @VT100Register.register(r"\x1b\>") - def set_numeric_keypad_mode (self): # > - pass - - @VT100Register.register(r"\x1b\=") - def set_alternate_keypad_mode (self): # = - pass - - @VT100Register.register(r"\x1b(?P(\(|\)))(?P(A|B|0|1|2))") - def set_character_set (self,paren,val): # = - pass - - - @VT100Register.register(r"\x1bN") - def set_single_shift2 (self): # N - pass - - @VT100Register.register(r"\x1bO") - def set_single_shift3 (self): # O - pass - - - @VT100Register.register(r"\x1b\[(?P\d+);(?P\d+)r") - def set_top_bottom_lines (self, line1, line2): - self._topline = line1 - self._bottomline = line2 - self.fix_buffer() - - - - - @VT100Register.register(r"\x1b\[(?P\d+)A",r"\x1b\[A") - def cursor_up (self,count=1): # [{COUNT}A ## NOT: or M - self._curline -= count - if self._curline < 0: self._curline = 0 - - - @VT100Register.register(r"\x1b\[(?P\d+)B",r"\x1b\[B") - def cursor_down (self,count=1): # [{COUNT}B (not confused with back) - - self._curline += count - n = len(self._lines) - if self._curline > n: self._lines += [""]*(self._curline - n) - - - @VT100Register.register(r"\x1b\[(?P\d+)C",r"\x1b\[C") - def cursor_forward (self,count=1): # [{COUNT}C - self._curchar += count - n = len(self._lines[self._curline]) - if self._curchar>n: self._lines[self._curline] += " "*(self._curchar - n) - - - @VT100Register.register(r"\x1b\[(?P\d+)D",r"\x1b\[D") - def cursor_back (self,count=1): # [{COUNT}D (not confused with down) - print "Cursor back" - self._curchar -= count - if self._curchar<0: self._curchar = 0 - - - - @VT100Register.register(r"\x1b\[(?P\d+);(?P\d+)H", r"\x1b\[;?H") - def cursor_home (self, line=1, char=1): - self._curline = line - self._curchar = char - self.fix_buffer() - - @VT100Register.register(r"\x1b\[(?P\d+);(?P\d+)f",r"\x1b\[;?f") - def cursor_force_position (self, line=1, char=1): # [{ROW};{COLUMN}f - self._curline = line - self._curchar = char - self.fix_buffer() - - @VT100Register.register(r"\x1b7") - def cursor_save_attrs (self): # 7 - self._home_line = self._curline - self._home_char = self._curchar - self._home_attributes = self._attributes - - @VT100Register.register(r"\x1b8") - def cursor_restore_attrs (self): # 8 - self._curline = self._home_line - self._curchar = self._home_char - self._attributes = self._home_attributes - self.fix_buffer() - - - - @VT100Register.register(r"\x1bH") - def set_tab (self): # H - pass - - @VT100Register.register(r"\x1b\[0?g") - def clear_tab (self): # [g - pass - - @VT100Register.register(r"\x1b\[3g") - def clear_all_tabs (self): # [3g - pass - - @VT100Register.register(r"(\x1b)#3") - def double_letters_height_top_half (self): # #3 - pass - - @VT100Register.register(r"(\x1b)#4") - def double_letters_height_bottom_half (self): # #4 - pass - - @VT100Register.register(r"(\x1b)#5") - def single_width (self): # #5 - pass - - @VT100Register.register(r"(\x1b)#6") - def double_width (self): # #6 - pass - - @VT100Register.register(r"(\x1b)#8") - def screen_alignment_display (self): # #8 - pass - - - - @VT100Register.register(r"\x1b\[0?K") - def erase_end_of_line (self): # [0K -or- [K - - line = self._lines[self._curline] - line = line[0:self._curchar] - self._lines[self._curline] = line - - @VT100Register.register(r"\x1b\[1K") - def erase_start_of_line (self): # [1K - - line = self._lines[self._curline] - line = line[self._curchar+1:len(line)] - self._lines[self._curline] = line - - @VT100Register.register(r"\x1b\[2K") - def erase_line (self): # [2K - - self._lines[self._curline+1] = "" - - @VT100Register.register(r"\x1b\[0?J") - def erase_down (self): # [0J -or- [J - n = len(self._lines) - self._lines = self._lines[0:self._curline] - - @VT100Register.register(r"\x1b\[1J") - def erase_up (self): # [1J - n = len(self._lines) - self._lines = self._lines[self._curline+1:n] - - @VT100Register.register(r"\x1b\[2J") - def erase_screen (self): # [2J - self._lines = [""] - self._curchar = 0 - self._curline = 0 - self.fix_buffer() - - ### TODO: Figure out which are terminal -> computer and vice versa - @VT100Register.register(r"(\x1b)5n") - def device_status_report (self): # 5n -# print "Reset device" - pass - - @VT100Register.register(r"(\x1b)0n") ## FIXME: RESPONSE TO PREVIOUS - def terminal_ok (self): # 0n -# print "Reset device" - pass - - @VT100Register.register(r"(\x1b)3n") ## FIXME: RESPONSE TO PREVIOUS - def terminal_not_ok (self): # 3n -# print "Reset device" - pass - - @VT100Register.register(r"(\x1b)6n") - def get_cursor_position (self): # 3n -# print "Reset device" - pass - - @VT100Register.register(r"\x1b(?P\d+);(?P\d+);") ## FIXME: RESPONSE - def respond_cursor_position (self): # 3n -# print "Reset device" - pass - - - - @VT100Register.register(r"\x1b\[c") - def identify_terminal (self): # c -# print "Reset device" - pass - - @VT100Register.register(r"\x1b\[0c") - def identify_terminal_another (self): # 0c -# print "Reset device" - pass - - @VT100Register.register(r"\x1b\[1;0c") ## FIXME: RESPONSE - def terminal_type_code (self): # 0c -# print "Reset device" - pass - - - - @VT100Register.register(r"(\x1b)c") - def reset_terminal (self): # c -# print "Reset device" - pass - - - @VT100Register.register(r"\x1b\[(?P\d+)q") - def switch_leds (self, val = 0): # 0q -# print "Reset device" - pass - - - - @VT100Register.register(r"\x1b\]0;.*\x07") - def unknown1 (self): # ]0; -# print "UNKNOWN: ]0;"%variable - pass - @VT100Register.register(r"\x1b\[(?P\d+)d") - def unknown2 (self, variable): # ]0; -# print "UNKNOWN: [%dd"%variable - pass - - @VT100Register.register(r"\x1b\[(?P\d+)G") - def unknown3 (self, variable): # ]0; -# print "UNKNOWN: [%dG"%variable - pass - - - - - - - - - -if __name__ == "__main__": - test = VT100Interpreter() - test.write("Hello0\x1b[1D world\n!!!\x1b[2D\x1b[1A3"); - diff --git a/batchq/queues/__init__.py~ b/batchq/queues/__init__.py~ deleted file mode 100644 index 396e5bc..0000000 --- a/batchq/queues/__init__.py~ +++ /dev/null @@ -1,9 +0,0 @@ -from batchq.core.library import Library -from batchq.queues.nohup import NoHUP,NoHUPSSH,LSFBSub,create_configuration,list,help -Library.queues.register("nohup",NoHUP) -Library.queues.register("ssh-nohup",NoHUPSSH) -Library.queues.register("lsf",LSFBSub) -Library.functions.register("configuration",create_configuration) -Library.functions.register("list",list) -Library.functions.register("help",help) - diff --git a/batchq/queues/lsf.py~ b/batchq/queues/lsf.py~ deleted file mode 100644 index dae1788..0000000 --- a/batchq/queues/lsf.py~ +++ /dev/null @@ -1,42 +0,0 @@ - -class LSFBSub(NoHUPSSH): - _1 = batch.WildCard() - _2 = batch.WildCard() - - options = batch.Property("", verbose = False) - - - ## TESTED AND WORKING - running = batch.Function(NoHUP.pid,verbose=True) \ - .Qjoin("bjobs ",_1," | awk '{ if($1 == ",_2,") {printf $3}}'").send_command(_1) \ - .Qstrip(_1).Qlower(_1).Qequal(_1,"run") - - ## TESTED AND WORKING - startjob = batch.Function(NoHUP.create_workdir,verbose=True) \ - .send_command(NoHUP.prior) \ - .Qcall(NoHUP.identifier_filename) \ - .Qjoin("(touch ",_1, " && bsub ", options," \"", NoHUP.command, " > ",_2," \" | awk '{ if(match($0,/([0-9]+)/)) { printf substr($0, RSTART,RLENGTH) } }' > .batchq.pid )") \ - .send_command(_1) - - ## TESTED AND WORKING - finished = batch.Function(NoHUP.pid,verbose=True).Qcontroller("terminal") \ - .Qjoin("bjobs ",_1," | awk '{ if($1 == ",_2,") {printf $3}}' ").send_command(_1) \ - .Qstrip(_1).Qlower(_1).Qequal(_1,"done") - - - failed = batch.Function(NoHUP.pid,verbose=True).Qcontroller("terminal") \ - .Qjoin("bjobs ",_1," | awk '{ if($1 == ",_2,") {printf $3}}' ").send_command(_1) \ - .Qstrip(_1).Qlower(_1).Qequal(_1,"exit") - - ## TESTED AND WORKING - pending = batch.Function(NoHUP.pid,verbose=True).Qcontroller("terminal") \ - .Qjoin("bjobs ",_1," | awk '{ if($1 == ",_2,") {printf $3}}' ").send_command(_1) \ - .Qstrip(_1).Qlower(_1).Qequal(_1,"pend") - - ## TESTED AND WORKING - status = batch.Function(NoHUP.pid,verbose=True).Qcontroller("terminal") \ - .Qjoin("bjobs ",_1," | awk '{ if($1 == ",_2,") {printf $3}}' ").send_command(_1) - - ## TODO: write this function - cancel = batch.Function().Qstr("TODO: This function needs to be implemented") - diff --git a/batchq/queues/nohup.py~ b/batchq/queues/nohup.py~ deleted file mode 100644 index 99cd2cd..0000000 --- a/batchq/queues/nohup.py~ +++ /dev/null @@ -1,293 +0,0 @@ -from batchq.core import batch -from batchq.pipelines.shell.bash import BashTerminal -from batchq.pipelines.shell.ssh import SSHTerminal -from batchq.pipelines.shell.utils import FileCommander -from batchq.shortcuts.shell import home_create_dir, send_command -from datetime import timedelta -import re - - -str_time_regex = re.compile(r'((?P\d+?)days?, *)?((?P\d+?)hr *)?((?P\d+?)m *)?((?P\d+?)s *)?') -def str_to_timedelta(s): - m = str_time_regex.search(s) - if not m: return timedelta(0) - kwargs = dict([(a,int(b)) for a,b in m.groupdict().iteritems() if b]) - return timedelta(**kwargs) - -def format_time(t): - if isinstance(t, str): - t = str_to_timedelta(t).total_seconds() - elif isinstance(t, timedelta): - t = t.total_seconds() - - - if t < 0: return -1 - if t/60 - t/60. == 0.: - return int(t/60) - return int(t/60+1) - -class NoHUP(batch.BatchQ): - - _r = batch.WildCard(reverse = True) - _ = batch.WildCard() - _2 = batch.WildCard() - _last = batch.WildCard( select = 0, reverse = True) - - command = batch.Property("echo Hello World > execution.txt",display="Command to execute: ") - working_directory = batch.Property(display="Specify a working directory: ") - input_directory = batch.Property(".", display="Specify an input directory: ") - output_directory = batch.Property(".", display="Specify an output directory: ") - - subdirectory = batch.Property(".", verbose = False) - prior = batch.Property("", verbose = False) - post = batch.Property("", verbose = False) - - server = batch.Property("localhost", display="Server: ", verbose=False, invariant = True) - processes = batch.Property(1, display="Number of processes: ", verbose = False, type = int) - time = batch.Property(-1, display="Wall time: ", verbose = False, formatter = format_time, type = int) - mpi = batch.Property(False, display="Use MPI: ", verbose = False, type = bool) - threads = batch.Property(1, display="OpenMP threads: ", verbose = False, type = int) - memory = batch.Property(-1, display="Max. memory: ", verbose = False, type = int) - diskspace = batch.Property(-1, display="Max. space: ", verbose = False, type = int) - - overwrite_nodename_with = batch.Property("", verbose = False) - overwrite_submission_id = batch.Property("", verbose = False) - - terminal = batch.Controller(BashTerminal) - - - #### STUFF FOR SYSTEM IDENTIFICATION - whoami = batch.Function() \ - .whoami() - - overwrite_nodename = batch.Function(verbose=True)\ - .overwrite_nodename(overwrite_nodename_with) - - nodename = batch.Function(verbose=True)\ - .nodename() - - nodeid = batch.Function(verbose=True) \ - .nodename().Qslugify(_) - - system_info = batch.Function(verbose=True) \ - .system_info() - - system_string = batch.Function(verbose=True) \ - .system_string() - - ## TESTED AND WORKING - hash_input = batch.Function(verbose=True,cache = 5) \ - .entrance().chdir(_).directory_hash(input_directory,True,True) \ - .Qslugify(command).Qjoin(_,"-",_) - - ## TESTED AND WORKING - identifier = batch.Function(cache = 5) \ - .Qslugify(command).Qstr(overwrite_submission_id).Qjoin(_,"-",_).Qstore("id") \ - .Qstr(overwrite_submission_id).Qequal("",_).Qdo(2).Qcall(hash_input).Qstore("id") \ - .Qget("id") - - identifier_filename = batch.Function(identifier,cache = 5).Qjoin(".batchq_",_) - - ## TESTED AND WORKING - get_subdirectory = batch.Function(verbose=True, cache = 5) \ - .Qstr(subdirectory).Qstore("subdir") \ - .Qequal(_,".").Qdo(3).Qcall(identifier_filename).Qget("id").Qstore("subdir") \ - .Qget("subdir") - - - ## TESTED AND WORKING - _create_workdir = batch.Function(verbose=True,cache=5) \ - .entrance().chdir(_) \ - .isdir(working_directory).Qdon(1).mkdir(working_directory, True) \ - .chdir(working_directory) \ - .Qcall(get_subdirectory) \ - .isdir(_).Qdon(2).Qget("subdir").mkdir(_, True) \ - .Qget("subdir").chdir(_).pwd().Qstore("workdir") - - create_workdir = batch.Function(verbose=True) \ - .Qcall(_create_workdir).Qget("workdir").chdir(_) \ - .Qget("workdir") - - ### TESTED AND WORKING - prepare_incopy = batch.Function(create_workdir, verbose=True) \ - .Qset("sync_cache",False) \ - .entrance().Qpjoin(_,input_directory).Qstore("indir") \ - .Qpjoin(_,"*").Qget("workdir") - - ### TESTED AND WORKING - prepare_outcopy = batch.Function(create_workdir, verbose=True) \ - .Qset("sync_cache",False) \ - .entrance().Qpjoin(_,output_directory).Qstore("outdir") \ - .Qget("workdir").Qpjoin(_,"*").Qget("outdir") - - ### TESTED AND WORKING - send = batch.Function(prepare_incopy , verbose=True, enduser=True, type=None) \ - .cp(_r, _r, True).Qdon(1).Qthrow("Failed to transfer files.") - - ### TESTED AND WORKING - recv = batch.Function(prepare_outcopy, verbose=True, enduser=True, type=None) \ - .cp(_r, _r).Qdon(1).Qthrow("Failed to transfer files.") - - ### TESTED AND WORKING - pid = batch.Function(create_workdir,verbose=True, enduser=True, cache=5, type=int) \ - .Qjoin(identifier_filename,".pid").Qstore("pid_file") \ - .isfile(_).Qdon(2).Qstr("-1").Qreturn() \ - .Qget("pid_file").cat(_) - - - - ### TESTED AND WORKING - pending = batch.Function(verbose=True, enduser=True, cache=5, type=bool) \ - .Qbool(False) - - ## TESTED AND WORKING - submitted = batch.Function(create_workdir,verbose=True, enduser=True, cache=5, type=bool) \ - .Qcall(identifier_filename).Qjoin(_,".pid").isfile(_) - - lazy_finished = batch.Function(create_workdir,verbose=True, cache=5) \ - .Qjoin(identifier_filename,".finished").isfile(_).Qdo(2).Qbool(True).Qreturn() \ - .Qbool(False) - - finished = batch.Function(lazy_finished,verbose=True, enduser=True, cache=5, type=bool) - - lazy_running = batch.Function(lazy_finished,verbose=True, cache=5) \ - .Qdo(2).Qbool(False).Qreturn() \ - .Qjoin(identifier_filename,".running").isfile(_).Qdo(2).Qbool(True).Qreturn() \ - .Qbool(False) - - running = batch.Function(lazy_running,verbose=True, enduser=True, cache=5, type=bool) \ - .Qdon(2).Qbool(False).Qreturn() \ - .Qcall(pid).Qstore("pid").Qequal(_,"-1").Qdo(2).Qbool(False).Qreturn() \ - .Qget("pid").isrunning(_) - - failed = batch.Function(lazy_finished, enduser=True, cache=5, type=bool) \ - .Qdon(2).Qbool(False).Qreturn() \ - .Qjoin(identifier_filename,".finished").cat(_).Qint(_).Qequal(_,0).Qnot(_) - - - - ## TODO: TEST - prepare_submission = batch.Function() \ - .Qstr(threads).Qjoin("export OMP_NUM_THREADS=",_).send_command(_) \ - .Qset("command_prepend","").Qbool(mpi).Qdo(3).Qstr(processes).Qjoin("mpirun -np ", _, " ").Qstore("command_prepend") - - ## TESTED AND WORKING - startjob = batch.Function(create_workdir,verbose=True) \ - .Qclear_cache() \ - .Qcall(prepare_submission) \ - .send_command(prior) \ - .Qget("command_prepend") \ - .Qcall(identifier_filename, 1) \ - .Qjoin("(touch ",_last, ".submitted ; touch ",_last,".running ; ", command, " 1> ",_last,".running 2> ",_last,".error && echo $! > ",_last,".pid ; echo $? > ",_last,".finished )") \ - .send_command(_) - - - ## TESTED AND WORKING - submit = batch.Function(send, verbose=True, enduser=True, type=None) \ - .Qcall(startjob) - - - ## TESTED AND WORKING - stdout = batch.Function(submitted,verbose=True, enduser=True, type=str) \ - .Qdo(2).Qcall(identifier_filename).cat(_) - - # TODO: - stderr = batch.Function(submitted,verbose=True, enduser=True, type=str) \ - .Qprint("not implemented yet") - - log = batch.Function(submitted,verbose=True, enduser=True, type=str) \ - .Qprint("No log implemented for SSH. Use stderr and stdout") - - - ## TESTED AND WORKING - clean = batch.Function(create_workdir,verbose=True, enduser=True, type=None) \ - .Qdo(1).rm(".batchq*", force = True) - - ## TESTED AND WORKING - delete_working_directory = batch.Function(create_workdir,verbose=True, enduser=True, type=None) \ - .Qdo(2).Qget("workdir").rm(_, force = True, recursively = True) - - ## TODO: TEST - cancel = batch.Function(pid, verbose=True, enduser=True, type=None).kill(_) - - ### TESTED AND WORKING - status = batch.Function(verbose=True, enduser=True, type=str) \ - .Qcall(submitted).Qdon(2).Qstr("was not submitted").Qreturn() \ - .Qcall(running).Qdo(2).Qstr("running").Qreturn() \ - .Qcall(finished).Qdo(2).Qstr("finished").Qreturn() \ - .Qcall(pending).Qdo(2).Qstr("pending").Qreturn() \ - .Qcall(failed).Qdo(2).Qstr("failed").Qreturn() \ - .Qcall(submitted).Qdo(2).Qstr("pre-pending").Qreturn() \ - .Qstr("unknown status") - - - run_job = batch.Function(verbose=True, enduser=True,type=batch.FunctionMessage) \ - .Qcall(submitted).Qdon(6).Qprint("Uploading input directory:",input_directory,"->",create_workdir).Qcall(send).Qprint("Submitting job on ",server).Qcall(startjob).Qstr("").Qreturn(1,"Submitting job on ", server) \ - .Qcall(pending).Qdo(3).Qprint("Job is pending.").Qreturn(3, "Job is pending.") \ - .Qcall(running).Qdo(3).Qprint("Job is running.").Qreturn(4, "Job is running.") \ - .Qcall(failed).Qdo(3).Qprint("Job has failed:\n\n").Qcall(log).Qreturn(-5, _) \ - .Qcall(finished).Qdo(8).Qprint("Job has finished.").Qcall(recv).Qdo(1).Qprint("Using cache.").Qdon(1).Qprint("Files retrieved.").Qreturn(0) \ - .Qcall(submitted).Qdo(3).Qprint("Job is pre-pending (i.e. submitted but not in the batch system).").Qreturn(2,"Job is pre-pending (i.e. submitted but not in the batch system).") \ - .Qprint("Your job has an unknown status.").Qreturn(-1,"Your job has an unkown status. This is usually a bad thing and you should probably file a bug report.") - - -class NoHUPSSH(NoHUP): - _ = batch.WildCard() - _r = batch.WildCard(reverse=True) - - username = batch.Property(display="Username: ", invariant = True) - password = batch.Property(password = True, display="Password: ", invariant = True) - server = batch.Property("localhost", display="Server: ", invariant = True) - port = batch.Property(22, display="Port: ", invariant = True) - - filecommander = batch.Controller(FileCommander,server,username, password, port) - local_terminal = batch.Controller(BashTerminal,q_instance = filecommander.local) - terminal = batch.Controller(BashTerminal,q_instance = filecommander.remote) - - reconnect = batch.Function(enduser=True, type=None) \ - .connection_lost().Qdo(1).connect(server,username, password, port) \ - .Qcontroller("filecommander") \ - .connection_lost().Qdo(1).connect(server,username, password, port) - -# hash_input_directory = batch.Function(verbose=True) \ -# .Qcontroller("local_terminal") \ -# .entrance().chdir(_).directory_hash(NoHUP.input_directory,True,True) - - ## TESTED AND WORKING - hash_input = batch.Function(verbose=True).Qcontroller("local_terminal") \ - .entrance().chdir(_).directory_hash(NoHUP.input_directory,True,True) \ - .Qdo(2).Qslugify(NoHUP.command).Qjoin(_,"-",_) - - ## TESTED AND WORKING -# hash_output = batch.Function(verbose=True).Qcontroller("local_terminal") \ -# .entrance().chdir(_).directory_hash(NoHUP.output_directory,True,True) \ -# .Qdo(2).Qslugify(NoHUP.command).Qjoin(_,"-",_,"-",_,"-",_) - - ## TESTED AND WORKING - prepare_incopy = batch.Function(NoHUP.create_workdir) \ - .Qset("sync_cache",False) \ - .Qcontroller("local_terminal") \ - .pwd().Qpjoin(_,NoHUP.input_directory).Qstore("indir") \ - .Qget("workdir") - - ## TESTED AND WORKING - prepare_outcopy = batch.Function(NoHUP.create_workdir).Qcontroller("local_terminal") \ - .Qset("sync_cache",False) \ - .push_entrance().isdir(NoHUP.output_directory).Qdon(1).mkdir(NoHUP.output_directory, True).popd() \ - .entrance().Qpjoin(_,NoHUP.output_directory).Qstore("outdir") \ - .entrance().Qpjoin(_,NoHUP.input_directory).Qstore("indir") \ - .Qget("outdir").Qget("workdir").Qget("indir") - - ## TESTED AND WORKING - send = batch.Function(prepare_incopy , verbose=True) \ - .Qcontroller("filecommander").sync(_r,_r, mode = FileCommander.MODE_LOCAL_REMOTE) \ - .Qequal(_, False).Qdo(1).Qset("sync_cache",True) \ - .Qget("sync_cache") - - ## TESTED AND WORKING - recv = batch.Function(prepare_outcopy , verbose=True) \ - .Qcontroller("filecommander").sync(_r,_r, mode = FileCommander.MODE_REMOTE_LOCAL, diff_local_dir =_r) \ - .Qequal(_, False).Qdo(1).Qset("sync_cache",True) \ - .Qget("sync_cache") - - diff --git a/docs/.DS_Store b/docs/.DS_Store deleted file mode 100644 index cd0f04f..0000000 Binary files a/docs/.DS_Store and /dev/null differ