diff --git a/HIVEapi/HIVEWebAPIVersion0.pdf b/HIVEapi/HIVEWebAPIVersion0.pdf new file mode 100644 index 0000000..b35f930 Binary files /dev/null and b/HIVEapi/HIVEWebAPIVersion0.pdf differ diff --git a/HIVEapi/apiHIVE.py b/HIVEapi/apiHIVE.py new file mode 100755 index 0000000..14ba8fa --- /dev/null +++ b/HIVEapi/apiHIVE.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3.5 +# -*- coding: utf-8 -*- + +################################################################################ + ##docHIVE## +""""gets a file based on HIVE id""" +################################################################################ + +from cookielib import CookieJar +import urllib, urllib2, re, csv, requests +from urllib2 import URLError +#from bs4 import BeautifulSoup + +hive_URL = 'https://hive.biochemistry.gwu.edu/dna.cgi?' +email = '' +pswd = '' +cj = CookieJar() +opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj)) +log_in = 'api=0&cmdr=login&login='+email+'&pswd='+pswd +response = opener.open(hive_URL, log_in) + + +print "You are logged in to the HIVE..." + +valuz = [ + ('cmdr','alCount'), + ('objs','540758'), + ('down','1') +] + +comand = urllib.urlencode(valuz) +response2 = opener.open(hive_URL, comand) +print response2.read() + +# cmdr=objFile +# ids= +# filename= +# propname= +# +# https://hive.biochemistry.gwu.edu/dna.cgi? +# cmd=alCount +# start=0 +# cnt=0 +# objs=540758 +# down=1 \ No newline at end of file diff --git a/HIVEapi/censusHIVE.py b/HIVEapi/censusHIVE.py new file mode 100755 index 0000000..b6e115c --- /dev/null +++ b/HIVEapi/censusHIVE.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3.5 +# -*- coding: utf-8 -*- + +################################################################################ + ##censusHIVE## +""""Submit a CensuScopr Job""" +################################################################################ + +from cookielib import CookieJar +import urllib, urllib2, re, csv, requests, json, os +from urllib2 import URLError +from bs4 import BeautifulSoup + +hive_URL = 'https://hive.biochemistry.gwu.edu/dna.cgi?' +os.system("stty -echo") +email = raw_input('Enter Email:') +pwd = raw_input('Enter Password:') +os.system("stty echo") +cj = CookieJar() +opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj)) +log_in = str('api=0&cmdr=login&login='+email+'&pswd='+pwd) +response = opener.open(hive_URL, log_in) + +print "You are logged in to the HIVE..." + +valuz = [ +('prop.svc-align-blast.name.1','filteredHMB01-1-contig.fa versus hgdbv1.6'), +('prop.svc-align-blast.alignSelector.36','svc-align-blast'), +('prop.svc-align-blast.force_reindex.37','0'), +('prop.svc-align-blast.submitter.40','dna-hexagon&cmdMode','blast-bacteria'), +('prop.svc-align-blast.maxNumberQuery.15','all'), +('prop.svc-align-blast.batch_svc.6','single'), +('prop.svc-align-blast.blastProgram.7','blastn'), +('prop.svc-align-blast.evalueFilter.12','1e-4'), +('prop.svc-align-blast.cmdLine.9','-task megablast -num_threads 1'), +('prop.svc-align-blast.scissors.26','hiveseq'), +('prop.svc-align-blast.complexityRefEntropy.10.1.0','1.2'), +('prop.svc-align-blast.complexityRefWindow.10.1.0','30'), +('prop.svc-align-blast.acceptNNNQuaTrheshold.10.1.0','1'), +('prop.svc-align-blast.complexityEntropy.10.0.0','1.2'), +('prop.svc-align-blast.complexityWindow.10.0.0','30'), +('prop.svc-align-blast.maximumPercentLowQualityAllowed.10.0.0','0'), +('prop.svc-align-blast.keepAllMatches.13','4'), +('prop.svc-align-blast.maxHitsPerRead.14','200'), +('prop.svc-align-blast.minMatchLen.16.0','75'), +('prop.svc-align-blast.minMatchUnit.16.0','0'), +('prop.svc-align-blast.maxMissQueryPercent.17.0','15'), +('prop.svc-align-blast.considerGoodSubalignments.17.0','1'), +('prop.svc-align-blast.num_alignments.19','10'), +('prop.svc-align-blast.random_seed.23','0'), +('prop.svc-align-blast.resolveConflicts.24.1','2'), +('prop.svc-align-blast.resolveConflictsScore.24.2','2'), +('prop.svc-align-blast.resolveConfictsUnique.24.0','1'), +('prop.svc-align-blast.scoreFilter.27','None'), +('prop.svc-align-blast.splitType.31.3','sequences'), +('prop.svc-align-blast.nrepeat.31.0','1'), +('prop.svc-align-blast.splitField.31.1','query'), +('prop.svc-align-blast.splitSize.31.2','4000'), +('prop.svc-align-blast.isPostponed.34.3','0'), +('prop.svc-align-blast.reqPriority.34.7','0'), +('prop.svc-align-blast.split.30','query'), +('prop.svc-align-blast.seedSize.28','28'), +('prop.svc-align-blast.slice.29','4000'), +('prop.svc-align-blast.query.1.1','557823'), +('prop.svc-align-blast.subject.1.1','557831'), +('prop.svc-align-blast.svc','dna-alignx') +] + +comand = urllib.urlencode(valuz) +print comand +response2 = opener.open(hive_URL, comand) +new_id = response2.read() +new_id = new_id.split(',')[1] +print new_id + diff --git a/HIVEapi/getDT_HIVE.py b/HIVEapi/getDT_HIVE.py new file mode 100755 index 0000000..994ee30 --- /dev/null +++ b/HIVEapi/getDT_HIVE.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3.5 +# -*- coding: utf-8 -*- + +################################################################################ + ##getDT_HIVE## +""""gets a DataType in JSON""" +################################################################################ + +from cookielib import CookieJar +import urllib, urllib2, re, csv, requests +hive_URL = 'https://hive.biochemistry.gwu.edu/dna.cgi?' +hiveType = raw_input('Name of type?\n') +f = str('def.'+hiveType+'.json') + +writer = open(f, 'w') + +email = raw_input('Enter Email:') +pwd = raw_input('Enter Password:') +cj = CookieJar() +opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj)) +log_in = 'api=0&cmdr=login&login='+email+'&pswd='+pswd +response = opener.open(hive_URL, log_in) +(hive_URL, log_in) + +print "You are logged in to the HIVE..." + +valuz = [ + ('cmdr','propspec'), + ('type',hiveType), + ('mode','json') +] + +comand = urllib.urlencode(valuz) +response2 = opener.open(hive_URL, comand) +writer.write(response2.read()) \ No newline at end of file diff --git a/HIVEapi/hiveSeq.py b/HIVEapi/hiveSeq.py new file mode 100755 index 0000000..84bf567 --- /dev/null +++ b/HIVEapi/hiveSeq.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3.5 +# -*- coding: utf-8 -*- + +################################################################################ + ##apiHIVE## +""""returns CensuScope HitList file""" +################################################################################ + +from cookielib import CookieJar +import urllib, urllib2, re, csv, requests, os +from urllib2 import URLError +from bs4 import BeautifulSoup + +hive_URL = 'https://hive.biochemistry.gwu.edu/dna.cgi?' +os.system("stty -echo") +email = = raw_input('Enter Email:') +pwd = raw_input('Enter Password:') +os.system("stty echo") +cj = CookieJar() +opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj)) +log_in = str('api=0&cmdr=login&login='+email+'&pswd='+pwd) +response = opener.open(hive_URL, log_in) + +print "You are logged in to the HIVE..." + +hiveseqQry = '515362,1,1;515593,1,1;516397,1,1;516398,1,1;516690,1,1;516692,1,1;523037,1,1;532707,1,1;536714,1,1;538148,1,1;538163,1,1;538175,1,1;538322,1,1;538326,1,1;538364,1,1;538507,1,1;538617,1,1;538619,1,1;538620,1,1;538623,1,1;538625,1,1;538626,1,1;538629,1,1;538630,1,1;538633,1,1;538634,1,1;538637,1,1;538638,1,1;538641,1,1;538642,1,1;538645,1,1;538646,1,1;538649,1,1;538650,1,1;538652,1,1;538654,1,1;538656,1,1;538658,1,1;538662,1,1;538665,1,1;538666,1,1;538668,1,1;538671,1,1;538673,1,1;538674,1,1;538689,1,1;538690,1,1;538691,1,1;538692,1,1;538697,1,1;538698,1,1;538699,1,1;538701,1,1;538705,1,1;538706,1,1;538707,1,1;538708,1,1;538714,1,1;538715,1,1;538717,1,1;538722,1,1;538723,1,1;538724,1,1;538725,1,1;538730,1,1;538731,1,1;538732,1,1;538734,1,1;538738,1,1;538739,1,1;538740,1,1;538741,1,1;538746,1,1;538747,1,1;538748,1,1;538749,1,1;538754,1,1;538755,1,1;538757,1,1;538758,1,1;538883,1,1;538885,1,1;538989,1,1;538995,1,1;539002,1,1;539006,1,1;539019,1,1;539323,1,1;539455,1,1;539457,1,1;539463,1,1;539464,1,1;539465,1,1;539470,1,1;539547,1,1;539548,1,1;539555,1,1;539556,1,1;539600,1,1;539602,1,1;539610,1,1;539618,1,1;539620,1,1;539624,1,1;539625,1,1;539629,1,1;539632,1,1;539657,1,1;539693,1,1;539696,1,1;539737,1,1;539738,1,1;539740,1,1;539745,1,1;539749,1,1;539752,1,1;539753,1,1;539754,1,1;539756,1,1;539760,1,1;539769,1,1;539837,1,1;539867,1,1;539894,1,1;539896,1,1;540017,1,1;540018,1,1;540019,1,1;540020,1,1;540025,1,1;540026,1,1;540028,1,1;540035,1,1;540041,1,1;540102,1,1;540173,1,1;540362,1,1;540368,1,1;540369,1,1;540530,1,1;540532,1,1;540584,1,1;540588,1,1;540652,1,1;540653,1,1;540791,1,1;540976,1,1;541019,1,1;546774,1,1;546776,1,1;546777,1,1;546778,1,1;546782,1,1;553136,1,1;553362,1,1;553364,1,1;553365,1,1;553368,1,1;553370,1,1;553371,1,1;553372,1,1;553375,1,1;553378,1,1;553380,1,1;553382,1,1;553383,1,1;553386,1,1;553387,1,1;553388,1,1;553389,1,1;553390,1,1;553391,1,1;' + +valuz = [ +('cmdr','-qpProcSubmit'), +('svc','dna-hiveseq'), +('raw','1'), +('prop.svc-hiveseq.name.1','testHiveseqName'), +('prop.svc-hiveseq.AlgorithmsName.11.0','0'), +('prop.svc-hiveseq.lowcomplexityactive.12.0','0'), +('prop.svc-hiveseq.lowcomplexityWindow.12.0','15'), +('prop.svc-hiveseq.lowcomplexityEntropy.12.0','1'), +('prop.svc-hiveseq.lowcomplexityOption.12.0','0'), +('prop.svc-hiveseq.AdaptersFilter.13','0'), +('prop.svc-hiveseq.adaptersactive.13.0','0'), +('prop.svc-hiveseq.adaptersObjId.13.0','0'), +('prop.svc-hiveseq.adaptersReverse.13.0.0','0'), +('prop.svc-hiveseq.adaptersComplement.13.0.1','0'), +('prop.svc-hiveseq.adaptersMaxmissmatches.13.0','2'), +('prop.svc-hiveseq.adaptersMinLength.13.0','10'), +('prop.svc-hiveseq.primersactive.17.0','0'), +('prop.svc-hiveseq.primersReverse.17.0.0','0'), +('prop.svc-hiveseq.primersComplement.17.0.1','0'), +('prop.svc-hiveseq.primersKeep.17.0','0'), +('prop.svc-hiveseq.primersMaxmissmatches.17.0','2'), +('prop.svc-hiveseq.primersMinLength.17.0','100'), +('prop.svc-hiveseq.QualityFilter.19','0'), +('prop.svc-hiveseq.qualityactive.19.0','0'), +('prop.svc-hiveseq.qualityPercentage.19.0','100'), +('prop.svc-hiveseq.qualityThreshold.19.0','50'), +('prop.svc-hiveseq.trimMinimum.20.0','-1'), +('prop.svc-hiveseq.trimMaximum.20.0','-1'), +('prop.svc-hiveseq.removeMin.21.0.0','-1'), +('prop.svc-hiveseq.removeMax.21.0.0','-1'), +('prop.svc-hiveseq.lengthSeqFilter.16','0'), +('prop.svc-hiveseq.minimumSeqLength.16.0','0'), +('prop.svc-hiveseq.idfilteractive.15.0','0'), +('prop.svc-hiveseq.idlistObjId.15.0','0'), +('prop.svc-hiveseq.listExclusion.15.0','0'), +('prop.svc-hiveseq.randomizerNumValue.22.0','100'), +('prop.svc-hiveseq.randomizerNoise.22.1','0'), +('prop.svc-hiveseq.randomizerMinLength.22.2','100'), +('prop.svc-hiveseq.randomizerMaxLength.22.3','100'), +('prop.svc-hiveseq.randomizerRevComp.22.6','0'), +('prop.svc-hiveseq.randLengthNorm.22.7','1'), +('prop.svc-hiveseq.seqExclusionOption.23.1','0'), +('prop.svc-hiveseq.seqExclusionRevComp.23.2','0'), +('prop.svc-hiveseq.hiveseqQry.24',hiveseqQry), +('prop.svc-hiveseq.inputMode.25','0'), +('prop.svc-hiveseq.pecPairEndFile.18.0','0'), +('prop.svc-hiveseq.pecMinMatchLen.18.0','15'), +('prop.svc-hiveseq.pecQualities.18.0','1'), +('prop.svc-hiveseq.pecKeepAlignments.18.0','0'), +('prop.svc-hiveseq.denovoExtSizemer.14.0','16'), +('prop.svc-hiveseq.denovoExtSizeFilter.14.0','0'), +('prop.svc-hiveseq.denovoExtRptFilter.14.0','0'), +('prop.svc-hiveseq.denovoExtFirstStage.14.0','1'), +('prop.svc-hiveseq.denovoOutFilterLength.14.0','1000'), +('prop.svc-hiveseq.denovoMissmatchesPercentage.14.0','2'), +('prop.svc-hiveseq.maxNumberQueryForHiveseq.25','-1'), +('prop.svc-hiveseq.inputMode.27.0','1'), # +('prop.svc-hiveseq.isFastQFile.26.0','1'), +('prop.svc-hiveseq.keepOriginalID.26.1','1'), +('prop.svc-hiveseq.appendLength.26.2','1'), +('prop.svc-hiveseq.isHiveseq.26.3','0'), +('prop.svc-hiveseq.isRevComp.26.4','0'), +('prop.svc-hiveseq.submitter.29','dna-hiveseq'), +('prop.svc-hiveseq._type','svc-hiveseq'), +('prop.svc-hiveseq.parent_proc_ids.9','3101923'), +('prop.svc-hiveseq.action.10.0','2'), +('prop.svc-hiveseq.isPostponed.10.3','0'), +('prop.svc-hiveseq.reqPriority.10.7','0') +] + +comand = urllib.urlencode(valuz) +print comand +response2 = opener.open(hive_URL, comand) +new_id = response2.read() +print new_id \ No newline at end of file diff --git a/HIVEapi/list_censusHIVE.py b/HIVEapi/list_censusHIVE.py new file mode 100755 index 0000000..853b6fc --- /dev/null +++ b/HIVEapi/list_censusHIVE.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3.5 +# -*- coding: utf-8 -*- + +################################################################################ + ##list_censusHIVE## +""""Submit a CensuScopr Job and waits for completion before submitting the next one in a list""" +################################################################################ + +from cookielib import CookieJar +import urllib, urllib2, re, csv, requests, time, os, json + + +login = raw_input('Enter Email:') +os.system("stty -echo") +pwd = raw_input('Enter Password:') +os.system("stty echo") +lstfile = '/Users/hadleyking/Downloads/reads_ids.csv' +gutDB = '513957' +hive_URL = 'https://hive.biochemistry.gwu.edu/dna.cgi?' +opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(CookieJar())) + +#______________________________________________________________________________# +def logInHIVE ( login, pwd ): + """Login to HIVE """ + log_in = str('api=0&cmdr=login&login='+login+'&pswd='+pwd) + response = opener.open(hive_URL, log_in) + print "login successfull" + return response +#______________________________________________________________________________# +def readIDlist ( file ): + """reads list of Obj Ids""" + samples = {} + with open(file, 'rU') as lst: + read = csv.reader(lst) + for i in read: + sample = i[1].split('_')[0] + objid = i[0] + if sample in samples.keys(): samples[sample].append(objid) + else: samples[sample] = [objid] + return samples +#______________________________________________________________________________# +def submitCensus ( sample, query1, query2, nt_db ): + """Submits the API request for a CensuScope computation""" + print "Submitting CensuScope Process\n" + censuName = sample + valuzCensus = [ + ('cmdr','-qpProcSubmit'), + ('svc','svc-dna-screening'), + ('prop.svc-dna-screening.action','2'), + ('prop.svc-dna-screening._type','svc-dna-screening'), + ('prop.svc-dna-screening.alignSelector.21','svc-align-blast'), + ('prop.svc-dna-screening.automanual.22','0'), + ('prop.svc-dna-screening.batch_svc.5','single'), + ('prop.svc-dna-screening.blastWordSize.29','28'), + ('prop.svc-dna-screening.CensusIterations.6','1'), + ('prop.svc-dna-screening.CensuslimitIterations.7','5'), + ('prop.svc-dna-screening.chunk_size.9','4000'), + ('prop.svc-dna-screening.cutOffvalue.11','0.0005'), + ('prop.svc-dna-screening.filterNs.12','1'), + ('prop.svc-dna-screening.isPostponed.20.3','0'), + ('prop.svc-dna-screening.name.1',censuName), + ('prop.svc-dna-screening.query.1.1',query1), + ('prop.svc-dna-screening.query.1.23',query2), + ('prop.svc-dna-screening.reqPriority.20.7','0'), + ('prop.svc-dna-screening.resultInQueryDir.26','0'), + ('prop.svc-dna-screening.Sample.15','2500'), + ('prop.svc-dna-screening.scissors.16','hiveseq'), + ('prop.svc-dna-screening.selfStopping.27','0'), + ('prop.svc-dna-screening.slice.17','0'), + ('prop.svc-dna-screening.split.18','-'), + ('prop.svc-dna-screening.storeAlignments.28','1'), + ('prop.svc-dna-screening.subject',nt_db), + ('prop.svc-dna-screening.submitter.33','dna-screening'), + ('prop.svc-dna-screening.svc','dna-screening'), + ('prop.svc-dna-screening.svcTitle','CensuScope'), + ('prop.svc-dna-screening.systemVersion.20.11','1.1'), + ('prop.svc-dna-screening.taxDepth.25','leaf'), + ('prop.svc-dna-screening.textBasedColumn.31.1','0'), + ('prop.svc-dna-screening.textBasedFileSeparator.31.2','1') + ] + + comandCensus = urllib.urlencode(valuzCensus) +# print comandCensus + response2 = opener.open(hive_URL, comandCensus) + census_id = response2.read() + census_id = census_id.split(',')[1] + print "Your CensuScope ID is: ",census_id + return census_id +#______________________________________________________________________________# +def checkProcess ( objID ): + """Takes an object Id and checks for process status. Will loop until process is complete.""" + + valuzCheck_census = [ + ('cmdr','-qpRawCheck'), + ('showreqs','0'), + ('reqObjID', objID) + ] + comandCheck = urllib.urlencode(valuzCheck_census) + stat = 0 + while stat <= 5: + print "Progress report on object", objID + response3 = opener.open(hive_URL, comandCheck) + check = response3.read() + stat = int(check.split('\n')[1].split(',')[6]) + prog = int(check.split('\n')[1].split(',')[8]) + if stat == 5: print "DONE!"; stat = 6; continue + elif stat == 4: print "Process in Line" + elif stat == 3: print "Process Running.", prog, "percent complete" + elif stat == 2: print "Dont care" + elif stat == 1: print "Process in submision" + else: print "PROGRAM ERROR!!!!"; break + time.sleep(120) +#______________________________________________________________________________# +def getHitList ( objID, sample ): + """Uses ObjID to retreive the Census Hit List""" + print "Getting Hit List from CensuScope...\n" + + valuzCensusList =[ + ('cmdr','alCount'), + ('objs',objID) + ] + + comand = urllib.urlencode(valuzCensusList) + response4 = opener.open(hive_URL, comand) + censusHit = response4.read() + hitListCen = 'cen'+sample+'.csv' + with open(hitListCen, 'w') as cenHitLst: + cenHitLst.write(censusHit) + print hitListCen, ' written to file' + lines = censusHit.split('\n') +#______________________________________________________________________________# +def main (): + response = logInHIVE (login,pwd) + objDict = readIDlist(lstfile) + for sample in objDict.keys(): + print sample, objDict[sample][1], objDict[sample][0] + census_id = submitCensus(sample, objDict[sample][1], objDict[sample][0], gutDB) + checkProcess (census_id) + getHitList (census_id, sample) +#______________________________________________________________________________# +if __name__ == '__main__': main() \ No newline at end of file diff --git a/HIVEapi/nameChange.py b/HIVEapi/nameChange.py new file mode 100755 index 0000000..f043f29 --- /dev/null +++ b/HIVEapi/nameChange.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3.5 +# -*- coding: utf-8 -*- + +################################################################################ + ##nameChange## +""""Takes a csv of [objectIds,Names] and submits a 'propset' request to update names""" +################################################################################ + +from cookielib import CookieJar +import urllib, urllib2, re, csv, requests, time, os, json + + +login = raw_input('Enter Email:') +os.system("stty -echo") +pwd = raw_input('Enter Password:') +os.system("stty echo") +lstfile = '/Users/hadleyking/Downloads/reads_ids.csv' +gutDB = '513957' +hive_URL = 'https://hive.biochemistry.gwu.edu/dna.cgi?' +opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(CookieJar())) + +#______________________________________________________________________________# +def logInHIVE ( login, pwd ): + """Login to HIVE """ + log_in = str('api=0&cmdr=login&login='+login+'&pswd='+pwd) + response = opener.open(hive_URL, log_in) + print "login successfull" + return response +#______________________________________________________________________________# +def nameChange ( file ): + """submits propset change for i in list""" + ids = {} + with open(file, 'rU') as lst: + read = csv.reader(lst) + for i in read: + sample, objid = i[1], i[0] + print sample, objid + valuzPropSet =[ + ('cmdr','propset'), + ('prop.'+str(objid)+'.name.1', sample) + ] + comand = urllib.urlencode(valuzPropSet) + response4 = opener.open(hive_URL, comand) +#______________________________________________________________________________# +def main (): + response = logInHIVE (login,pwd) + nameChange(lstfile) +#______________________________________________________________________________# +if __name__ == '__main__': main() \ No newline at end of file