diff --git a/errorScreenshots.JPG b/errorScreenshots.JPG new file mode 100644 index 0000000..a43aa1c Binary files /dev/null and b/errorScreenshots.JPG differ diff --git a/git commands.txt b/git commands.txt new file mode 100644 index 0000000..eb0e1d4 --- /dev/null +++ b/git commands.txt @@ -0,0 +1,44 @@ +touch filename.txt +--put timestamp on existed file or create file if file is not existed + +--first time setup, to set up user name and password, only for new installation +git config --global user.name "YOUR_FULL_NAME" +git config --global user.email "YOUR_EMAIL_ADDRESS" + + +git init +--initalize the git tracking for current folder + +git status +=--to track the status of the folder + +git add radsketch.pde +--add file to be version control + + git commit -m "Created a triangle in a circle" +--add a commit comment + +git remote add origin https://github.com/zx2160/edav.git + +git remote add mike https://github.com/malecki/edav.git +git fetch mike + +git remote rm mike + +git commit -m "commit comment here" +git remote -v +-- to see all remote + +git push origin master + +git log + +git reset HEAD filename to put it back to untracked file from stage file +--like undo the git add + +git add . +---add unstage changes to stage area +git branch---to see all the branches out there +git branch newbranchname --to create a new branch +git check branchname -- to switch branch +git branch -b \ No newline at end of file diff --git a/projects/darrenxing/ProjectPlan.docx b/projects/darrenxing/ProjectPlan.docx new file mode 100644 index 0000000..7875ee9 Binary files /dev/null and b/projects/darrenxing/ProjectPlan.docx differ diff --git a/projects/darrenxing/USpop.twbx b/projects/darrenxing/USpop.twbx new file mode 100644 index 0000000..0947e1e Binary files /dev/null and b/projects/darrenxing/USpop.twbx differ diff --git a/projects/darrenxing/citypop.csv b/projects/darrenxing/citypop.csv new file mode 100644 index 0000000..f45267d --- /dev/null +++ b/projects/darrenxing/citypop.csv @@ -0,0 +1,461 @@ +year,rank,city,pop +1790,1,Philadelphia,44.1 +1790,2,New York,33.1 +1790,3,Boston,18.3 +1790,4,Charleston,16.4 +1790,5,"Salem, MA",13.6 +1790,6,Baltimore,13.5 +1790,7,"Newport, RI",6.7 +1790,8,Providence,6.4 +1790,9,"Gloucester, MA",5.3 +1790,10,"Newburyport, MA",4.8 +1790,11,"Portsmouth, NH",4.7 +1790,12,Nantucket,4.6 +1790,13,"Middleborough, MA",4.5 +1790,14,New Haven,4.5 +1790,15,Richmond,3.8 +1790,16,Albany,3.5 +1790,17,Norfolk,3 +1790,18,"Petersburg, VA",2.8 +1790,19,"Alexandria, VA",2.8 +1790,20,Hartford,2.7 +1800,1,Philadelphia,61.6 +1800,2,New York,60.5 +1800,3,Baltimore,26.5 +1800,4,Boston,24.9 +1800,5,Charleston,18.8 +1800,6,"Salem, MA",14.7 +1800,7,Washington,11.2 +1800,8,Providence,7.6 +1800,9,Norfolk,6.9 +1800,10,"Newport, RI",6.7 +1800,11,"Newburyport, MA",6 +1800,12,Richmond,5.7 +1800,13,Nantucket,5.6 +1800,14,"Portsmouth, NH",5.3 +1800,15,"Gloucester, MA",5.3 +1800,16,"Schenectady, NY",5.3 +1800,17,Albany,5.3 +1800,18,"New London, CT",5.2 +1800,19,Savannah,5.2 +1800,20,"Middleborough, MA",4.5 +1810,1,New York,101 +1810,2,Philadelphia,87.3 +1810,3,Baltimore,46.6 +1810,4,Boston,38.7 +1810,5,Charleston,24.7 +1810,6,"Salem, MA",23.1 +1810,7,Washington,20.4 +1810,8,New Orleans,17.2 +1810,9,Albany,10.8 +1810,10,Providence,10.1 +1810,11,Richmond,9.7 +1810,12,Norfolk,9.2 +1810,13,"Newport, RI",7.9 +1810,14,"Newburyport, MA",7.6 +1810,15,"Portland, ME",7.2 +1810,16,"Portsmouth, NH",6.9 +1810,17,Nantucket,6.8 +1810,18,"Gloucester, MA",5.9 +1810,19,"Schenectady, NY",5.9 +1810,20,New Haven,5.8 +1820,1,New York,131 +1820,2,Philadelphia,109 +1820,3,Baltimore,62.7 +1820,4,Boston,54 +1820,5,Washington,28.8 +1820,6,New Orleans,27.2 +1820,7,Charleston,24.8 +1820,8,"Salem, MA",22.6 +1820,9,Albany,17.9 +1820,10,Richmond,12.1 +1820,11,Providence,11.8 +1820,12,Cincinnati,9.6 +1820,13,"Portland, ME",8.6 +1820,14,Norfolk,8.5 +1820,15,Savannah,7.5 +1820,16,"Portsmouth, NH",7.3 +1820,17,"Newport, RI",7.3 +1820,18,Nantucket,7.3 +1820,19,Pittsburgh,7.3 +1820,20,New Haven,7.2 +1830,1,New York,215 +1830,2,Philadelphia,161 +1830,3,Boston,85.6 +1830,4,Baltimore,80.6 +1830,5,New Orleans,46.1 +1830,6,Albany,35.8 +1830,7,Washington,35.5 +1830,8,Charleston,30.3 +1830,9,"Salem, MA",27.3 +1830,10,Cincinnati,24.8 +1830,11,Providence,22.4 +1830,12,Richmond,16.1 +1830,13,Pittsburgh,15.4 +1830,14,Newark,14.4 +1830,15,"Portland, ME",12.6 +1830,16,Louisville,10.3 +1830,17,New Haven,10.2 +1830,18,Norfolk,9.8 +1830,19,Rochester,9.2 +1830,20,Buffalo,8.7 +1840,1,New York,374 +1840,2,Philadelphia,259 +1840,3,Boston,183 +1840,4,Baltimore,110 +1840,5,New Orleans,105 +1840,6,Albany,72 +1840,7,Cincinnati,54.8 +1840,8,Washington,50.2 +1840,9,Pittsburgh,43.7 +1840,10,Charleston,42.6 +1840,11,Providence,40.9 +1840,12,Louisville,34.2 +1840,13,Rochester,31.4 +1840,14,Newark,29.8 +1840,15,Buffalo,29.3 +1840,16,"Portland, ME",28.6 +1840,17,St. Louis,28.4 +1840,18,New Bedford,24.1 +1840,19,New Haven,21.9 +1840,20,Detroit,21.2 +1850,1,New York,650 +1850,2,Philadelphia,405 +1850,3,Boston,308 +1850,4,Baltimore,179 +1850,5,Cincinnati,133 +1850,6,New Orleans,123 +1850,7,Albany,107 +1850,8,St. Louis,95 +1850,9,Pittsburgh,86 +1850,10,Buffalo,80 +1850,11,Washington,67 +1850,12,Providence,65 +1850,13,Louisville,61 +1850,14,Newark,57 +1850,15,Charleston,50 +1850,16,Rochester,49 +1850,17,Chicago,40 +1850,18,Syracuse,38 +1850,19,Detroit,38 +1850,20,"Portland, ME",36 +1860,1,New York,1143 +1860,2,Philadelphia,608 +1860,3,Boston,374 +1860,4,Baltimore,221 +1860,5,Cincinnati,192 +1860,6,St. Louis,176 +1860,7,New Orleans,172 +1860,8,Chicago,123 +1860,9,Albany,116 +1860,10,Newark,103 +1860,11,Pittsburgh,93 +1860,12,Buffalo,90 +1860,13,Louisville,88 +1860,14,Washington,80 +1860,15,Providence,69 +1860,16,Detroit,59 +1860,17,San Francisco,57 +1860,18,Rochester,56 +1860,19,Cleveland,49 +1860,20,Milwaukee,48 +1870,1,New York,1687 +1870,2,Philadelphia,747 +1870,3,Boston,501 +1870,4,St. Louis,345 +1870,5,Chicago,324 +1870,6,Baltimore,283 +1870,7,Cincinnati,257 +1870,8,New Orleans,196 +1870,9,Pittsburgh,170 +1870,10,Albany,157 +1870,11,San Francisco,151 +1870,12,Buffalo,133 +1870,13,Louisville,129 +1870,14,Washington,123 +1870,15,Providence,101 +1870,16,Detroit,101 +1870,17,Cleveland,101 +1870,18,Milwaukee,75 +1870,19,Rochester,73 +1870,20,New Haven,65 +1880,1,New York,2234 +1880,2,Philadelphia,949 +1880,3,Boston,658 +1880,4,Chicago,543 +1880,5,St. Louis,386 +1880,6,Baltimore,353 +1880,7,Cincinnati,307 +1880,8,Pittsburgh,265 +1880,9,San Francisco,236 +1880,10,New Orleans,219 +1880,11,Albany,178 +1880,12,Buffalo,171 +1880,13,Cleveland,169 +1880,14,Washington,164 +1880,15,Detroit,147 +1880,16,Louisville,143 +1880,17,Providence,128 +1880,18,Milwaukee,121 +1880,19,Rochester,103 +1880,20,Minneapolis,94 +1890,1,New York,2977 +1890,2,Philadelphia,1180 +1890,3,Chicago,1141 +1890,4,Boston,818 +1890,5,St. Louis,490 +1890,6,Baltimore,453 +1890,7,Pittsburgh,396 +1890,8,Cincinnati,344 +1890,9,Minneapolis,305 +1890,10,San Francisco,302 +1890,11,Cleveland,274 +1890,12,Buffalo,272 +1890,13,Washington,253 +1890,14,New Orleans,245 +1890,15,Detroit,237 +1890,16,Milwaukee,212 +1890,17,Albany,189 +1890,18,Louisville,183 +1890,19,Kansas City,165 +1890,20,Providence,163 +1900,1,New York,4266 +1900,2,Chicago,1759 +1900,3,Philadelphia,1454 +1900,4,Boston,1009 +1900,5,St. Louis,626 +1900,6,Pittsburgh,532 +1900,7,Baltimore,532 +1900,8,Cleveland,396 +1900,9,Cincinnati,379 +1900,10,San Francisco,375 +1900,11,Minneapolis,374 +1900,12,Buffalo,373 +1900,13,Detroit,321 +1900,14,Washington,302 +1900,15,Milwaukee,296 +1900,16,New Orleans,291 +1900,17,Providence,243 +1900,18,Kansas City,242 +1900,19,Louisville,227 +1900,20,Albany,188 +1910,1,New York,6021 +1910,2,Chicago,2283 +1910,3,Philadelphia,1746 +1910,4,Boston,1213 +1910,5,St. Louis,760 +1910,6,Pittsburgh,655 +1910,7,San Francisco,604 +1910,8,Baltimore,589 +1910,9,Cleveland,580 +1910,10,Minneapolis,526 +1910,11,Detroit,503 +1910,12,Cincinnati,425 +1910,13,Buffalo,420 +1910,14,Milwaukee,389 +1910,15,Los Angeles,374 +1910,16,Kansas City,357 +1910,17,Washington,357 +1910,18,New Orleans,344 +1910,19,Albany,283 +1910,20,Providence,274 +1920,1,New York,7041 +1920,2,Chicago,2859 +1920,3,Philadelphia,2072 +1920,4,Boston,1366 +1920,5,Detroit,1071 +1920,6,St. Louis,859 +1920,7,Cleveland,834 +1920,8,Pittsburgh,775 +1920,9,San Francisco,771 +1920,10,Baltimore,753 +1920,11,Los Angeles,682 +1920,12,Minneapolis,626 +1920,13,Buffalo,539 +1920,14,Milwaukee,478 +1920,15,Cincinnati,470 +1920,16,Washington,467 +1920,17,Kansas City,455 +1920,18,New Orleans,393 +1920,19,Seattle,334 +1920,20,Indianapolis,323 +1930,1,New York,8667 +1930,2,Chicago,3718 +1930,3,Philadelphia,2264 +1930,4,Detroit,1721 +1930,5,Los Angeles,1617 +1930,6,Boston,1479 +1930,7,San Francisco,996 +1930,8,Cleveland,976 +1930,9,Pittsburgh,960 +1930,10,St. Louis,950 +1930,11,Baltimore,836 +1930,12,Minneapolis,753 +1930,13,Buffalo,620 +1930,14,Milwaukee,615 +1930,15,Cincinnati,580 +1930,16,Kansas City,561 +1930,17,Washington,527 +1930,18,New Orleans,469 +1930,19,Seattle,390 +1930,20,Indianapolis,379 +1940,1,New York,10135 +1940,2,Chicago,4210 +1940,3,Philadelphia,2538 +1940,4,Los Angeles,2268 +1940,5,Detroit,2041 +1940,6,Boston,1746 +1940,7,San Francisco,1156 +1940,8,Pittsburgh,1134 +1940,9,St. Louis,1102 +1940,10,Cleveland,1079 +1940,11,Baltimore,992 +1940,12,Minneapolis,886 +1940,13,Washington,800 +1940,14,Buffalo,708 +1940,15,Milwaukee,705 +1940,16,Kansas City,632 +1940,17,Cincinnati,559 +1940,18,New Orleans,557 +1940,19,Houston,471 +1940,20,Seattle,451 +1950,1,New York,12604 +1950,2,Chicago,5208 +1950,3,Los Angeles,4250 +1950,4,Philadelphia,3297 +1950,5,Detroit,2884 +1950,6,Boston,2301 +1950,7,San Francisco,2131 +1950,8,St. Louis,1541 +1950,9,Cleveland,1425 +1950,10,Pittsburgh,1400 +1950,11,Washington,1287 +1950,12,Baltimore,1162 +1950,13,Minneapolis,987 +1950,14,Buffalo,895 +1950,15,Dallas,855 +1950,16,Milwaukee,829 +1950,17,Cincinnati,813 +1950,18,Houston,701 +1950,19,Kansas City,698 +1950,20,New Orleans,660 +1960,1,New York,14437 +1960,2,Los Angeles,6805 +1960,3,Chicago,6377 +1960,4,Philadelphia,3989 +1960,5,Detroit,3750 +1960,6,San Francisco,2607 +1960,7,Boston,2501 +1960,8,Pittsburgh,2105 +1960,9,Washington,1905 +1960,10,St. Louis,1864 +1960,11,Cleveland,1785 +1960,12,Dallas,1435 +1960,13,Baltimore,1419 +1960,14,Minneapolis,1377 +1960,15,Miami,1173 +1960,16,Milwaukee,1150 +1960,17,Houston,1140 +1960,18,Buffalo,1054 +1960,19,Cincinnati,994 +1960,20,Kansas City,921 +1970,1,New York,16193 +1970,2,Los Angeles,7984 +1970,3,Chicago,7164 +1970,4,Philadelphia,4419 +1970,5,Detroit,4085 +1970,6,San Francisco,3049 +1970,7,Boston,2703 +1970,8,Washington,2671 +1970,9,Pittsburgh,2124 +1970,10,St. Louis,2123 +1970,11,Dallas,2016 +1970,12,Cleveland,1960 +1970,13,Miami,1834 +1970,14,Minneapolis,1701 +1970,15,Houston,1678 +1970,16,Baltimore,1580 +1970,17,Milwaukee,1252 +1970,18,Seattle,1238 +1970,19,San Diego,1198 +1970,20,Atlanta,1172 +1980,1,New York,16500 +1980,2,Los Angeles,10841 +1980,3,Chicago,7325 +1980,4,Philadelphia,4830 +1980,5,Detroit,4214 +1980,6,San Francisco,4185 +1980,7,Boston,3064 +1980,8,Washington,2912 +1980,9,Houston,2757 +1980,10,Dallas,2713 +1980,11,Miami,2616 +1980,12,St. Louis,1849 +1980,13,Pittsburgh,1810 +1980,14,Minneapolis,1788 +1980,15,Baltimore,1755 +1980,16,Cleveland,1752 +1980,17,San Diego,1704 +1980,18,Atlanta,1613 +1980,19,Phoenix,1409 +1980,20,Seattle,1392 +1990,1,New York,16754 +1990,2,Los Angeles,13522 +1990,3,Chicago,7373 +1990,4,San Francisco,5386 +1990,5,Philadelphia,4970 +1990,6,Miami,3948 +1990,7,Detroit,3698 +1990,8,Washington,3363 +1990,9,Boston,3355 +1990,10,Dallas,3265 +1990,11,Houston,3088 +1990,12,Seattle,2354 +1990,13,San Diego,2348 +1990,14,Atlanta,2158 +1990,15,Minneapolis,2080 +1990,16,Phoenix,2006 +1990,17,St. Louis,1947 +1990,18,Baltimore,1890 +1990,19,Pittsburgh,1744 +1990,20,Tampa,1709 +2000,1,New York,18689 +2000,2,Los Angeles,14661 +2000,3,Chicago,8419 +2000,4,San Francisco,5973 +2000,5,Philadelphia,5418 +2000,6,Miami,4919 +2000,7,Dallas,4445 +2000,8,Houston,4063 +2000,9,Boston,4032 +2000,10,Washington,3934 +2000,11,Detroit,3903 +2000,12,Atlanta,3500 +2000,13,Seattle,3018 +2000,14,Phoenix,2975 +2000,15,San Diego,2674 +2000,16,Minneapolis,2389 +2000,17,Baltimore,2251 +2000,18,Denver,2231 +2000,19,St. Louis,2078 +2000,20,Tampa,2062 +2010,1,New York,20009 +2010,2,Los Angeles,15750 +2010,3,Chicago,9023 +2010,4,San Francisco,6828 +2010,5,Philadelphia,6003 +2010,6,Dallas,5685 +2010,7,Miami,5513 +2010,8,Houston,5382 +2010,9,Atlanta,4743 +2010,10,Washington,4697 +2010,11,Boston,4407 +2010,12,Detroit,4160 +2010,13,Phoenix,3863 +2010,14,Seattle,3446 +2010,15,San Diego,2985 +2010,16,Denver,2716 +2010,17,Minneapolis,2651 +2010,18,Baltimore,2497 +2010,19,Tampa,2442 +2010,20,St. Louis,2246 diff --git a/projects/darrenxing/darren_getcitydatafromweb.py b/projects/darrenxing/darren_getcitydatafromweb.py new file mode 100644 index 0000000..ba6b5ee --- /dev/null +++ b/projects/darrenxing/darren_getcitydatafromweb.py @@ -0,0 +1,79 @@ +from bs4 import BeautifulSoup +'''import urllib2''' +try: + import urllib.request as urllib2 +except: + import urllib2 + + +from re import sub +import operator +import csv +from functools import reduce +import os + +class PopulationTableGrabber(object): + def __init__(self, url): + self.page = urllib2.urlopen(url) + self.soup = BeautifulSoup(self.page) + + def find_all_tabs(self): + def table_has_enough_rows(elm): + ''' actually table of interest has a border, +AND enough rows.''' + return elm.name == 'table' and \ + elm.has_attr('bordercolor') and \ + len(elm.find_all('tr')) > 20 + return self.soup.find_all(table_has_enough_rows) + + def parse_one_table(self, tab): + ''' somewhat dirty, but grabs the year, skips a row, and parses the rest +would break if there were and asterisk row for example +''' + yearElement = tab.find(colspan='3') + year = int(yearElement.get_text()) + rows = yearElement.find_parent().find_next_sibling().find_next_siblings() + out = [] + for row in rows: + tds = row.find_all('td') + rowdata = { + 'year': year, + 'rank': self.fix_ordinals(tds[0].get_text()), + 'city': tds[1].get_text(), + 'pop': float(tds[2].get_text()) + } + out.append(rowdata) + return out + + def reshape_city_data(self, all_tabs): + ''' list += rows, for all the tables''' + return reduce(operator.add, + [self.parse_one_table(tab) for tab in all_tabs] + ) + + + def fix_ordinals(self, string): + ''' oh look, "1." +''' + return int(sub('[.]', '', string)) + + + def write_csv(self, dicts, filename="citypop.csv"): + ''' could do without ceremony, but this preserves key order +''' + keys = ['year', 'rank', 'city', 'pop'] + filename=os.path.join('C:/Users/Zhe.Xing/Documents/datavisizalation/edav-1/projects/darrenxing', filename) + f = open(filename, 'w', newline='') + dict_writer = csv.DictWriter(f, keys) + dict_writer.writer.writerow(keys) + dict_writer.writerows(dicts) + f.close() + + +pop = PopulationTableGrabber('http://www.peakbagger.com/pbgeog/histmetropop.aspx') +pop.write_csv(pop.reshape_city_data(pop.find_all_tabs())) + +## I transcribed total population in millions: +## https://docs.google.com/spreadsheet/pub?key=0AruyJI76uB8RdFBySUhzRldyalJGTXZtY1NxT0E2Z1E&output=csv +## source: http://www.census.gov/population/censusdata/table-4.pdf +## and census factfinder2 tables for 2000, 2010. diff --git a/projects/popgraph/get-data.py b/projects/popgraph/get-data.py index 83dba71..6801bcc 100644 --- a/projects/popgraph/get-data.py +++ b/projects/popgraph/get-data.py @@ -1,7 +1,10 @@ from bs4 import BeautifulSoup import urllib2 +<<<<<<< HEAD from re import sub import operator +======= +>>>>>>> upstream/uspop import csv class PopulationTableGrabber(object): @@ -19,6 +22,7 @@ def table_has_enough_rows(elm): return self.soup.find_all(table_has_enough_rows) def parse_one_table(self, tab): +<<<<<<< HEAD ''' somewhat dirty, but grabs the year, skips a row, and parses the rest would break if there were and asterisk row for example ''' @@ -31,6 +35,23 @@ def parse_one_table(self, tab): rowdata = { 'year': year, 'rank': self.fix_ordinals(tds[0].get_text()), +======= + ''' munge the table rows + ''' + yearElement = tab.find(colspan='3') + year = int(yearElement.get_text()) + rows = tab.find_all('tr') + print rows + out = [] + for row in rows: + if row.find('td').has_attr('colspan'): + continue + tds = row.find_all('td') + print tds + rowdata = { + 'year': year, + 'rank': tds[0].get_text(), +>>>>>>> upstream/uspop 'city': tds[1].get_text(), 'pop': float(tds[2].get_text()) } @@ -38,6 +59,7 @@ def parse_one_table(self, tab): return out def reshape_city_data(self, all_tabs): +<<<<<<< HEAD ''' list += rows, for all the tables''' return reduce(operator.add, [self.parse_one_table(tab) for tab in all_tabs] @@ -49,10 +71,18 @@ def fix_ordinals(self, string): ''' return int(sub('[.]', '', string)) +======= + ''' list of lists''' + return [self.parse_one_table(tab) for tab in all_tabs] +>>>>>>> upstream/uspop def write_csv(self, dicts, filename="citypop.csv"): ''' could do without ceremony, but this preserves key order ''' +<<<<<<< HEAD +======= + print dicts +>>>>>>> upstream/uspop keys = ['year', 'rank', 'city', 'pop'] f = open(filename, 'wb') dict_writer = csv.DictWriter(f, keys) @@ -69,3 +99,7 @@ def write_csv(self, dicts, filename="citypop.csv"): ## source: http://www.census.gov/population/censusdata/table-4.pdf ## and census factfinder2 tables for 2000, 2010. +<<<<<<< HEAD +## Change this file. +======= +>>>>>>> upstream/uspop diff --git a/radsketch.pde b/radsketch.pde new file mode 100644 index 0000000..4f439ff --- /dev/null +++ b/radsketch.pde @@ -0,0 +1,7 @@ +<<<<<<< HEAD +ask +======= +ask + +test it +>>>>>>> 59fcefec425c9729cee15fdedf1ed5080ffa8969 diff --git a/readme.txt b/readme.txt new file mode 100644 index 0000000..e69de29 diff --git a/readme2.txt b/readme2.txt new file mode 100644 index 0000000..e69de29