web-crawler-using-python/crawl words.py at master · bvpranu97/web-crawler-using-python · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
import requests
from bs4 import BeautifulSoup
import operator
from collections import Counter

def start(url):


	wordlist = []
	source_code = requests.get(url).text


	soup = BeautifulSoup(source_code, 'html.parser')


	for each_text in soup.findAll('div', {'class':'entry-content'}):
		content = each_text.text

		words = content.lower().split()

		for each_word in words:
			wordlist.append(each_word)
		clean_wordlist(wordlist)


def clean_wordlist(wordlist):

	clean_list =[]
	for word in wordlist:
		symbols = '!@#$%^&*()_-+={[}]|\;:"<>?/., '

		for i in range (0, len(symbols)):
			word = word.replace(symbols[i], '')

		if len(word) > 0:
			clean_list.append(word)
	create_dictionary(clean_list)

def create_dictionary(clean_list):
	word_count = {}

	for word in clean_list:
		if word in word_count:
			word_count[word] += 1
		else:
			word_count[word] = 1


	c = Counter(word_count)

	# returns the most occurring elements
	top = c.most_common(10)
	print(top)


if __name__ == '__main__':
	start("https://github.com")