web-crawler/Utilities.cs at master · yafithekid/web-crawler · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

namespace WebCrawler
{
    class Configuration
    {
        static string crawlerListLocation;
        static string ignoredExtensionLocation;
        static string indexTagListLocation;
        static string savedPagesLocation;
        static string indexLocation;
        static string crawlResultLocation;
        static string traversalMode;
        static bool enableParagraphOutput;
        static int maximumDepth;

        public static void setCrawlerListLocation(string s)
        {
            crawlerListLocation = s;
        }


        public static void setIgnoredExtensionLocation(string s)
        {
            ignoredExtensionLocation = s;
        }

        public static void setIndexTagListLocation(string s)
        {
            indexTagListLocation = s;
        }

        public static void setIndexLocation(string s)
        {
            indexLocation = s;
        }
        public static void setSavedPagesLocation(string s)
        {
            savedPagesLocation = s;
        }
        public static void setEnableParagraphOutput(bool s)
        {
            enableParagraphOutput = s;
        }

        public static void setCrawlResultLocation(string s)
        {
            crawlResultLocation = s;
        }
        public static void setTraversalMode(string s)
        {
            if (s != "DFS" && s != "BFS") s = "BFS";
            traversalMode = s;
        }
        public static void setMaximumDepth(int depth)
        {
            maximumDepth = depth;
        }
        public static string getTraversalMode()
        {
            return traversalMode;
        }
        public static int getMaximumDepth()
        {
            return maximumDepth;
        }
        public static string getCrawlerListLocation()
        {
            return crawlerListLocation;
        }

        public static string getIgnoredExtensionLocation()
        {
            return ignoredExtensionLocation;
        }

        public static string getIndexTagListLocation()
        {
            return indexTagListLocation;
        }
        public static string getIndexLocation()
        {
            return indexLocation;
        }
        public static string getSavedPagesLocation()
        {
            return savedPagesLocation;
        }

        public static string getCrawlResultLocation()
        {
            return crawlResultLocation;
        }
        public static bool getEnableParagraphOutput()
        {
            return enableParagraphOutput;
        }
        public static void setDefaultConfiguration()
        {
            Configuration.setCrawlerListLocation("config/crawlerList.conf");
            Configuration.setIgnoredExtensionLocation("config/ignoredExtension.conf");
            Configuration.setIndexTagListLocation("config/indexTagList.conf");
            Configuration.setIndexLocation("index/");
            Configuration.setSavedPagesLocation("savedpages/");
            Configuration.setCrawlResultLocation("crawlresult/");
            Configuration.enableParagraphOutput = true;
            Configuration.setMaximumDepth(2);
        }
    }
    class WordProcessor
    {
        private string currentWord;
        private System.IO.StreamReader inputFile;

        public WordProcessor(string fileName)
        {
            openFile(fileName);
            startWord();
        }

        public string getWord()
        {
            return currentWord;
        }

        public void openFile(string fileName)
        {
            try
            {
                inputFile = new System.IO.StreamReader(fileName);
            }
            catch (Exception e)
            {
                Console.WriteLine(e.Message);
            }
        }

        public void startWord()
        {
            if (!inputFile.EndOfStream)
            {
                currentWord = inputFile.ReadLine();
            }
        }

        public void advanceWord()
        {
            if (!inputFile.EndOfStream)
            {
                currentWord = inputFile.ReadLine();
            }
        }

        public bool isEOF()
        {
            return inputFile.EndOfStream;
        }

        public void closeFile()
        {
            inputFile.Close();
        }
    }
}