Merge branch 'develop'

Author: leafcoder

demo_proj/ncov/settings.py

@@ -186,6 +186,9 @@
     # 每分钟抓取丁香园数据一次
     ('*/1 * * * *', 'django.core.management.call_command', ['crawl', 'dxy']),
 
-    # 每小时0分时开始抓取 covidtracking 数据一次
-    ('0 * * * *', 'django.core.management.call_command', ['crawl', 'covidtracking'])
+    # 每隔1小时开始抓取 covidtracking 数据一次
+    ('*/60 * * * *', 'django.core.management.call_command', ['crawl', 'covidtracking']),
 )
+
+# scrapy 日志文件路径
+SCRAPY_LOG_FILE = '/var/tmp/django-covid19-spider.log'

demo_proj/requirements.txt

@@ -0,0 +1,35 @@
+attrs==19.3.0
+Automat==20.2.0
+cffi==1.14.0
+constantly==15.1.0
+cryptography==2.9.2
+cssselect==1.1.0
+Django==2.2.13
+django-cors-headers==3.3.0
+django-covid19
+django-crontab==0.7.1
+django-filter==2.2.0
+django-mysql==3.5.0
+djangorestframework==3.11.0
+hyperlink==19.0.0
+idna==2.9
+incremental==17.5.0
+lxml==4.5.1
+parsel==1.6.0
+Protego==0.1.16
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycparser==2.20
+PyDispatcher==2.0.5
+PyHamcrest==2.0.2
+pyOpenSSL==19.1.0
+pytz==2020.1
+queuelib==1.5.0
+Scrapy==2.0.1
+scrapy-djangoitem==1.1.1
+service-identity==18.1.0
+six==1.15.0
+sqlparse==0.3.1
+Twisted==20.3.0
+w3lib==1.22.0
+zope.interface==5.1.0

django_covid19/admin.py

@@ -63,7 +63,7 @@ class CityAdmin(BaseAdmin):
     list_display = (
         'countryCode', 'provinceName', 'provinceCode', 'cityName',
         'currentConfirmedCount', 'confirmedCount', 'suspectedCount',
-        'curedCount', 'deadCount'
+        'curedCount', 'deadCount', 'createTime', 'modifyTime'
     )
     search_fields = (
         'cityName', 'countryCode', 'provinceCode', 'provinceName'
@@ -76,7 +76,7 @@ class ProvinceAdmin(BaseAdmin):
     list_display = (
         'countryCode', 'provinceName',
         'currentConfirmedCount', 'confirmedCount', 'suspectedCount',
-        'curedCount', 'deadCount'
+        'curedCount', 'deadCount', 'createTime', 'modifyTime'
     )
     search_fields = ('provinceName', 'countryCode')
 
@@ -86,8 +86,8 @@ class CountryAdmin(BaseAdmin):
 
     list_display = (
         'continents', 'countryCode', 'countryName', 'countryFullName',
-        'currentConfirmedCount', 'confirmedCount',
-        'suspectedCount', 'curedCount', 'deadCount'
+        'currentConfirmedCount', 'confirmedCount', 'suspectedCount',
+        'curedCount', 'deadCount', 'createTime', 'modifyTime'
     )
     search_fields = (
         'continents', 'countryFullName', 'countryCode', 'countryName'

django_covid19/fixtures/city.json

@@ -46,7 +46,7 @@ class Country(models.Model):
     curedCount = models.IntegerField(default=0)
     deadCount = models.IntegerField(default=0)
 
-    showRank = models.BooleanField(null=True)
+    showRank = models.BooleanField(default=False)
     deadRateRank = models.IntegerField(null=True)
     deadCountRank = models.IntegerField(null=True)
     confirmedCountRank = models.FloatField(null=True)

django_covid19/fixtures/country.json

@@ -1,3 +1,5 @@
 from django.conf import settings
 
-CACHE_PAGE_TIMEOUT = getattr(settings, 'CACHE_PAGE_TIMEOUT', 24*60*60)
+CACHE_PAGE_TIMEOUT = getattr(settings, 'CACHE_PAGE_TIMEOUT', 24*60*60)
+
+SCRAPY_LOG_FILE = getattr(settings, 'SCRAPY_LOG_FILE', None)

django_covid19/fixtures/province.json

@@ -89,3 +89,12 @@
 #HTTPCACHE_DIR = 'httpcache'
 #HTTPCACHE_IGNORE_HTTP_CODES = []
 #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
+
+# Get settings from django
+from django.conf import settings
+
+for name in dir(settings):
+    if not name.startswith('SCRAPY_'):
+        continue
+    scrapy_name = name[7:]
+    globals()[scrapy_name] = getattr(settings, name, None)