Make sure accept-encoding is sent

2018-03-08 21:40:48 +01:00 · 2018-03-08 21:40:48 +01:00 · a7c9aba58d
commit a7c9aba58d
parent 85d5d1bc36
1 changed files with 2 additions and 1 deletions
--- a/jobs/crawler/crawler.go
+++ b/jobs/crawler/crawler.go
@ -39,6 +39,7 @@ func main() {
 	malCrawler := crawler.New(
 		map[string]string{
 			"User-Agent":      userAgent,
+			"Accept-Encoding": "gzip",
 		},
 		1500*time.Millisecond,
 		len(animes),