From 9d33cedec2ccf10573dec138aac5e5160b48dfd2 Mon Sep 17 00:00:00 2001 From: Eduard Urbach Date: Wed, 4 Apr 2018 18:59:20 +0200 Subject: [PATCH] New image downloader --- jobs/anime-images/anime-images.go | 247 ------------------ .../kitsu-download-anime-images.go | 82 ++++++ 2 files changed, 82 insertions(+), 247 deletions(-) delete mode 100644 jobs/anime-images/anime-images.go create mode 100644 patches/kitsu-download-anime-images/kitsu-download-anime-images.go diff --git a/jobs/anime-images/anime-images.go b/jobs/anime-images/anime-images.go deleted file mode 100644 index a714c7ff..00000000 --- a/jobs/anime-images/anime-images.go +++ /dev/null @@ -1,247 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "os" - "path" - "runtime" - "strconv" - "time" - - _ "image/gif" - _ "image/jpeg" - _ "image/png" - - "github.com/aerogo/ipo" - "github.com/aerogo/ipo/inputs" - "github.com/aerogo/ipo/outputs" - "github.com/animenotifier/arn" - "github.com/fatih/color" -) - -var ticker = time.NewTicker(50 * time.Millisecond) - -// Shell parameters -var from int -var to int -var useCache bool - -// Shell flags -func init() { - flag.IntVar(&from, "from", 0, "From index") - flag.IntVar(&to, "to", 0, "To index") - flag.BoolVar(&useCache, "cache", false, "Use cache") - flag.Parse() -} - -func main() { - color.Yellow("Downloading anime images") - - defer color.Green("Finished.") - defer arn.Node.Close() - - if from < 0 { - from = 0 - } - - allAnime := arn.FilterAnime(func(anime *arn.Anime) bool { - id, _ := strconv.Atoi(anime.ID) - return id >= from && id <= to - }) - - for index, anime := range allAnime { - fmt.Printf("%d / %d\n", index+1, len(allAnime)) - work(anime) - } - - // Give file buffers some time, just to be safe - time.Sleep(time.Second) -} - -func work(anime *arn.Anime) error { - <-ticker.C - - originals := path.Join(os.Getenv("GOPATH"), "/src/github.com/animenotifier/notify.moe/images/anime/original/") - large := path.Join(os.Getenv("GOPATH"), "/src/github.com/animenotifier/notify.moe/images/anime/large/") - medium := path.Join(os.Getenv("GOPATH"), "/src/github.com/animenotifier/notify.moe/images/anime/medium/") - small := path.Join(os.Getenv("GOPATH"), "/src/github.com/animenotifier/notify.moe/images/anime/small/") - - largeSizeX := 250 - largeSizeY := 0 - - mediumSizeX := 142 - mediumSizeY := 200 - - smallSizeX := 55 - smallSizeY := 78 - - webpQuality := 70 - jpegQuality := 70 - - qualityBonusLowDPI := 10 - qualityBonusMedium := 10 - qualityBonusSmall := 10 - - kitsuOriginal := fmt.Sprintf("https://media.kitsu.io/anime/poster_images/%s/original", anime.GetMapping("kitsu/anime")) - - // Define the input sources - sources := []ipo.Input{} - - // If we use the file system cache, start by searching those files - if useCache { - sources = append(sources, - &inputs.FileSystemImage{ - URL: path.Join(originals, anime.ID+".png"), - }, - &inputs.FileSystemImage{ - URL: path.Join(originals, anime.ID+".jpg"), - }, - &inputs.FileSystemImage{ - URL: path.Join(originals, anime.ID+".jpeg"), - }, - &inputs.FileSystemImage{ - URL: path.Join(originals, anime.ID+".gif"), - }, - ) - } - - // Afterwards, use the network sources - sources = append(sources, - &inputs.NetworkImage{ - URL: kitsuOriginal + anime.Image.Extension, - }, - &inputs.NetworkImage{ - URL: kitsuOriginal + ".png", - }, - &inputs.NetworkImage{ - URL: kitsuOriginal + ".jpg", - }, - &inputs.NetworkImage{ - URL: kitsuOriginal + ".jpeg", - }, - ) - - // Define the system - system := ipo.System{ - Inputs: sources, - Outputs: []ipo.Output{ - // Original - &outputs.ImageFile{ - Directory: originals, - BaseName: anime.ID, - }, - - // Large - &outputs.ImageFile{ - Directory: large, - BaseName: anime.ID, - Width: largeSizeX, - Height: largeSizeY, - Quality: jpegQuality + qualityBonusLowDPI, - }, - &outputs.ImageFile{ - Directory: large, - BaseName: anime.ID + "@2", - Width: largeSizeX * 2, - Height: largeSizeY * 2, - Quality: jpegQuality, - }, - &outputs.ImageFile{ - Directory: large, - BaseName: anime.ID, - Width: largeSizeX, - Height: largeSizeY, - Format: "webp", - Quality: webpQuality + qualityBonusLowDPI, - }, - &outputs.ImageFile{ - Directory: large, - BaseName: anime.ID + "@2", - Width: largeSizeX * 2, - Height: largeSizeY * 2, - Format: "webp", - Quality: webpQuality, - }, - - // Medium - &outputs.ImageFile{ - Directory: medium, - BaseName: anime.ID, - Width: mediumSizeX, - Height: mediumSizeY, - Quality: jpegQuality + qualityBonusLowDPI + qualityBonusMedium, - }, - &outputs.ImageFile{ - Directory: medium, - BaseName: anime.ID + "@2", - Width: mediumSizeX * 2, - Height: mediumSizeY * 2, - Quality: jpegQuality, - }, - &outputs.ImageFile{ - Directory: medium, - BaseName: anime.ID, - Width: mediumSizeX, - Height: mediumSizeY, - Format: "webp", - Quality: webpQuality + qualityBonusLowDPI + qualityBonusMedium, - }, - &outputs.ImageFile{ - Directory: medium, - BaseName: anime.ID + "@2", - Width: mediumSizeX * 2, - Height: mediumSizeY * 2, - Format: "webp", - Quality: webpQuality, - }, - - // Small - &outputs.ImageFile{ - Directory: small, - BaseName: anime.ID, - Width: smallSizeX, - Height: smallSizeY, - Quality: jpegQuality + qualityBonusLowDPI + qualityBonusSmall, - }, - &outputs.ImageFile{ - Directory: small, - BaseName: anime.ID + "@2", - Width: smallSizeX * 2, - Height: smallSizeY * 2, - Quality: jpegQuality, - }, - &outputs.ImageFile{ - Directory: small, - BaseName: anime.ID, - Width: smallSizeX, - Height: smallSizeY, - Format: "webp", - Quality: webpQuality + qualityBonusLowDPI + qualityBonusSmall, - }, - &outputs.ImageFile{ - Directory: small, - BaseName: anime.ID + "@2", - Width: smallSizeX * 2, - Height: smallSizeY * 2, - Format: "webp", - Quality: webpQuality, - }, - }, - InputProcessor: ipo.SequentialInputs, - OutputProcessor: ipo.ParallelOutputs, - } - - err := system.Run() - - if err != nil { - fmt.Println(err) - } - - // Try to free up some memory - system.Inputs = nil - system.Outputs = nil - runtime.GC() - - return nil -} diff --git a/patches/kitsu-download-anime-images/kitsu-download-anime-images.go b/patches/kitsu-download-anime-images/kitsu-download-anime-images.go new file mode 100644 index 00000000..eece2d57 --- /dev/null +++ b/patches/kitsu-download-anime-images/kitsu-download-anime-images.go @@ -0,0 +1,82 @@ +package main + +import ( + "flag" + "fmt" + "net/http" + "runtime" + "strconv" + "time" + + _ "image/gif" + _ "image/jpeg" + _ "image/png" + + "github.com/aerogo/http/client" + "github.com/animenotifier/arn" + "github.com/fatih/color" +) + +var ticker = time.NewTicker(50 * time.Millisecond) + +// Shell parameters +var from int +var to int +var useCache bool + +// Shell flags +func init() { + flag.IntVar(&from, "from", 0, "From index") + flag.IntVar(&to, "to", 0, "To index") + flag.Parse() +} + +func main() { + color.Yellow("Downloading anime images") + + defer color.Green("Finished.") + defer arn.Node.Close() + + if from < 0 { + from = 0 + } + + allAnime := arn.FilterAnime(func(anime *arn.Anime) bool { + id, _ := strconv.Atoi(anime.GetMapping("kitsu/anime")) + return id >= from && id <= to + }) + + for index, anime := range allAnime { + fmt.Printf("%d / %d\n", index+1, len(allAnime)) + work(anime) + } + + // Give file buffers some time, just to be safe + time.Sleep(time.Second) +} + +func work(anime *arn.Anime) error { + <-ticker.C + + kitsuOriginal := fmt.Sprintf("https://media.kitsu.io/anime/poster_images/%s/original%s", anime.GetMapping("kitsu/anime"), anime.Image.Extension) + + // Download kitsu image + response, err := client.Get(kitsuOriginal).End() + + if err != nil { + color.Red("%s (%s)", err.Error(), kitsuOriginal) + return err + } + + if response.StatusCode() != http.StatusOK { + color.Red("Status %d (%s)", response.StatusCode(), kitsuOriginal) + return err + } + + anime.SetImageBytes(response.Bytes()) + + // Try to free up some memory + runtime.GC() + + return nil +}