nyaanime/downloader
1
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity

check local files for already downloaded torrents and currently downloading torrents

Browse Source
This commit is contained in:
Timon Ringwald 2022-08-22 14:14:14 +02:00
parent 820d45d80d
commit 95f77cf189
12 changed files with 276 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
anime_episode_filepath.go
View File

@ -1,40 +0,0 @@
package main
import (
"path/filepath"
"strings"
"git.milar.in/milarin/anilist"
"git.milar.in/nyaanime/model"
)
type AnimePathPatternData struct {
Title anilist.MediaTitle
Episode int
Ext string
}
func GetAnimeEpFilepath(animeEp model.AnimeEpisode, ext string) string {
tmplData := AnimePathPatternData{
Title: animeEp.Anime.Title,
Episode: animeEp.Episode,
Ext: ext,
}
b := new(strings.Builder)
if err := AnimeEpFilepathPattern.Execute(b, tmplData); err != nil {
panic(err)
}
return b.String()
}
func AnimeEpExistsLocally(animeEp model.AnimeEpisode) bool {
animeEpPath := filepath.Join(AnimePath, GetAnimeEpFilepath(animeEp, "*"))
files, err := filepath.Glob(animeEpPath)
if err != nil {
panic(err)
}
return len(files) > 0
}

2
errors.go
View File

@ -12,4 +12,6 @@ var (
ErrAnimeListNotObtainable = adverr.NewErrTmpl("ErrAnimeListNotObtainable", "anime list from anilist.co not obtainable (reason: %s)")
ErrTorrentNotObtainable = adverr.NewErrTmpl("ErrTorrentNotObtainable", "torrents from nyaa.si not obtainable (reason: %s)")
ErrInvalidAnimeStatus = adverr.NewErrTmpl("ErrInvalidAnimeStatus", "invalid status '%s' in ANIME_STATUS (allowed: %s)")
ErrInvalidGlobSyntax = adverr.NewErrTmpl("ErrInvalidGlobSyntax", "invalid filepath.Glob syntax: '%s'")
ErrNoSuitableFileFound = adverr.NewErrTmpl("ErrNoSuitableFileFound", "no file found with essential properties for anime '%s' and episode %d")
)

25
file_filter.go Normal file
View File

@ -0,0 +1,25 @@
package main
import (
"git.milar.in/milarin/slices"
)
func HasFileEssentialProperties(props *FileProperties) bool {
if props.Resolution < MinResolution || props.Resolution > MaxResolution {
return false
}
for _, essentialLanguage := range EssentialLanguages {
if !slices.Contains(props.Languages, essentialLanguage) {
return false
}
}
for _, essentialSubtitle := range EssentialSubtitles {
if !slices.Contains(props.Subtitles, essentialSubtitle) {
return false
}
}
return true
}

17
file_priority.go Normal file
View File

@ -0,0 +1,17 @@
package main
type FilePriority struct {
Properties *FileProperties
Priority int
PreferredProperties map[string]int
}
func NewFilePriority(props *FileProperties) *FilePriority {
priority, preferredProperties := DeterminePriority(props)
return &FilePriority{
Properties: props,
Priority: priority,
PreferredProperties: preferredProperties,
}
}

79
file_properties.go Normal file
View File

@ -0,0 +1,79 @@
package main
import (
"context"
"fmt"
"os"
"git.milar.in/nyaanime/model"
"git.milar.in/nyaanime/parsers"
ffprobe "gopkg.in/vansante/go-ffprobe.v2"
)
type FileProperties struct {
Filepath string
Languages []string
Subtitles []string
Resolution model.Resolution
}
var _ model.PropertyHolder = &FileProperties{}
// TODO cache
func AnalyzeFile(path string) (*FileProperties, error) {
props := &FileProperties{Filepath: path}
file, err := os.Open(path)
if err != nil {
return nil, err
}
defer file.Close()
data, err := ffprobe.ProbeReader(context.Background(), file)
if err != nil {
return nil, err
}
defaultVideoLang := ""
for _, s := range data.StreamType(ffprobe.StreamVideo) {
if s.Disposition.Default > 0 {
props.Resolution = model.Resolution(s.Height)
defaultVideoLang = parsers.ParseLanguage(s.Tags.Language)
break
}
}
for _, s := range data.StreamType(ffprobe.StreamAudio) {
if s.Tags.Language != "" {
props.Languages = append(props.Languages, parsers.ParseLanguage(s.Tags.Language))
} else if s.Disposition.Default > 0 {
props.Languages = append(props.Languages, defaultVideoLang)
}
}
for _, s := range data.StreamType(ffprobe.StreamSubtitle) {
if s.Tags.Language != "" {
props.Subtitles = append(props.Subtitles, parsers.ParseLanguage(s.Tags.Language))
} else if s.Disposition.Default > 0 {
props.Subtitles = append(props.Subtitles, defaultVideoLang)
}
}
return props, nil
}
func (fp *FileProperties) GetLanguages() []string {
return fp.Languages
}
func (fp *FileProperties) GetSubtitles() []string {
return fp.Subtitles
}
func (fp *FileProperties) GetResolution() model.Resolution {
return fp.Resolution
}
func (fp *FileProperties) String() string {
return fmt.Sprintf("")
}

5
go.mod
View File

@ -9,9 +9,10 @@ require (
git.milar.in/milarin/envvars v1.0.3
git.milar.in/milarin/gmath v0.0.2
git.milar.in/milarin/slices v0.0.2
git.milar.in/nyaanime/model v0.0.0-20220821215714-9959e681573f
git.milar.in/nyaanime/parsers v0.0.0-20220815144327-52de61265e27
git.milar.in/nyaanime/model v0.0.0-20220822093541-87208e95e7ac
git.milar.in/nyaanime/parsers v0.0.0-20220822100125-2813a7868f6a
github.com/PuerkitoBio/goquery v1.8.0
gopkg.in/vansante/go-ffprobe.v2 v2.1.0
)
require (

16
go.sum
View File

@ -10,20 +10,18 @@ git.milar.in/milarin/gmath v0.0.2 h1:avz+75f8XqAYA1wEB6kis0R5xvRuepBKTqBuJBjh6Yw
git.milar.in/milarin/gmath v0.0.2/go.mod h1:HDLftG5RLpiNGKiIWh+O2G1PYkNzyLDADO8Cd/1abiE=
git.milar.in/milarin/slices v0.0.2 h1:j92MuP0HWKSaHJMq/FRxDtSDIGiOTvw+KogUTwuulr0=
git.milar.in/milarin/slices v0.0.2/go.mod h1:XRNfE99aNKeaPOY1phjOlpIQqeGCW1LOqqh8UHS+vAk=
git.milar.in/nyaanime/model v0.0.0-20220821124037-0a28c6b41556 h1:RplYz4+CMK9ByI3ELusBltWFRlDs6VMOGk5EyENnLi0=
git.milar.in/nyaanime/model v0.0.0-20220821124037-0a28c6b41556/go.mod h1:OzhQgj0b/Hf9fg8VXYxYt8ONQOvHm8xC44TmS9kQ150=
git.milar.in/nyaanime/model v0.0.0-20220821212334-5e9f052f86b6 h1:vmc1G5uVkYAYJYcaQKLrr+JE6u2780YR6R00EdF5A5I=
git.milar.in/nyaanime/model v0.0.0-20220821212334-5e9f052f86b6/go.mod h1:OzhQgj0b/Hf9fg8VXYxYt8ONQOvHm8xC44TmS9kQ150=
git.milar.in/nyaanime/model v0.0.0-20220821212807-44de43f4c500 h1:zwYRzcbRiS62hDu6hy4OGxMbhLnTsL13VEXNIoTodsk=
git.milar.in/nyaanime/model v0.0.0-20220821212807-44de43f4c500/go.mod h1:OzhQgj0b/Hf9fg8VXYxYt8ONQOvHm8xC44TmS9kQ150=
git.milar.in/nyaanime/model v0.0.0-20220821215714-9959e681573f h1:BKxr1hWCl2QfSiOis1NaRcoG1mZlwo5VnB+0mYhmfEw=
git.milar.in/nyaanime/model v0.0.0-20220821215714-9959e681573f/go.mod h1:OzhQgj0b/Hf9fg8VXYxYt8ONQOvHm8xC44TmS9kQ150=
git.milar.in/nyaanime/model v0.0.0-20220822093541-87208e95e7ac h1:rM5Mpo4/OJuZaBNZdylag+gi8giWVwDbqsoPjhDP9+g=
git.milar.in/nyaanime/model v0.0.0-20220822093541-87208e95e7ac/go.mod h1:OzhQgj0b/Hf9fg8VXYxYt8ONQOvHm8xC44TmS9kQ150=
git.milar.in/nyaanime/parsers v0.0.0-20220815144327-52de61265e27 h1:0+5j9MMJQS8+Luss19hD6hvNFxcBDRal2XwSUTyq7WU=
git.milar.in/nyaanime/parsers v0.0.0-20220815144327-52de61265e27/go.mod h1:qm6fIFBFs90uz7IJ8RKgDir0K8Fa8isixGLgrtC6kgU=
git.milar.in/nyaanime/parsers v0.0.0-20220822100125-2813a7868f6a h1:7vrKOL/vpqJ8YFZ9tmq9iPLoBuLnZgptHWaScyFOFFo=
git.milar.in/nyaanime/parsers v0.0.0-20220822100125-2813a7868f6a/go.mod h1:qm6fIFBFs90uz7IJ8RKgDir0K8Fa8isixGLgrtC6kgU=
github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
github.com/vansante/go-ffprobe v1.1.0 h1:Tz5X+38tF8YYEFVz+PUTrtvlED35IorB7XI0USOqZWU=
github.com/vansante/go-ffprobe v1.1.0/go.mod h1:AEIxsTWYTTeXpel90yu5J/QxuDWNaKCO50xRBN4rdac=
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220812174116-3211cb980234 h1:RDqmgfe7SvlMWoqC3xwQ2blLO3fcWcxMa3eBLRdRW7E=
golang.org/x/net v0.0.0-20220812174116-3211cb980234/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
@ -32,3 +30,5 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/vansante/go-ffprobe.v2 v2.1.0 h1:Gh8oVkvOSZG/DgEMmBw8h4oLvJhSQHROEGp3TaVGq08=
gopkg.in/vansante/go-ffprobe.v2 v2.1.0/go.mod h1:qF0AlAjk7Nqzqf3y333Ly+KxN3cKF2JqA3JT5ZheUGE=

87
local_file_check.go Normal file
View File

@ -0,0 +1,87 @@
package main
import (
"fmt"
"os"
"path/filepath"
"strings"
"git.milar.in/milarin/anilist"
"git.milar.in/nyaanime/model"
)
type AnimePathPatternData struct {
Title anilist.MediaTitle
Episode int
Ext string
}
func GetAnimeEpFilepath(animeEp model.AnimeEpisode, ext string) string {
tmplData := AnimePathPatternData{
Title: animeEp.Anime.Title,
Episode: animeEp.Episode,
Ext: ext,
}
b := new(strings.Builder)
if err := AnimeEpFilepathPattern.Execute(b, tmplData); err != nil {
panic(err)
}
return b.String()
}
func AnimeEpExistsLocally(animeEp model.AnimeEpisode) bool {
animeEpPath := filepath.Join(AnimePath, GetAnimeEpFilepath(animeEp, "*"))
files, err := filepath.Glob(animeEpPath)
if err != nil {
panic(err)
}
return len(files) > 0
}
func GetLocalAnimeEpProperties(animeEp model.AnimeEpisode) (*FilePriority, bool, error) {
animeEpPath := filepath.Join(AnimePath, GetAnimeEpFilepath(animeEp, "*"))
files, err := filepath.Glob(animeEpPath)
if err != nil {
return nil, false, ErrInvalidGlobSyntax.Wrap(err, animeEpPath)
}
var mostPrio *FilePriority
for _, file := range files {
props, err := AnalyzeFile(file)
if err != nil {
continue
}
if !HasFileEssentialProperties(props) {
continue
}
fp := NewFilePriority(props)
if mostPrio == nil || fp.Priority > mostPrio.Priority {
mostPrio = fp
}
}
return mostPrio, mostPrio != nil, nil
}
// TODO cache?
func TorrentFileDownloading(torrent *model.ParsedTorrent) bool {
torrentFile := filepath.Join(TorrentPath, fmt.Sprintf("%s.torrent", torrent.Torrent.ID))
if _, err := os.Stat(torrentFile); err == nil {
return true
}
addedTorrentFile := filepath.Join(TorrentPath, fmt.Sprintf("%s.torrent.added", torrent.Torrent.ID))
if _, err := os.Stat(addedTorrentFile); err == nil {
return true
}
return false
}

36
main.go
View File

@ -22,8 +22,8 @@ var (
StorageUser = envvars.String("STORAGE_USER", "")
StoragePass = envvars.String("STORAGE_PASS", "")
DownloadPath = envvars.String("DOWNLOAD_PATH", "")
AnimePath = envvars.String("ANIME_PATH", "")
TorrentPath = envvars.String("TORRENT_PATH", "")
AnimePath = envvars.String("ANIME_PATH", "")
AnimeEpFilepathPattern = envvars.Object(
"EPISODE_FILEPATH_PATTERN",
@ -111,24 +111,34 @@ func checkTorrents() {
parsedTorrents := ParseTorrentsByAnimeEpSortedByProperties(torrents)
for animeEp, torrentPriorities := range parsedTorrents {
fmt.Printf("\nanime: %s | episode: %d | torrents found: %d\n", animeEp.Anime.Title.Romaji, animeEp.Episode, len(torrentPriorities))
_, animeOnList := animes[animeEp.Anime.ID]
props, found, err := GetLocalAnimeEpProperties(animeEp)
episodeInCollection := err == nil && found
fmt.Print(FormatFilePriority(animeEp, props, animeOnList, episodeInCollection, len(torrentPriorities)))
if !animeOnList {
continue
}
for _, torrentPriority := range torrentPriorities {
/*animeListEntry*/ _, ok := animes[torrentPriority.ParsedTorrent.Anime.ID]
if !ok {
fmt.Printf("%s | NOT ON LIST\n", FormatTorrentPriority(torrentPriority))
if TorrentFileDownloading(torrentPriority.ParsedTorrent) {
fmt.Printf("%s | CURRENTLY DOWNLOADING\n", FormatTorrentPriority(torrentPriority))
continue
}
if AnimeEpExistsLocally(animeEp) {
// TODO check if current torrent has higher priority than downloaded one (might not have any priority)
continue
if episodeInCollection {
if props.Priority > torrentPriority.Priority {
fmt.Printf("%s | LOWER PRIORITY\n", FormatTorrentPriority(torrentPriority))
} else {
fmt.Printf("%s | HIGHER PRIORITY | STARTING DOWNLOAD\n", FormatTorrentPriority(torrentPriority))
// TODO start download
}
} else {
fmt.Printf("%s | NOT IN COLLECTION | STARTING DOWNLOAD\n", FormatTorrentPriority(torrentPriority))
// TODO start download
}
// TODO is currently downloading? (database query)
fmt.Println(FormatTorrentPriority(torrentPriority))
// TODO download anime episode with highest priority (first one in slice)
}
}

0
parse_preferred_props.go → preferred_props.go
View File

10
torrent_sort.go
View File

@ -19,26 +19,26 @@ func SortTorrentsByPreferredProperties(torrents map[model.AnimeEpisode][]*model.
return torrentsWithPrio
}
func DeterminePriority(torrent *model.ParsedTorrent) (priority int, preferredProperties map[string]int) {
func DeterminePriority(props model.PropertyHolder) (priority int, preferredProperties map[string]int) {
preferredProperties = map[string]int{}
for _, lang := range torrent.Languages {
for _, lang := range props.GetLanguages() {
if langPriority, ok := PreferredLanguages[lang]; ok {
priority += langPriority
preferredProperties["lang/"+lang] = langPriority
}
}
for _, sub := range torrent.Subtitles {
for _, sub := range props.GetSubtitles() {
if subPriority, ok := PreferredSubtitles[sub]; ok {
priority += subPriority
preferredProperties["sub/"+sub] = subPriority
}
}
if prefRes, ok := PreferredResolutions[torrent.Resolution]; ok {
if prefRes, ok := PreferredResolutions[props.GetResolution()]; ok {
priority += prefRes
preferredProperties["res/"+torrent.Resolution.String()] = prefRes
preferredProperties["res/"+props.GetResolution().String()] = prefRes
}
return

27
utils.go
View File

@ -6,6 +6,7 @@ import (
"git.milar.in/milarin/anilist"
"git.milar.in/milarin/slices"
"git.milar.in/nyaanime/model"
)
var AllMediaListStatuses = []anilist.MediaListStatus{
@ -57,3 +58,29 @@ func FormatTorrentPriority(torrentPriority *TorrentPriority) string {
torrentPriority.Priority,
)
}
func FormatFilePriority(animeEp model.AnimeEpisode, props *FilePriority, onList, inCollection bool, torrentCount int) string {
if !onList {
return fmt.Sprintf("\nanime: %s | episode: %d | torrents found: %d | NOT ON LIST\n",
animeEp.Anime.Title.Romaji,
animeEp.Episode,
torrentCount,
)
}
if inCollection {
return fmt.Sprintf("\nanime: %s | episode: %d | torrents found: %d | IN COLLECTION | file properties: %s | file priority: %d\n",
animeEp.Anime.Title.Romaji,
animeEp.Episode,
torrentCount,
Map2Str(props.PreferredProperties),
props.Priority,
)
}
return fmt.Sprintf("\nanime: %s | episode: %d | torrents found: %d\n",
animeEp.Anime.Title.Romaji,
animeEp.Episode,
torrentCount,
)
}