From 95f77cf189b6788df70ff59804937bdb1f622fe8 Mon Sep 17 00:00:00 2001 From: Timon Ringwald Date: Mon, 22 Aug 2022 14:14:14 +0200 Subject: [PATCH] check local files for already downloaded torrents and currently downloading torrents --- anime_episode_filepath.go | 40 --------- errors.go | 2 + file_filter.go | 25 ++++++ file_priority.go | 17 ++++ file_properties.go | 79 +++++++++++++++++ go.mod | 5 +- go.sum | 16 ++-- local_file_check.go | 87 +++++++++++++++++++ main.go | 36 +++++--- ...e_preferred_props.go => preferred_props.go | 0 torrent_sort.go | 10 +-- utils.go | 27 ++++++ 12 files changed, 276 insertions(+), 68 deletions(-) delete mode 100644 anime_episode_filepath.go create mode 100644 file_filter.go create mode 100644 file_priority.go create mode 100644 file_properties.go create mode 100644 local_file_check.go rename parse_preferred_props.go => preferred_props.go (100%) diff --git a/anime_episode_filepath.go b/anime_episode_filepath.go deleted file mode 100644 index a01c12a..0000000 --- a/anime_episode_filepath.go +++ /dev/null @@ -1,40 +0,0 @@ -package main - -import ( - "path/filepath" - "strings" - - "git.milar.in/milarin/anilist" - "git.milar.in/nyaanime/model" -) - -type AnimePathPatternData struct { - Title anilist.MediaTitle - Episode int - Ext string -} - -func GetAnimeEpFilepath(animeEp model.AnimeEpisode, ext string) string { - tmplData := AnimePathPatternData{ - Title: animeEp.Anime.Title, - Episode: animeEp.Episode, - Ext: ext, - } - - b := new(strings.Builder) - if err := AnimeEpFilepathPattern.Execute(b, tmplData); err != nil { - panic(err) - } - return b.String() -} - -func AnimeEpExistsLocally(animeEp model.AnimeEpisode) bool { - animeEpPath := filepath.Join(AnimePath, GetAnimeEpFilepath(animeEp, "*")) - - files, err := filepath.Glob(animeEpPath) - if err != nil { - panic(err) - } - - return len(files) > 0 -} diff --git a/errors.go b/errors.go index f8a8693..c39bb05 100644 --- a/errors.go +++ b/errors.go @@ -12,4 +12,6 @@ var ( ErrAnimeListNotObtainable = adverr.NewErrTmpl("ErrAnimeListNotObtainable", "anime list from anilist.co not obtainable (reason: %s)") ErrTorrentNotObtainable = adverr.NewErrTmpl("ErrTorrentNotObtainable", "torrents from nyaa.si not obtainable (reason: %s)") ErrInvalidAnimeStatus = adverr.NewErrTmpl("ErrInvalidAnimeStatus", "invalid status '%s' in ANIME_STATUS (allowed: %s)") + ErrInvalidGlobSyntax = adverr.NewErrTmpl("ErrInvalidGlobSyntax", "invalid filepath.Glob syntax: '%s'") + ErrNoSuitableFileFound = adverr.NewErrTmpl("ErrNoSuitableFileFound", "no file found with essential properties for anime '%s' and episode %d") ) diff --git a/file_filter.go b/file_filter.go new file mode 100644 index 0000000..c31a2ee --- /dev/null +++ b/file_filter.go @@ -0,0 +1,25 @@ +package main + +import ( + "git.milar.in/milarin/slices" +) + +func HasFileEssentialProperties(props *FileProperties) bool { + if props.Resolution < MinResolution || props.Resolution > MaxResolution { + return false + } + + for _, essentialLanguage := range EssentialLanguages { + if !slices.Contains(props.Languages, essentialLanguage) { + return false + } + } + + for _, essentialSubtitle := range EssentialSubtitles { + if !slices.Contains(props.Subtitles, essentialSubtitle) { + return false + } + } + + return true +} diff --git a/file_priority.go b/file_priority.go new file mode 100644 index 0000000..48f14d7 --- /dev/null +++ b/file_priority.go @@ -0,0 +1,17 @@ +package main + +type FilePriority struct { + Properties *FileProperties + Priority int + PreferredProperties map[string]int +} + +func NewFilePriority(props *FileProperties) *FilePriority { + priority, preferredProperties := DeterminePriority(props) + + return &FilePriority{ + Properties: props, + Priority: priority, + PreferredProperties: preferredProperties, + } +} diff --git a/file_properties.go b/file_properties.go new file mode 100644 index 0000000..3745e0d --- /dev/null +++ b/file_properties.go @@ -0,0 +1,79 @@ +package main + +import ( + "context" + "fmt" + "os" + + "git.milar.in/nyaanime/model" + "git.milar.in/nyaanime/parsers" + ffprobe "gopkg.in/vansante/go-ffprobe.v2" +) + +type FileProperties struct { + Filepath string + Languages []string + Subtitles []string + Resolution model.Resolution +} + +var _ model.PropertyHolder = &FileProperties{} + +// TODO cache +func AnalyzeFile(path string) (*FileProperties, error) { + props := &FileProperties{Filepath: path} + + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + + data, err := ffprobe.ProbeReader(context.Background(), file) + if err != nil { + return nil, err + } + + defaultVideoLang := "" + for _, s := range data.StreamType(ffprobe.StreamVideo) { + if s.Disposition.Default > 0 { + props.Resolution = model.Resolution(s.Height) + defaultVideoLang = parsers.ParseLanguage(s.Tags.Language) + break + } + } + + for _, s := range data.StreamType(ffprobe.StreamAudio) { + if s.Tags.Language != "" { + props.Languages = append(props.Languages, parsers.ParseLanguage(s.Tags.Language)) + } else if s.Disposition.Default > 0 { + props.Languages = append(props.Languages, defaultVideoLang) + } + } + + for _, s := range data.StreamType(ffprobe.StreamSubtitle) { + if s.Tags.Language != "" { + props.Subtitles = append(props.Subtitles, parsers.ParseLanguage(s.Tags.Language)) + } else if s.Disposition.Default > 0 { + props.Subtitles = append(props.Subtitles, defaultVideoLang) + } + } + + return props, nil +} + +func (fp *FileProperties) GetLanguages() []string { + return fp.Languages +} + +func (fp *FileProperties) GetSubtitles() []string { + return fp.Subtitles +} + +func (fp *FileProperties) GetResolution() model.Resolution { + return fp.Resolution +} + +func (fp *FileProperties) String() string { + return fmt.Sprintf("") +} diff --git a/go.mod b/go.mod index f48fb33..e57d899 100644 --- a/go.mod +++ b/go.mod @@ -9,9 +9,10 @@ require ( git.milar.in/milarin/envvars v1.0.3 git.milar.in/milarin/gmath v0.0.2 git.milar.in/milarin/slices v0.0.2 - git.milar.in/nyaanime/model v0.0.0-20220821215714-9959e681573f - git.milar.in/nyaanime/parsers v0.0.0-20220815144327-52de61265e27 + git.milar.in/nyaanime/model v0.0.0-20220822093541-87208e95e7ac + git.milar.in/nyaanime/parsers v0.0.0-20220822100125-2813a7868f6a github.com/PuerkitoBio/goquery v1.8.0 + gopkg.in/vansante/go-ffprobe.v2 v2.1.0 ) require ( diff --git a/go.sum b/go.sum index 5be7d85..61a5b6b 100644 --- a/go.sum +++ b/go.sum @@ -10,20 +10,18 @@ git.milar.in/milarin/gmath v0.0.2 h1:avz+75f8XqAYA1wEB6kis0R5xvRuepBKTqBuJBjh6Yw git.milar.in/milarin/gmath v0.0.2/go.mod h1:HDLftG5RLpiNGKiIWh+O2G1PYkNzyLDADO8Cd/1abiE= git.milar.in/milarin/slices v0.0.2 h1:j92MuP0HWKSaHJMq/FRxDtSDIGiOTvw+KogUTwuulr0= git.milar.in/milarin/slices v0.0.2/go.mod h1:XRNfE99aNKeaPOY1phjOlpIQqeGCW1LOqqh8UHS+vAk= -git.milar.in/nyaanime/model v0.0.0-20220821124037-0a28c6b41556 h1:RplYz4+CMK9ByI3ELusBltWFRlDs6VMOGk5EyENnLi0= -git.milar.in/nyaanime/model v0.0.0-20220821124037-0a28c6b41556/go.mod h1:OzhQgj0b/Hf9fg8VXYxYt8ONQOvHm8xC44TmS9kQ150= -git.milar.in/nyaanime/model v0.0.0-20220821212334-5e9f052f86b6 h1:vmc1G5uVkYAYJYcaQKLrr+JE6u2780YR6R00EdF5A5I= -git.milar.in/nyaanime/model v0.0.0-20220821212334-5e9f052f86b6/go.mod h1:OzhQgj0b/Hf9fg8VXYxYt8ONQOvHm8xC44TmS9kQ150= -git.milar.in/nyaanime/model v0.0.0-20220821212807-44de43f4c500 h1:zwYRzcbRiS62hDu6hy4OGxMbhLnTsL13VEXNIoTodsk= -git.milar.in/nyaanime/model v0.0.0-20220821212807-44de43f4c500/go.mod h1:OzhQgj0b/Hf9fg8VXYxYt8ONQOvHm8xC44TmS9kQ150= -git.milar.in/nyaanime/model v0.0.0-20220821215714-9959e681573f h1:BKxr1hWCl2QfSiOis1NaRcoG1mZlwo5VnB+0mYhmfEw= -git.milar.in/nyaanime/model v0.0.0-20220821215714-9959e681573f/go.mod h1:OzhQgj0b/Hf9fg8VXYxYt8ONQOvHm8xC44TmS9kQ150= +git.milar.in/nyaanime/model v0.0.0-20220822093541-87208e95e7ac h1:rM5Mpo4/OJuZaBNZdylag+gi8giWVwDbqsoPjhDP9+g= +git.milar.in/nyaanime/model v0.0.0-20220822093541-87208e95e7ac/go.mod h1:OzhQgj0b/Hf9fg8VXYxYt8ONQOvHm8xC44TmS9kQ150= git.milar.in/nyaanime/parsers v0.0.0-20220815144327-52de61265e27 h1:0+5j9MMJQS8+Luss19hD6hvNFxcBDRal2XwSUTyq7WU= git.milar.in/nyaanime/parsers v0.0.0-20220815144327-52de61265e27/go.mod h1:qm6fIFBFs90uz7IJ8RKgDir0K8Fa8isixGLgrtC6kgU= +git.milar.in/nyaanime/parsers v0.0.0-20220822100125-2813a7868f6a h1:7vrKOL/vpqJ8YFZ9tmq9iPLoBuLnZgptHWaScyFOFFo= +git.milar.in/nyaanime/parsers v0.0.0-20220822100125-2813a7868f6a/go.mod h1:qm6fIFBFs90uz7IJ8RKgDir0K8Fa8isixGLgrtC6kgU= github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/vansante/go-ffprobe v1.1.0 h1:Tz5X+38tF8YYEFVz+PUTrtvlED35IorB7XI0USOqZWU= +github.com/vansante/go-ffprobe v1.1.0/go.mod h1:AEIxsTWYTTeXpel90yu5J/QxuDWNaKCO50xRBN4rdac= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220812174116-3211cb980234 h1:RDqmgfe7SvlMWoqC3xwQ2blLO3fcWcxMa3eBLRdRW7E= golang.org/x/net v0.0.0-20220812174116-3211cb980234/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= @@ -32,3 +30,5 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/vansante/go-ffprobe.v2 v2.1.0 h1:Gh8oVkvOSZG/DgEMmBw8h4oLvJhSQHROEGp3TaVGq08= +gopkg.in/vansante/go-ffprobe.v2 v2.1.0/go.mod h1:qF0AlAjk7Nqzqf3y333Ly+KxN3cKF2JqA3JT5ZheUGE= diff --git a/local_file_check.go b/local_file_check.go new file mode 100644 index 0000000..2feaa45 --- /dev/null +++ b/local_file_check.go @@ -0,0 +1,87 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "git.milar.in/milarin/anilist" + "git.milar.in/nyaanime/model" +) + +type AnimePathPatternData struct { + Title anilist.MediaTitle + Episode int + Ext string +} + +func GetAnimeEpFilepath(animeEp model.AnimeEpisode, ext string) string { + tmplData := AnimePathPatternData{ + Title: animeEp.Anime.Title, + Episode: animeEp.Episode, + Ext: ext, + } + + b := new(strings.Builder) + if err := AnimeEpFilepathPattern.Execute(b, tmplData); err != nil { + panic(err) + } + return b.String() +} + +func AnimeEpExistsLocally(animeEp model.AnimeEpisode) bool { + animeEpPath := filepath.Join(AnimePath, GetAnimeEpFilepath(animeEp, "*")) + + files, err := filepath.Glob(animeEpPath) + if err != nil { + panic(err) + } + + return len(files) > 0 +} + +func GetLocalAnimeEpProperties(animeEp model.AnimeEpisode) (*FilePriority, bool, error) { + animeEpPath := filepath.Join(AnimePath, GetAnimeEpFilepath(animeEp, "*")) + files, err := filepath.Glob(animeEpPath) + if err != nil { + return nil, false, ErrInvalidGlobSyntax.Wrap(err, animeEpPath) + } + + var mostPrio *FilePriority + + for _, file := range files { + props, err := AnalyzeFile(file) + if err != nil { + continue + } + + if !HasFileEssentialProperties(props) { + continue + } + + fp := NewFilePriority(props) + + if mostPrio == nil || fp.Priority > mostPrio.Priority { + mostPrio = fp + } + } + + return mostPrio, mostPrio != nil, nil +} + +// TODO cache? +func TorrentFileDownloading(torrent *model.ParsedTorrent) bool { + torrentFile := filepath.Join(TorrentPath, fmt.Sprintf("%s.torrent", torrent.Torrent.ID)) + + if _, err := os.Stat(torrentFile); err == nil { + return true + } + + addedTorrentFile := filepath.Join(TorrentPath, fmt.Sprintf("%s.torrent.added", torrent.Torrent.ID)) + if _, err := os.Stat(addedTorrentFile); err == nil { + return true + } + + return false +} diff --git a/main.go b/main.go index 72b3b04..d757bb7 100644 --- a/main.go +++ b/main.go @@ -22,8 +22,8 @@ var ( StorageUser = envvars.String("STORAGE_USER", "") StoragePass = envvars.String("STORAGE_PASS", "") - DownloadPath = envvars.String("DOWNLOAD_PATH", "") - AnimePath = envvars.String("ANIME_PATH", "") + TorrentPath = envvars.String("TORRENT_PATH", "") + AnimePath = envvars.String("ANIME_PATH", "") AnimeEpFilepathPattern = envvars.Object( "EPISODE_FILEPATH_PATTERN", @@ -111,24 +111,34 @@ func checkTorrents() { parsedTorrents := ParseTorrentsByAnimeEpSortedByProperties(torrents) for animeEp, torrentPriorities := range parsedTorrents { - fmt.Printf("\nanime: %s | episode: %d | torrents found: %d\n", animeEp.Anime.Title.Romaji, animeEp.Episode, len(torrentPriorities)) + _, animeOnList := animes[animeEp.Anime.ID] + props, found, err := GetLocalAnimeEpProperties(animeEp) + episodeInCollection := err == nil && found + + fmt.Print(FormatFilePriority(animeEp, props, animeOnList, episodeInCollection, len(torrentPriorities))) + + if !animeOnList { + continue + } for _, torrentPriority := range torrentPriorities { - /*animeListEntry*/ _, ok := animes[torrentPriority.ParsedTorrent.Anime.ID] - if !ok { - fmt.Printf("%s | NOT ON LIST\n", FormatTorrentPriority(torrentPriority)) + if TorrentFileDownloading(torrentPriority.ParsedTorrent) { + fmt.Printf("%s | CURRENTLY DOWNLOADING\n", FormatTorrentPriority(torrentPriority)) continue } - if AnimeEpExistsLocally(animeEp) { - // TODO check if current torrent has higher priority than downloaded one (might not have any priority) - continue + if episodeInCollection { + if props.Priority > torrentPriority.Priority { + fmt.Printf("%s | LOWER PRIORITY\n", FormatTorrentPriority(torrentPriority)) + } else { + fmt.Printf("%s | HIGHER PRIORITY | STARTING DOWNLOAD\n", FormatTorrentPriority(torrentPriority)) + // TODO start download + } + } else { + fmt.Printf("%s | NOT IN COLLECTION | STARTING DOWNLOAD\n", FormatTorrentPriority(torrentPriority)) + // TODO start download } - // TODO is currently downloading? (database query) - - fmt.Println(FormatTorrentPriority(torrentPriority)) - // TODO download anime episode with highest priority (first one in slice) } } diff --git a/parse_preferred_props.go b/preferred_props.go similarity index 100% rename from parse_preferred_props.go rename to preferred_props.go diff --git a/torrent_sort.go b/torrent_sort.go index ed19d0a..bdf6ac5 100644 --- a/torrent_sort.go +++ b/torrent_sort.go @@ -19,26 +19,26 @@ func SortTorrentsByPreferredProperties(torrents map[model.AnimeEpisode][]*model. return torrentsWithPrio } -func DeterminePriority(torrent *model.ParsedTorrent) (priority int, preferredProperties map[string]int) { +func DeterminePriority(props model.PropertyHolder) (priority int, preferredProperties map[string]int) { preferredProperties = map[string]int{} - for _, lang := range torrent.Languages { + for _, lang := range props.GetLanguages() { if langPriority, ok := PreferredLanguages[lang]; ok { priority += langPriority preferredProperties["lang/"+lang] = langPriority } } - for _, sub := range torrent.Subtitles { + for _, sub := range props.GetSubtitles() { if subPriority, ok := PreferredSubtitles[sub]; ok { priority += subPriority preferredProperties["sub/"+sub] = subPriority } } - if prefRes, ok := PreferredResolutions[torrent.Resolution]; ok { + if prefRes, ok := PreferredResolutions[props.GetResolution()]; ok { priority += prefRes - preferredProperties["res/"+torrent.Resolution.String()] = prefRes + preferredProperties["res/"+props.GetResolution().String()] = prefRes } return diff --git a/utils.go b/utils.go index 5388e8b..b520561 100644 --- a/utils.go +++ b/utils.go @@ -6,6 +6,7 @@ import ( "git.milar.in/milarin/anilist" "git.milar.in/milarin/slices" + "git.milar.in/nyaanime/model" ) var AllMediaListStatuses = []anilist.MediaListStatus{ @@ -57,3 +58,29 @@ func FormatTorrentPriority(torrentPriority *TorrentPriority) string { torrentPriority.Priority, ) } + +func FormatFilePriority(animeEp model.AnimeEpisode, props *FilePriority, onList, inCollection bool, torrentCount int) string { + if !onList { + return fmt.Sprintf("\nanime: %s | episode: %d | torrents found: %d | NOT ON LIST\n", + animeEp.Anime.Title.Romaji, + animeEp.Episode, + torrentCount, + ) + } + + if inCollection { + return fmt.Sprintf("\nanime: %s | episode: %d | torrents found: %d | IN COLLECTION | file properties: %s | file priority: %d\n", + animeEp.Anime.Title.Romaji, + animeEp.Episode, + torrentCount, + Map2Str(props.PreferredProperties), + props.Priority, + ) + } + + return fmt.Sprintf("\nanime: %s | episode: %d | torrents found: %d\n", + animeEp.Anime.Title.Romaji, + animeEp.Episode, + torrentCount, + ) +}