Merge pull request 'feature/add-genius' (#5) from feature/add-genius into main
Reviewed-on: #5
This commit is contained in:
commit
b37b2a2cfb
|
|
@ -0,0 +1,12 @@
|
||||||
|
## v0.1.0 (2024-04-20)
|
||||||
|
|
||||||
|
* Refactoring draft notes generation [View](https://git.iratusmachina.com/iratusmachina/lyricdownloader/commits/6bb0da8f79b97e1a59b77cb59ba206db1cea9574)
|
||||||
|
* Refactoring draft notes generation [View](https://git.iratusmachina.com/iratusmachina/lyricdownloader/commits/a41ce583d39926b39f2ca4c10f4f901f8952e005)
|
||||||
|
* fixing errcheck errors [View](https://git.iratusmachina.com/iratusmachina/lyricdownloader/commits/1ff8fde8384a34b2286918cd194ba081addfdf83)
|
||||||
|
* Adding a new flag [View](https://git.iratusmachina.com/iratusmachina/lyricdownloader/commits/555dbc2e0ddcc4b4c256407883c2486ae750b680)
|
||||||
|
* Adding woodpecker build files [View](https://git.iratusmachina.com/iratusmachina/lyricdownloader/commits/879f7473942b3f6cbef79af909d3edc1c5d736cc)
|
||||||
|
* Able to scrape lyrics from Google Search [View](https://git.iratusmachina.com/iratusmachina/lyricdownloader/commits/38f64dcb9293ee5fce323d14083be9cc8d75e918)
|
||||||
|
* Adding LICENSE [View](https://git.iratusmachina.com/iratusmachina/lyricdownloader/commits/a17fe93f2ef8320db4d20cb9c7cab4a0888ea295)
|
||||||
|
* Adding simple Makefile [View](https://git.iratusmachina.com/iratusmachina/lyricdownloader/commits/23f3d3b1356e89c03114cf8ee840d8df18f98ef2)
|
||||||
|
* Initial config [View](https://git.iratusmachina.com/iratusmachina/lyricdownloader/commits/a4284d56ec78356a5a152e3cc860eed2c5957f1e)
|
||||||
|
|
||||||
27
README.md
27
README.md
|
|
@ -1,10 +1,31 @@
|
||||||
# Lyricdownloader
|
# Lyricdownloader
|
||||||
|
|
||||||
This program is used to download lyrics for a song from the internet. The steps of operation are shown here:
|
This program is used to download lyrics for a song from the internet. There are two modes of operation:
|
||||||
|
|
||||||
1. It first opens a chrome window, searches for the lyrics , and copies the lyrics returned by Google Search to a file defined by you.
|
(a) Scraping from google: This does not run in headless mode as Google does not show the lyrics in headless mode. So the program has to first open a chrome window, searches for the lyrics, and copies the lyrics returned by Google Search to a file defined by you.
|
||||||
2. It then tries to get search for the same song using the Genius API. It then tries to compare the lyrics with the Genius one.
|
(b) Retrieving the lyrics using the Genius API.
|
||||||
|
|
||||||
## Caveats
|
## Caveats
|
||||||
|
|
||||||
This program requires chrome to use, that is, you should have chrome installed to use this program.
|
This program requires chrome to use, that is, you should have chrome installed to use this program.
|
||||||
|
|
||||||
|
## TODOs
|
||||||
|
|
||||||
|
- [x] ~~Refactor internal folder to this structure.~~ Simplify package structure.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
internal
|
||||||
|
| - usegenius.go # where you go all the genius processing
|
||||||
|
| - usegoogle.go # where you do all the google processing
|
||||||
|
| - cmdline.go # Where you initialize config and use flags
|
||||||
|
```
|
||||||
|
|
||||||
|
- [x] Setup the config file in its own config.go file
|
||||||
|
- [x] Specific the doc for `go doc` in the doc.go file. To see the generated
|
||||||
|
documentation, run `go doc lyricdownloader`
|
||||||
|
- [ ] Research how to view `go doc` locally
|
||||||
|
- [x] Fix generate_notes.py to fix generation of CHANGELOG.md
|
||||||
|
- [ ] Add a demo heading in README.md
|
||||||
|
- [ ] Sync over to Github
|
||||||
|
- [ ] Add podman/Docker instructions
|
||||||
|
- [ ] Add goreleaser
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,54 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path"
|
||||||
|
)
|
||||||
|
|
||||||
|
// isFile - check if fp is a valid file
|
||||||
|
func isFile(fp string) bool {
|
||||||
|
info, err := os.Stat(fp)
|
||||||
|
if os.IsNotExist(err) || !info.Mode().IsRegular() {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// getConfig - parse config json file
|
||||||
|
func getConfig(filepath string) (Config, error) {
|
||||||
|
// if filepath is not empty
|
||||||
|
// try to load the token from the file
|
||||||
|
// else load it from user homedir
|
||||||
|
var config Config
|
||||||
|
configFilePath := filepath
|
||||||
|
ok := isFile(filepath)
|
||||||
|
if !ok {
|
||||||
|
if printDebug {
|
||||||
|
mainLog.Println("Loading config from home directory...")
|
||||||
|
}
|
||||||
|
|
||||||
|
dirname, err := os.UserConfigDir()
|
||||||
|
if err != nil {
|
||||||
|
return config, err
|
||||||
|
}
|
||||||
|
configFilePath := path.Join(dirname, "lyricdownloader/config.json")
|
||||||
|
ok := isFile(configFilePath)
|
||||||
|
if !ok {
|
||||||
|
return config, fmt.Errorf("%s/lyricdownloader/config.json file is not found", dirname)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
configFile, err := os.Open(configFilePath)
|
||||||
|
if err != nil {
|
||||||
|
return config, err
|
||||||
|
}
|
||||||
|
defer configFile.Close()
|
||||||
|
|
||||||
|
err = json.NewDecoder(configFile).Decode(&config)
|
||||||
|
if err != nil {
|
||||||
|
return config, err
|
||||||
|
}
|
||||||
|
return config, nil
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,46 @@
|
||||||
|
/*
|
||||||
|
lyricdownloader is used to download lyrics for a song
|
||||||
|
from the internet. There are two modes of operation.
|
||||||
|
|
||||||
|
The first mode is scraping the lyrics from Google. So the operation is this:
|
||||||
|
It first opens a chrome window, searches for the lyrics
|
||||||
|
, and copies the lyrics returned by Google Search to a
|
||||||
|
file defined by you.
|
||||||
|
|
||||||
|
The second mode is getting the lyrics from Genius API. So the operation is this:
|
||||||
|
It then tries to get search for the same song using the
|
||||||
|
Genius API. It then tries to compare the lyrics with the
|
||||||
|
Genius one.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
|
||||||
|
lyricdownloader [flags]
|
||||||
|
|
||||||
|
The flags are:
|
||||||
|
|
||||||
|
-configFile string
|
||||||
|
Optional. Use with genius
|
||||||
|
-genius
|
||||||
|
Optional. Use genius
|
||||||
|
-google
|
||||||
|
Optional. Use google.
|
||||||
|
-help
|
||||||
|
Optional. Print Usage
|
||||||
|
-output string
|
||||||
|
Optional. Lyrics filename
|
||||||
|
-search string
|
||||||
|
Required. Name of song to search. If the name of the song is not a single word, put in quotes""
|
||||||
|
-verbose
|
||||||
|
Optional. Turn on debug. Default is false.
|
||||||
|
|
||||||
|
# Examples
|
||||||
|
|
||||||
|
To download song Jireh using google
|
||||||
|
|
||||||
|
lyricdownloader -search "Jireh" -google
|
||||||
|
|
||||||
|
To download song Jireh using genius
|
||||||
|
|
||||||
|
lyricdownloader -search "Jireh" -genius
|
||||||
|
*/
|
||||||
|
package main
|
||||||
|
|
@ -21,6 +21,20 @@ def full():
|
||||||
# Remove the first occurence of the word git starting from the end of the string
|
# Remove the first occurence of the word git starting from the end of the string
|
||||||
remote_url = remote_url[0:remote_url.rfind(".git")]
|
remote_url = remote_url[0:remote_url.rfind(".git")]
|
||||||
previous_tag = ""
|
previous_tag = ""
|
||||||
|
if len(tags) == 1:
|
||||||
|
# Extract the date of the commit
|
||||||
|
tag_date = subprocess.check_output(["git", "log", "-1", f"--pretty=format:'%ad'", "--date=short", f"{tags[0]}"], text=True)
|
||||||
|
tag_date = tag_date.replace("'", "")
|
||||||
|
|
||||||
|
# Get each commit of a tag formatted
|
||||||
|
formatted_lines = subprocess.check_output(["git", "log", f"{tags[0]}", f'--pretty=format:"* %s [View]({remote_url}/commits/%H)"'], text=True)
|
||||||
|
if formatted_lines:
|
||||||
|
fw.write(f"## {tags[0]} ({tag_date})\n\n")
|
||||||
|
# Remove merge commits or Changelog commits
|
||||||
|
lines = "\n".join([line.replace("\"", "") for line in formatted_lines.split("\n") if all(["merge" not in line.lower(), "changelog.md" not in line.lower()])])
|
||||||
|
fw.write(lines)
|
||||||
|
fw.write("\n\n")
|
||||||
|
else:
|
||||||
for tag in tags:
|
for tag in tags:
|
||||||
if previous_tag:
|
if previous_tag:
|
||||||
# Extract the date of the commit
|
# Extract the date of the commit
|
||||||
|
|
|
||||||
12
go.mod
12
go.mod
|
|
@ -4,17 +4,27 @@ go 1.20
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/antchfx/htmlquery v1.3.0
|
github.com/antchfx/htmlquery v1.3.0
|
||||||
github.com/joho/godotenv v1.5.1
|
github.com/gocolly/colly/v2 v2.1.0
|
||||||
github.com/playwright-community/playwright-go v0.4001.0
|
github.com/playwright-community/playwright-go v0.4001.0
|
||||||
golang.org/x/net v0.17.0
|
golang.org/x/net v0.17.0
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
github.com/PuerkitoBio/goquery v1.5.1 // indirect
|
||||||
|
github.com/andybalholm/cascadia v1.2.0 // indirect
|
||||||
|
github.com/antchfx/xmlquery v1.2.4 // indirect
|
||||||
github.com/antchfx/xpath v1.2.4 // indirect
|
github.com/antchfx/xpath v1.2.4 // indirect
|
||||||
github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 // indirect
|
github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 // indirect
|
||||||
github.com/go-jose/go-jose/v3 v3.0.1 // indirect
|
github.com/go-jose/go-jose/v3 v3.0.1 // indirect
|
||||||
github.com/go-stack/stack v1.8.1 // indirect
|
github.com/go-stack/stack v1.8.1 // indirect
|
||||||
|
github.com/gobwas/glob v0.2.3 // indirect
|
||||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||||
|
github.com/golang/protobuf v1.4.2 // indirect
|
||||||
|
github.com/kennygrant/sanitize v1.2.4 // indirect
|
||||||
|
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
|
||||||
|
github.com/temoto/robotstxt v1.1.1 // indirect
|
||||||
go.uber.org/multierr v1.11.0 // indirect
|
go.uber.org/multierr v1.11.0 // indirect
|
||||||
golang.org/x/text v0.13.0 // indirect
|
golang.org/x/text v0.13.0 // indirect
|
||||||
|
google.golang.org/appengine v1.6.6 // indirect
|
||||||
|
google.golang.org/protobuf v1.24.0 // indirect
|
||||||
)
|
)
|
||||||
|
|
|
||||||
101
go.sum
101
go.sum
|
|
@ -1,47 +1,115 @@
|
||||||
|
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||||
|
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||||
|
github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
|
||||||
|
github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
|
||||||
|
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
|
||||||
|
github.com/andybalholm/cascadia v1.2.0 h1:vuRCkM5Ozh/BfmsaTm26kbjm0mIOM3yS5Ek/F5h18aE=
|
||||||
|
github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY=
|
||||||
|
github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0=
|
||||||
github.com/antchfx/htmlquery v1.3.0 h1:5I5yNFOVI+egyia5F2s/5Do2nFWxJz41Tr3DyfKD25E=
|
github.com/antchfx/htmlquery v1.3.0 h1:5I5yNFOVI+egyia5F2s/5Do2nFWxJz41Tr3DyfKD25E=
|
||||||
github.com/antchfx/htmlquery v1.3.0/go.mod h1:zKPDVTMhfOmcwxheXUsx4rKJy8KEY/PU6eXr/2SebQ8=
|
github.com/antchfx/htmlquery v1.3.0/go.mod h1:zKPDVTMhfOmcwxheXUsx4rKJy8KEY/PU6eXr/2SebQ8=
|
||||||
|
github.com/antchfx/xmlquery v1.2.4 h1:T/SH1bYdzdjTMoz2RgsfVKbM5uWh3gjDYYepFqQmFv4=
|
||||||
|
github.com/antchfx/xmlquery v1.2.4/go.mod h1:KQQuESaxSlqugE2ZBcM/qn+ebIpt+d+4Xx7YcSGAIrM=
|
||||||
|
github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
|
||||||
|
github.com/antchfx/xpath v1.1.8/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
|
||||||
github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
|
github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
|
||||||
github.com/antchfx/xpath v1.2.4 h1:dW1HB/JxKvGtJ9WyVGJ0sIoEcqftV3SqIstujI+B9XY=
|
github.com/antchfx/xpath v1.2.4 h1:dW1HB/JxKvGtJ9WyVGJ0sIoEcqftV3SqIstujI+B9XY=
|
||||||
github.com/antchfx/xpath v1.2.4/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
|
github.com/antchfx/xpath v1.2.4/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
|
||||||
|
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||||
|
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||||
github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ=
|
github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ=
|
||||||
github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk=
|
github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk=
|
||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||||
|
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||||
github.com/go-jose/go-jose/v3 v3.0.1 h1:pWmKFVtt+Jl0vBZTIpz/eAKwsm6LkIxDVVbFHKkchhA=
|
github.com/go-jose/go-jose/v3 v3.0.1 h1:pWmKFVtt+Jl0vBZTIpz/eAKwsm6LkIxDVVbFHKkchhA=
|
||||||
github.com/go-jose/go-jose/v3 v3.0.1/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8=
|
github.com/go-jose/go-jose/v3 v3.0.1/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8=
|
||||||
github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw=
|
github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw=
|
||||||
github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4=
|
github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4=
|
||||||
|
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
|
||||||
|
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
|
||||||
|
github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA=
|
||||||
|
github.com/gocolly/colly/v2 v2.1.0 h1:k0DuZkDoCsx51bKpRJNEmcxcp+W5N8ziuwGaSDuFoGs=
|
||||||
|
github.com/gocolly/colly/v2 v2.1.0/go.mod h1:I2MuhsLjQ+Ex+IzK3afNS8/1qP3AedHOusRPcRdC5o0=
|
||||||
|
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||||
|
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
|
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
|
||||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||||
|
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||||
|
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||||
|
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||||
|
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||||
|
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
|
||||||
|
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
|
||||||
|
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
|
||||||
|
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
|
||||||
|
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
|
||||||
|
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
|
||||||
|
github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
|
||||||
|
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||||
|
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||||
|
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||||
|
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||||
|
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||||
|
github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
|
||||||
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||||
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
|
github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg=
|
||||||
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
|
||||||
|
github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
|
||||||
github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc=
|
github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc=
|
||||||
github.com/playwright-community/playwright-go v0.4001.0 h1:2cBiTIjCvFu7zUrZ48C0YC2DIp90Tbudueq4brUGjHM=
|
github.com/playwright-community/playwright-go v0.4001.0 h1:2cBiTIjCvFu7zUrZ48C0YC2DIp90Tbudueq4brUGjHM=
|
||||||
github.com/playwright-community/playwright-go v0.4001.0/go.mod h1:quEkYFrvvpQyGSxBjnYbGS52vrUDB2uaY1cOzkkSHCc=
|
github.com/playwright-community/playwright-go v0.4001.0/go.mod h1:quEkYFrvvpQyGSxBjnYbGS52vrUDB2uaY1cOzkkSHCc=
|
||||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
|
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||||
|
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI=
|
||||||
|
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
|
||||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
|
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
|
||||||
|
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||||
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
||||||
|
github.com/temoto/robotstxt v1.1.1 h1:Gh8RCs8ouX3hRSxxK7B1mO5RFByQ4CmJZDwgom++JaA=
|
||||||
|
github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
|
||||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||||
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||||
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
|
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||||
golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||||
|
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||||
|
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||||
|
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||||
|
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||||
|
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
|
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
|
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
|
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
|
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||||
|
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||||
|
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||||
|
golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||||
|
golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||||
golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
|
golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
|
||||||
golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
|
golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
|
||||||
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
|
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
|
||||||
|
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||||
|
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
|
@ -51,16 +119,45 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn
|
||||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||||
golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
|
golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
|
||||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
|
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||||
golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||||
golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
|
golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
|
||||||
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||||
|
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||||
|
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||||
|
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||||
|
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||||
|
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
|
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
||||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
|
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||||
|
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||||
|
google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
|
||||||
|
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||||
|
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||||
|
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||||
|
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
|
||||||
|
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||||
|
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
|
||||||
|
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||||
|
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||||
|
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||||
|
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||||
|
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
|
||||||
|
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
|
||||||
|
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||||
|
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||||
|
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||||
|
google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA=
|
||||||
|
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
|
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||||
|
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||||
|
|
|
||||||
|
|
@ -1,245 +0,0 @@
|
||||||
package cmdline
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"errors"
|
|
||||||
"flag"
|
|
||||||
"fmt"
|
|
||||||
"log"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/antchfx/htmlquery"
|
|
||||||
"github.com/playwright-community/playwright-go"
|
|
||||||
"golang.org/x/net/html"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
printDebug bool
|
|
||||||
useGenius bool
|
|
||||||
useGoogle bool
|
|
||||||
outputFile string
|
|
||||||
errorLog *log.Logger
|
|
||||||
mainLog *log.Logger
|
|
||||||
)
|
|
||||||
|
|
||||||
func recurseNodes(top *html.Node, sb *strings.Builder) {
|
|
||||||
if top.Type == html.ElementNode && top.Data == "span" {
|
|
||||||
sb.WriteString(htmlquery.InnerText(top) + "\n")
|
|
||||||
}
|
|
||||||
for c := top.FirstChild; c != nil; c = c.NextSibling {
|
|
||||||
recurseNodes(c, sb)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func searchGoogle(song string) (err error) {
|
|
||||||
runOption := &playwright.RunOptions{
|
|
||||||
SkipInstallBrowsers: true,
|
|
||||||
}
|
|
||||||
tempErr := playwright.Install(runOption)
|
|
||||||
if tempErr != nil {
|
|
||||||
err = fmt.Errorf("could not install playwright dependencies: %v", tempErr)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
pw, tempErr := playwright.Run()
|
|
||||||
if tempErr != nil {
|
|
||||||
err = fmt.Errorf("could not start playwright: %v", tempErr)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer func(pw *playwright.Playwright) {
|
|
||||||
tempErr := pw.Stop()
|
|
||||||
if tempErr != nil {
|
|
||||||
e := fmt.Errorf("could not stop Playwright: %v", tempErr)
|
|
||||||
err = errors.Join(err, e)
|
|
||||||
}
|
|
||||||
}(pw)
|
|
||||||
|
|
||||||
option := playwright.BrowserTypeLaunchOptions{
|
|
||||||
Channel: playwright.String("chrome"),
|
|
||||||
Headless: playwright.Bool(false),
|
|
||||||
}
|
|
||||||
browser, tempErr := pw.Chromium.Launch(option)
|
|
||||||
if tempErr != nil {
|
|
||||||
err = fmt.Errorf("could not launch browser: %v", tempErr)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func(browser playwright.Browser) {
|
|
||||||
tempErr = browser.Close()
|
|
||||||
if tempErr != nil {
|
|
||||||
e := fmt.Errorf("could not close browser: %v", tempErr)
|
|
||||||
err = errors.Join(err, e)
|
|
||||||
}
|
|
||||||
}(browser)
|
|
||||||
page, tempErr := browser.NewPage()
|
|
||||||
if tempErr != nil {
|
|
||||||
err = fmt.Errorf("could not create page: %v", tempErr)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if _, tempErr := page.Goto(fmt.Sprintf("https://www.google.com/search?q=%ss+lyrics", song),
|
|
||||||
playwright.PageGotoOptions{
|
|
||||||
WaitUntil: playwright.WaitUntilStateLoad,
|
|
||||||
}); tempErr != nil {
|
|
||||||
err = fmt.Errorf("could not goto: %v", tempErr)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
tempErr = page.Locator("body").WaitFor(playwright.LocatorWaitForOptions{
|
|
||||||
State: playwright.WaitForSelectorStateVisible,
|
|
||||||
})
|
|
||||||
if tempErr != nil {
|
|
||||||
err = fmt.Errorf("could not wait for body: %v", tempErr)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
html, tempErr := page.Locator("html").InnerHTML()
|
|
||||||
if tempErr != nil {
|
|
||||||
err = fmt.Errorf("could not get innerHtml: %v", tempErr)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
doc, tempErr := htmlquery.Parse(bytes.NewReader([]byte(html)))
|
|
||||||
if err != nil {
|
|
||||||
err = fmt.Errorf("could not parse the innerHtml: %v", tempErr)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
nodes, tempErr := htmlquery.QueryAll(doc, "//div[@data-lyricid]/div")
|
|
||||||
if err != nil {
|
|
||||||
err = fmt.Errorf("could not get the nodes: %v", tempErr)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
var sb strings.Builder
|
|
||||||
for _, node := range nodes {
|
|
||||||
recurseNodes(node, &sb)
|
|
||||||
}
|
|
||||||
if sb.Len() > 0 {
|
|
||||||
if printDebug {
|
|
||||||
mainLog.Println("Writing lyrics from Google...")
|
|
||||||
}
|
|
||||||
filename := fmt.Sprintf("%s_google.txt", outputFile)
|
|
||||||
tempErr = os.WriteFile(filename, []byte(sb.String()), os.ModePerm)
|
|
||||||
if tempErr != nil {
|
|
||||||
err = fmt.Errorf("could not write to %s: %v", filename, err)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
mainLog.Println("Lyrics cannot be found...")
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// func searchGenius(song string, lg *log.Logger) {
|
|
||||||
|
|
||||||
// }
|
|
||||||
|
|
||||||
func Main() int {
|
|
||||||
programName := os.Args[0]
|
|
||||||
errorLog = log.New(os.Stderr, "", 0)
|
|
||||||
mainLog = log.New(os.Stdout, "", 0)
|
|
||||||
|
|
||||||
flags := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
|
|
||||||
flags.Usage = func() {
|
|
||||||
out := flags.Output()
|
|
||||||
fmt.Fprintf(out, "Usage: %v \n\n", programName)
|
|
||||||
fmt.Fprint(out, " This program is used to download lyrics for a song\n")
|
|
||||||
fmt.Fprint(out, " from the internet. There are two modes of operation.\n\n")
|
|
||||||
fmt.Fprint(out, " The first mode is scraping the lyrics from Google. So the operation is this: \n")
|
|
||||||
fmt.Fprint(out, " It first opens a chrome window, searches for the lyrics \n")
|
|
||||||
fmt.Fprint(out, " , and copies the lyrics returned by Google Search to a \n")
|
|
||||||
fmt.Fprint(out, " file defined by you.\n\n")
|
|
||||||
fmt.Fprint(out, " The second mode is getting the lyrics from Genius API. So the operation is this: \n")
|
|
||||||
fmt.Fprint(out, " It then tries to get search for the same song using the \n")
|
|
||||||
fmt.Fprint(out, " Genius API. It then tries to compare the lyrics with the \n")
|
|
||||||
fmt.Fprint(out, " Genius one.\n\n")
|
|
||||||
flags.PrintDefaults()
|
|
||||||
}
|
|
||||||
|
|
||||||
outputFlag := flags.String("output", "", "Optional. Lyrics filename")
|
|
||||||
verboseFlag := flags.Bool("verbose", false, "Optional. Turn on debug. Default is false.")
|
|
||||||
searchFlag := flags.String("search", "", "Required. Name of song to search. If the name of the song is not a single word, put in quotes\"\"")
|
|
||||||
helpFlag := flags.Bool("help", false, "Optional. Print Usage")
|
|
||||||
useGoogleFlag := flags.Bool("google", false, "Optional. Use google.")
|
|
||||||
useGeniusFlag := flags.Bool("genius", false, "Optional. Use genius")
|
|
||||||
|
|
||||||
err := flags.Parse(os.Args[1:])
|
|
||||||
if err != nil {
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(flags.Args()) > 1 {
|
|
||||||
errorLog.Println("Error: too many command-line arguments")
|
|
||||||
flags.Usage()
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
allSetFlags := flagsSet(flags)
|
|
||||||
|
|
||||||
if allSetFlags["help"] && (allSetFlags["output"] || allSetFlags["search"] || allSetFlags["verbose"]) {
|
|
||||||
errorLog.Println("Error: if -help is set, -output, -search and -verbose must remain unset")
|
|
||||||
flags.Usage()
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
if !allSetFlags["google"] && !allSetFlags["genius"] {
|
|
||||||
errorLog.Println("Error: One of -google or -genius must be set")
|
|
||||||
flags.Usage()
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
if allSetFlags["google"] && allSetFlags["genius"] {
|
|
||||||
errorLog.Println("Error: if -google is set, -genius must remain unset and vice versa")
|
|
||||||
flags.Usage()
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
if *helpFlag {
|
|
||||||
flags.Usage()
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
songToSearch := *searchFlag
|
|
||||||
|
|
||||||
if len(songToSearch) == 0 {
|
|
||||||
errorLog.Println("Error: the song name must be provided for search")
|
|
||||||
flags.Usage()
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
if allSetFlags["output"] {
|
|
||||||
outputFile = *outputFlag
|
|
||||||
} else {
|
|
||||||
mainLog.Printf("Using %s as the name of the file(s) for downloaded lyrics..\n", songToSearch)
|
|
||||||
outputFile = fmt.Sprintf("%s_lyrics", songToSearch)
|
|
||||||
}
|
|
||||||
|
|
||||||
printDebug = *verboseFlag
|
|
||||||
if printDebug {
|
|
||||||
mainLog.Printf("Output flag: %s, Debug flag: %t, Search flag: %s\n", outputFile, printDebug, songToSearch)
|
|
||||||
}
|
|
||||||
|
|
||||||
useGenius = *useGeniusFlag
|
|
||||||
useGoogle = *useGoogleFlag
|
|
||||||
|
|
||||||
if useGoogle {
|
|
||||||
err := searchGoogle(songToSearch)
|
|
||||||
if err != nil {
|
|
||||||
errorLog.Printf("Err: %+v", err)
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if useGenius {
|
|
||||||
fmt.Println("No op")
|
|
||||||
}
|
|
||||||
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
// flagsSet returns a set of all the flags what were actually set on the
|
|
||||||
// command line.
|
|
||||||
func flagsSet(flags *flag.FlagSet) map[string]bool {
|
|
||||||
s := make(map[string]bool)
|
|
||||||
flags.Visit(func(f *flag.Flag) {
|
|
||||||
s[f.Name] = true
|
|
||||||
})
|
|
||||||
return s
|
|
||||||
}
|
|
||||||
152
main.go
152
main.go
|
|
@ -1,11 +1,10 @@
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"lyricdownloader/internal/cmdline"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
_ "github.com/joho/godotenv/autoload"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Possible algorithm
|
// Possible algorithm
|
||||||
|
|
@ -15,12 +14,147 @@ import (
|
||||||
|
|
||||||
// Possible flags -o (output) -versbose (print all debug) -search (the actual song you want to search)
|
// Possible flags -o (output) -versbose (print all debug) -search (the actual song you want to search)
|
||||||
|
|
||||||
var geniusApiToken = os.Getenv("GENIUS_API_TOKEN")
|
var (
|
||||||
|
printDebug bool
|
||||||
|
useGenius bool
|
||||||
|
useGoogle bool
|
||||||
|
outputFile string
|
||||||
|
errorLog *log.Logger
|
||||||
|
mainLog *log.Logger
|
||||||
|
)
|
||||||
|
|
||||||
|
// flagsSet returns a set of all the flags what were actually set on the
|
||||||
|
// command line.
|
||||||
|
func flagsSet(flags *flag.FlagSet) map[string]bool {
|
||||||
|
s := make(map[string]bool)
|
||||||
|
flags.Visit(func(f *flag.Flag) {
|
||||||
|
s[f.Name] = true
|
||||||
|
})
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
func start() int {
|
||||||
|
programName := os.Args[0]
|
||||||
|
errorLog = log.New(os.Stderr, "", 0)
|
||||||
|
mainLog = log.New(os.Stdout, "", 0)
|
||||||
|
|
||||||
|
flags := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
|
||||||
|
flags.Usage = func() {
|
||||||
|
out := flags.Output()
|
||||||
|
fmt.Fprintf(out, "Usage: %v \n\n", programName)
|
||||||
|
fmt.Fprint(out, " This program is used to download lyrics for a song\n")
|
||||||
|
fmt.Fprint(out, " from the internet. There are two modes of operation.\n\n")
|
||||||
|
fmt.Fprint(out, " The first mode is scraping the lyrics from Google. So the operation is this: \n")
|
||||||
|
fmt.Fprint(out, " It first opens a chrome window, searches for the lyrics \n")
|
||||||
|
fmt.Fprint(out, " , and copies the lyrics returned by Google Search to a \n")
|
||||||
|
fmt.Fprint(out, " file defined by you.\n\n")
|
||||||
|
fmt.Fprint(out, " The second mode is getting the lyrics from Genius API. So the operation is this: \n")
|
||||||
|
fmt.Fprint(out, " It then tries to get search for the same song using the \n")
|
||||||
|
fmt.Fprint(out, " Genius API. It then tries to compare the lyrics with the \n")
|
||||||
|
fmt.Fprint(out, " Genius one.\n\n")
|
||||||
|
flags.PrintDefaults()
|
||||||
|
}
|
||||||
|
|
||||||
|
outputFlag := flags.String("output", "", "Optional. Lyrics filename")
|
||||||
|
verboseFlag := flags.Bool("verbose", false, "Optional. Turn on debug. Default is false.")
|
||||||
|
searchFlag := flags.String("search", "", "Required. Name of song to search. If the name of the song is not a single word, put in quotes\"\"")
|
||||||
|
helpFlag := flags.Bool("help", false, "Optional. Print Usage")
|
||||||
|
useGoogleFlag := flags.Bool("google", false, "Optional. Use google.")
|
||||||
|
useGeniusFlag := flags.Bool("genius", false, "Optional. Use genius")
|
||||||
|
configFileFlag := flags.String("configFile", "", "Optional. Use with genius")
|
||||||
|
|
||||||
|
err := flags.Parse(os.Args[1:])
|
||||||
|
if err != nil {
|
||||||
|
errorLog.Printf("Err: %+v", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(flags.Args()) > 1 {
|
||||||
|
errorLog.Println("Error: too many command-line arguments")
|
||||||
|
flags.Usage()
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
allSetFlags := flagsSet(flags)
|
||||||
|
|
||||||
|
if allSetFlags["help"] && (allSetFlags["output"] || allSetFlags["search"] || allSetFlags["verbose"]) {
|
||||||
|
errorLog.Println("Error: if -help is set, -output, -search and -verbose must remain unset")
|
||||||
|
flags.Usage()
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
if !allSetFlags["google"] && !allSetFlags["genius"] {
|
||||||
|
errorLog.Println("Error: One of -google or -genius must be set")
|
||||||
|
flags.Usage()
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
if allSetFlags["google"] && allSetFlags["genius"] {
|
||||||
|
errorLog.Println("Error: if -google is set, -genius must remain unset and vice versa")
|
||||||
|
flags.Usage()
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
if allSetFlags["google"] && allSetFlags["configFile"] {
|
||||||
|
errorLog.Println("Error: if -google is set, -configFile must remain unset and vice versa")
|
||||||
|
flags.Usage()
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
if *helpFlag {
|
||||||
|
flags.Usage()
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
songToSearch := *searchFlag
|
||||||
|
|
||||||
|
if len(songToSearch) == 0 {
|
||||||
|
errorLog.Println("Error: the song name must be provided for search")
|
||||||
|
flags.Usage()
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
if allSetFlags["output"] {
|
||||||
|
outputFile = *outputFlag
|
||||||
|
} else {
|
||||||
|
mainLog.Printf("Using %s as the name of the file(s) for downloaded lyrics..\n", songToSearch)
|
||||||
|
outputFile = fmt.Sprintf("%s_lyrics", songToSearch)
|
||||||
|
}
|
||||||
|
|
||||||
|
printDebug = *verboseFlag
|
||||||
|
if printDebug {
|
||||||
|
mainLog.Printf("Output flag: %s, Debug flag: %t, Search flag: %s\n", outputFile, printDebug, songToSearch)
|
||||||
|
}
|
||||||
|
|
||||||
|
useGenius = *useGeniusFlag
|
||||||
|
useGoogle = *useGoogleFlag
|
||||||
|
|
||||||
|
if useGoogle {
|
||||||
|
err := searchGoogle(songToSearch)
|
||||||
|
if err != nil {
|
||||||
|
errorLog.Printf("Err: %+v", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if useGenius {
|
||||||
|
configFile := *configFileFlag
|
||||||
|
config, err := getConfig(configFile)
|
||||||
|
if err != nil {
|
||||||
|
errorLog.Printf("Err: %+v", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
GENIUS_API_TOKEN = config.GeniusApiToken
|
||||||
|
err = searchGenius(songToSearch)
|
||||||
|
if err != nil {
|
||||||
|
errorLog.Printf("Err: %+v", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
|
os.Exit(start())
|
||||||
fmt.Println(geniusApiToken)
|
|
||||||
|
|
||||||
os.Exit(cmdline.Main())
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,75 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
// GeniusSearchResponseHit - the path of the response from the Genuis Search Api
|
||||||
|
// that contains all the relevant information
|
||||||
|
type GeniusSearchResponseHit struct {
|
||||||
|
AnnotationCount int `json:"annotation_count"`
|
||||||
|
APIPath string `json:"api_path"`
|
||||||
|
ArtistNames string `json:"artist_names"`
|
||||||
|
FullTitle string `json:"full_title"`
|
||||||
|
HeaderImageThumbnailURL string `json:"header_image_thumbnail_url"`
|
||||||
|
HeaderImageURL string `json:"header_image_url"`
|
||||||
|
ID int `json:"id"`
|
||||||
|
LyricsOwnerID int `json:"lyrics_owner_id"`
|
||||||
|
LyricsState string `json:"lyrics_state"`
|
||||||
|
Path string `json:"path"`
|
||||||
|
PyongsCount any `json:"pyongs_count"`
|
||||||
|
RelationshipsIndexURL string `json:"relationships_index_url"`
|
||||||
|
ReleaseDateComponents struct {
|
||||||
|
Year int `json:"year"`
|
||||||
|
Month int `json:"month"`
|
||||||
|
Day int `json:"day"`
|
||||||
|
} `json:"release_date_components"`
|
||||||
|
ReleaseDateForDisplay string `json:"release_date_for_display"`
|
||||||
|
ReleaseDateWithAbbreviatedMonthForDisplay string `json:"release_date_with_abbreviated_month_for_display"`
|
||||||
|
SongArtImageThumbnailURL string `json:"song_art_image_thumbnail_url"`
|
||||||
|
SongArtImageURL string `json:"song_art_image_url"`
|
||||||
|
Stats struct {
|
||||||
|
UnreviewedAnnotations int `json:"unreviewed_annotations"`
|
||||||
|
Hot bool `json:"hot"`
|
||||||
|
Pageviews int `json:"pageviews"`
|
||||||
|
} `json:"stats"`
|
||||||
|
Title string `json:"title"`
|
||||||
|
TitleWithFeatured string `json:"title_with_featured"`
|
||||||
|
URL string `json:"url"`
|
||||||
|
FeaturedArtists []struct {
|
||||||
|
APIPath string `json:"api_path"`
|
||||||
|
HeaderImageURL string `json:"header_image_url"`
|
||||||
|
ID int `json:"id"`
|
||||||
|
ImageURL string `json:"image_url"`
|
||||||
|
IsMemeVerified bool `json:"is_meme_verified"`
|
||||||
|
IsVerified bool `json:"is_verified"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
URL string `json:"url"`
|
||||||
|
} `json:"featured_artists"`
|
||||||
|
PrimaryArtist struct {
|
||||||
|
APIPath string `json:"api_path"`
|
||||||
|
HeaderImageURL string `json:"header_image_url"`
|
||||||
|
ID int `json:"id"`
|
||||||
|
ImageURL string `json:"image_url"`
|
||||||
|
IsMemeVerified bool `json:"is_meme_verified"`
|
||||||
|
IsVerified bool `json:"is_verified"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
URL string `json:"url"`
|
||||||
|
} `json:"primary_artist"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Response for Genuis Search Api - Generated using https://mholt.github.io/json-to-go/
|
||||||
|
type GeniusSearchOtherResponse struct {
|
||||||
|
Meta struct {
|
||||||
|
Status int `json:"status"`
|
||||||
|
} `json:"meta"`
|
||||||
|
Response struct {
|
||||||
|
Hits []struct {
|
||||||
|
Highlights []any `json:"highlights"`
|
||||||
|
Index string `json:"index"`
|
||||||
|
Type string `json:"type"`
|
||||||
|
Result GeniusSearchResponseHit `json:"result"`
|
||||||
|
} `json:"hits"`
|
||||||
|
} `json:"response"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Config is the representation for the Genius API token
|
||||||
|
type Config struct {
|
||||||
|
GeniusApiToken string `json:"genius_api_token"`
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,156 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httputil"
|
||||||
|
"os"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/antchfx/htmlquery"
|
||||||
|
"github.com/gocolly/colly/v2"
|
||||||
|
"golang.org/x/net/html"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
client *http.Client = &http.Client{Timeout: 10 * time.Second}
|
||||||
|
genuisAPI = "https://api.genius.com/search"
|
||||||
|
GENIUS_API_TOKEN = ""
|
||||||
|
)
|
||||||
|
|
||||||
|
// makeRequest - generic function for making Http request
|
||||||
|
func makeRequest(endpoint string, apiToken string) (GeniusSearchOtherResponse, http.Header, error) {
|
||||||
|
var geniusApiResponse GeniusSearchOtherResponse
|
||||||
|
req, _ := http.NewRequest(http.MethodGet, endpoint, nil)
|
||||||
|
bearerHeader := fmt.Sprintf("Bearer %s", apiToken)
|
||||||
|
req.Header.Add("Authorization", bearerHeader)
|
||||||
|
|
||||||
|
reqDump, err := httputil.DumpRequestOut(req, true)
|
||||||
|
if err != nil {
|
||||||
|
return geniusApiResponse, nil, fmt.Errorf("error printing request: %+v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if printDebug {
|
||||||
|
mainLog.Printf("REQUEST:\n%s\n", string(reqDump))
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return geniusApiResponse, nil, fmt.Errorf("error receiving response: %+v", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
respDump, err := httputil.DumpResponse(resp, true)
|
||||||
|
if err != nil {
|
||||||
|
return geniusApiResponse, nil, fmt.Errorf("error printing response: %+v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if printDebug {
|
||||||
|
mainLog.Printf("RESPONSE:\n%s\n", string(respDump))
|
||||||
|
}
|
||||||
|
|
||||||
|
err = json.NewDecoder(resp.Body).Decode(&geniusApiResponse)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return geniusApiResponse, nil, fmt.Errorf("error decoding response: %+v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if printDebug {
|
||||||
|
mainLog.Printf("Decoded => %+v\n", geniusApiResponse)
|
||||||
|
}
|
||||||
|
|
||||||
|
return geniusApiResponse, resp.Header, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// getLyricUrl - get the url of the html page of the song to scrape
|
||||||
|
func getLyricUrl(searchTerm string) (string, error) {
|
||||||
|
ret, _, err := makeRequest(fmt.Sprintf("%s?q=%s", genuisAPI, searchTerm), GENIUS_API_TOKEN)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(ret.Response.Hits) == 0 {
|
||||||
|
return "", fmt.Errorf("there is no song on the Genius Api for the song: %s", searchTerm)
|
||||||
|
}
|
||||||
|
|
||||||
|
var songs []GeniusSearchResponseHit
|
||||||
|
for _, item := range ret.Response.Hits {
|
||||||
|
if item.Type == "song" {
|
||||||
|
songs = append(songs, item.Result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract the first one as that is most of the time the correct one
|
||||||
|
return songs[0].URL, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// stripUnneccessaryChars - unescape all escaped characters
|
||||||
|
func stripUnneccessaryChars(nodeHtml string) string {
|
||||||
|
captureBr := regexp.MustCompile("<br/>")
|
||||||
|
// Replace all br with a new line
|
||||||
|
convertedBr := captureBr.ReplaceAllString(nodeHtml, "\n")
|
||||||
|
|
||||||
|
captureAngleBrackets := regexp.MustCompile(`(?i)\<(.*?)\>`)
|
||||||
|
|
||||||
|
// Remove all content between the angle brackets and the angle brackets themselves
|
||||||
|
removedAngleBrackets := captureAngleBrackets.ReplaceAllString(convertedBr, "")
|
||||||
|
|
||||||
|
// Unescape all escaped string
|
||||||
|
return html.UnescapeString(removedAngleBrackets)
|
||||||
|
}
|
||||||
|
|
||||||
|
// searchGenius - print to file scraped song from Genius Search API
|
||||||
|
func searchGenius(searchTerm string) error {
|
||||||
|
var errs error
|
||||||
|
lyricUrl, err := getLyricUrl(searchTerm)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Join(err, errs)
|
||||||
|
}
|
||||||
|
|
||||||
|
if printDebug {
|
||||||
|
mainLog.Printf("lyricUrl: %s", lyricUrl)
|
||||||
|
}
|
||||||
|
|
||||||
|
var sb strings.Builder
|
||||||
|
|
||||||
|
c := colly.NewCollector()
|
||||||
|
c.OnHTML("body", func(e *colly.HTMLElement) {
|
||||||
|
doc, errT := htmlquery.Parse(bytes.NewReader(e.Response.Body))
|
||||||
|
if errT != nil {
|
||||||
|
errs = fmt.Errorf("error parsing html : %+v", errT)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// fmt.Println("doc: ", htmlquery.OutputHTML(doc, true))
|
||||||
|
|
||||||
|
// Use XPath to find nodes
|
||||||
|
nodes, errT := htmlquery.QueryAll(doc, "//div[contains(@class, 'Lyrics__Container')]") // Example XPath query
|
||||||
|
if err != nil {
|
||||||
|
errs = fmt.Errorf("error in query: %+v", errT)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, node := range nodes {
|
||||||
|
sb.WriteString(stripUnneccessaryChars(htmlquery.OutputHTML(node, true)))
|
||||||
|
}
|
||||||
|
|
||||||
|
})
|
||||||
|
err = c.Visit(lyricUrl)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Join(err, errs)
|
||||||
|
}
|
||||||
|
|
||||||
|
filename := fmt.Sprintf("%s_genius.txt", searchTerm)
|
||||||
|
err = os.WriteFile(filename, []byte(sb.String()), os.ModePerm)
|
||||||
|
if err != nil {
|
||||||
|
g := fmt.Errorf("could not write to %s: %v", filename, err)
|
||||||
|
return errors.Join(errs, g)
|
||||||
|
}
|
||||||
|
|
||||||
|
return errs
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,119 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/antchfx/htmlquery"
|
||||||
|
"github.com/playwright-community/playwright-go"
|
||||||
|
"golang.org/x/net/html"
|
||||||
|
)
|
||||||
|
|
||||||
|
// recurseNodes - walk HtmlNodes of a Html document
|
||||||
|
func recurseNodes(top *html.Node, sb *strings.Builder) {
|
||||||
|
if top.Type == html.ElementNode && top.Data == "span" {
|
||||||
|
sb.WriteString(htmlquery.InnerText(top) + "\n")
|
||||||
|
}
|
||||||
|
for c := top.FirstChild; c != nil; c = c.NextSibling {
|
||||||
|
recurseNodes(c, sb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// searchGenius - print to file scraped song from google search
|
||||||
|
func searchGoogle(song string) (err error) {
|
||||||
|
runOption := &playwright.RunOptions{
|
||||||
|
SkipInstallBrowsers: true,
|
||||||
|
}
|
||||||
|
tempErr := playwright.Install(runOption)
|
||||||
|
if tempErr != nil {
|
||||||
|
err = fmt.Errorf("could not install playwright dependencies: %v", tempErr)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
pw, tempErr := playwright.Run()
|
||||||
|
if tempErr != nil {
|
||||||
|
err = fmt.Errorf("could not start playwright: %v", tempErr)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer func(pw *playwright.Playwright) {
|
||||||
|
tempErr := pw.Stop()
|
||||||
|
if tempErr != nil {
|
||||||
|
e := fmt.Errorf("could not stop Playwright: %v", tempErr)
|
||||||
|
err = errors.Join(err, e)
|
||||||
|
}
|
||||||
|
}(pw)
|
||||||
|
|
||||||
|
option := playwright.BrowserTypeLaunchOptions{
|
||||||
|
Channel: playwright.String("chrome"),
|
||||||
|
Headless: playwright.Bool(false),
|
||||||
|
}
|
||||||
|
browser, tempErr := pw.Chromium.Launch(option)
|
||||||
|
if tempErr != nil {
|
||||||
|
err = fmt.Errorf("could not launch browser: %v", tempErr)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func(browser playwright.Browser) {
|
||||||
|
tempErr = browser.Close()
|
||||||
|
if tempErr != nil {
|
||||||
|
e := fmt.Errorf("could not close browser: %v", tempErr)
|
||||||
|
err = errors.Join(err, e)
|
||||||
|
}
|
||||||
|
}(browser)
|
||||||
|
page, tempErr := browser.NewPage()
|
||||||
|
if tempErr != nil {
|
||||||
|
err = fmt.Errorf("could not create page: %v", tempErr)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, tempErr := page.Goto(fmt.Sprintf("https://www.google.com/search?q=%ss+lyrics", song),
|
||||||
|
playwright.PageGotoOptions{
|
||||||
|
WaitUntil: playwright.WaitUntilStateLoad,
|
||||||
|
}); tempErr != nil {
|
||||||
|
err = fmt.Errorf("could not goto: %v", tempErr)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
tempErr = page.Locator("body").WaitFor(playwright.LocatorWaitForOptions{
|
||||||
|
State: playwright.WaitForSelectorStateVisible,
|
||||||
|
})
|
||||||
|
if tempErr != nil {
|
||||||
|
err = fmt.Errorf("could not wait for body: %v", tempErr)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
html, tempErr := page.Locator("html").InnerHTML()
|
||||||
|
if tempErr != nil {
|
||||||
|
err = fmt.Errorf("could not get innerHtml: %v", tempErr)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
doc, tempErr := htmlquery.Parse(bytes.NewReader([]byte(html)))
|
||||||
|
if tempErr != nil {
|
||||||
|
err = fmt.Errorf("could not parse the innerHtml: %v", tempErr)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
nodes, tempErr := htmlquery.QueryAll(doc, "//div[@data-lyricid]/div")
|
||||||
|
if tempErr != nil {
|
||||||
|
err = fmt.Errorf("could not get the nodes: %v", tempErr)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
var sb strings.Builder
|
||||||
|
for _, node := range nodes {
|
||||||
|
recurseNodes(node, &sb)
|
||||||
|
}
|
||||||
|
if sb.Len() > 0 {
|
||||||
|
if printDebug {
|
||||||
|
mainLog.Println("Writing lyrics from Google...")
|
||||||
|
}
|
||||||
|
filename := fmt.Sprintf("%s_google.txt", outputFile)
|
||||||
|
tempErr = os.WriteFile(filename, []byte(sb.String()), os.ModePerm)
|
||||||
|
if tempErr != nil {
|
||||||
|
err = fmt.Errorf("could not write to %s: %v", filename, err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
mainLog.Println("Lyrics cannot be found...")
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue