From 234b593c26adcfaff3109734784a9a2c2902481b Mon Sep 17 00:00:00 2001 From: OLUWADAMILOLA OKUSANYA Date: Sat, 17 Jun 2023 00:08:50 -0400 Subject: [PATCH] Publish, View and Tar commands added Adding middlewares. Adding a filesystem service. Adding types to represent the put request and the index.json that is stored at the root of the package. Adding conf.go to pull config variables from json --- .gitignore | 1 + app.go | 18 ++-- config/conf.go | 36 ++++++++ handler/get.go | 9 +- handler/notfound.go | 11 +++ handler/publish.go | 116 +++++++++++++++++++++++++- handler/tar.go | 34 ++++++++ main.go | 36 +++++++- middlewares/auth.go | 55 ++++++++++++ middlewares/logRequest.go | 33 ++++++++ middlewares/middlewaretype.go | 7 ++ serviceidos/responseidos.go | 42 ++++++++++ storage/fs.go | 153 ++++++++++++++++++++++++++++++++-- 13 files changed, 533 insertions(+), 18 deletions(-) create mode 100644 .gitignore create mode 100644 config/conf.go create mode 100644 handler/notfound.go create mode 100644 middlewares/auth.go create mode 100644 middlewares/logRequest.go create mode 100644 middlewares/middlewaretype.go create mode 100644 serviceidos/responseidos.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..eb6256b --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +**/userdata/* diff --git a/app.go b/app.go index 8ddca7b..1ebac57 100644 --- a/app.go +++ b/app.go @@ -1,28 +1,36 @@ package main import ( + "gosimplenpm/config" + "gosimplenpm/handler" + "gosimplenpm/middlewares" "log" + "net/http" "github.com/gorilla/mux" - - "gosimplenpm/handler" ) type application struct { logger *log.Logger + conf config.Config } func (app *application) Routes() *mux.Router { - m := mux.NewRouter() + + // Need to use UseEncodedPath as shown here https://github.com/gorilla/mux/blob/master/mux.go#L269 + m := mux.NewRouter().StrictSlash(true).UseEncodedPath() + + m.Use(middlewares.LogMiddleware) // main handler - m.HandleFunc("/{name}", handler.Get).Methods("GET") - m.HandleFunc("/{name}", handler.Publish).Methods("PUT") + m.HandleFunc("/{name}", middlewares.AuthMiddleware(app.conf)(handler.Get)).Methods("GET") + m.HandleFunc("/{name}", middlewares.AuthMiddleware(app.conf)(handler.Publish)).Methods("PUT") // tar handlers m.HandleFunc("/{name}/-/{tar}", handler.Tar).Methods("GET") // tag handlers m.HandleFunc("/-/package/{name}/dist-tags/{tag}", handler.DistTagDelete).Methods("DELETE") m.HandleFunc("/-/package/{name}/dist-tags/{tag}", handler.DistTagPut).Methods("PUT") m.HandleFunc("/-/package/{name}/dist-tags", handler.DistTagGet).Methods("GET") + m.NotFoundHandler = http.HandlerFunc(handler.NotFound) return m } diff --git a/config/conf.go b/config/conf.go new file mode 100644 index 0000000..c51e526 --- /dev/null +++ b/config/conf.go @@ -0,0 +1,36 @@ +package config + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" +) + +type Config struct { + Token string `json:"token"` +} + +func LoadConfiguration(file string, config *Config) error { + filePath, err := filepath.Abs(file) + if err != nil { + fmt.Printf("File repo not found: +%v\n", err) + return err + } + configFile, err := os.Open(filePath) + // From https://stackoverflow/a/76287159 + defer func() { + err = errors.Join(err, configFile.Close()) + if err != nil { + fmt.Printf("File cannot be closed: +%v\n", err) + } + }() + if err != nil { + fmt.Printf("File cannot be opened: +%v\n", err) + return err + } + json.NewDecoder(configFile).Decode(config) + fmt.Println("Json loaded") + return nil +} diff --git a/handler/get.go b/handler/get.go index 0fe3ec7..7e35660 100644 --- a/handler/get.go +++ b/handler/get.go @@ -3,6 +3,7 @@ package handler import ( "fmt" "net/http" + "net/url" "github.com/gorilla/mux" @@ -10,15 +11,17 @@ import ( ) func Get(w http.ResponseWriter, r *http.Request) { - packageName := mux.Vars(r)["name"] + escapedName := mux.Vars(r)["name"] + packageName, _ := url.PathUnescape(escapedName) + fmt.Printf("Package name => %s\n", packageName) - fileToServe, err := storage.GetPackageFromStore(packageName) + fileToServe, found, err := storage.GetIndexJsonFromStore(packageName) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - if fileToServe == "" { + if !found { ret := fmt.Sprintf("Package not found: %s", packageName) http.Error(w, ret, http.StatusNotFound) return diff --git a/handler/notfound.go b/handler/notfound.go new file mode 100644 index 0000000..e5f6da8 --- /dev/null +++ b/handler/notfound.go @@ -0,0 +1,11 @@ +package handler + +import ( + "fmt" + "net/http" +) + +// This handler is executed when the router cannot match any route +func NotFound(w http.ResponseWriter, r *http.Request) { + fmt.Printf("%s - %s - %s\n", r.Method, r.URL, r.Host) +} diff --git a/handler/publish.go b/handler/publish.go index f99312c..54c6c0b 100644 --- a/handler/publish.go +++ b/handler/publish.go @@ -1,9 +1,121 @@ package handler import ( + "encoding/json" + "fmt" + "gosimplenpm/serviceidos" + "gosimplenpm/storage" "net/http" + "net/url" + "path" + "strings" + + "github.com/gorilla/mux" ) -func Publish(w http.ResponseWriter, r *http.Request) { - +type NPMClientPutRequest struct { + Request serviceidos.IndexJson +} + +func Publish(w http.ResponseWriter, r *http.Request) { + // (1) Parse Json Body + // (2) Check if package exists in the folder. + // (a) if it does, ckeck if it is the same version. If it is, return error. Else modify index.json from (2) + // (b) If it does not, add the latest tag to the new index.json + + escapedName := mux.Vars(r)["name"] + packageName, _ := url.PathUnescape(escapedName) + fmt.Printf("Package name => %s\n", packageName) + + var cr NPMClientPutRequest + // Parse json body + err := json.NewDecoder(r.Body).Decode(&cr.Request) + + if err != nil { + fmt.Printf("Error unmarshaling put request: %+v\n", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Extract relevant data from index.json + index := 0 + var tag string + var version string + var versionData serviceidos.IndexJsonVersions + // TODO: Fix this as the order is not guaranteed + for key, value := range cr.Request.DistTags { + if index == 0 { + tag = key + version = value + break + } + index++ + } + versionData = cr.Request.Versions[version] + fmt.Printf("For version(%s) with tag(%s), versionData => %+v\n", version, tag, versionData) + + // Rewrite the tarball path + tarballFileName := strings.Split(versionData.Dist.Tarball, "/-/")[1] + fmt.Printf("TarballName => %s\n", tarballFileName) + // versionData.Dist.Tarball = fmt.Sprintf("file://%s", packageFilePath) + versionData.Dist.Tarball = fmt.Sprintf("http://%s/%s/-/%s", r.Host, url.PathEscape(packageName), url.PathEscape(tarballFileName)) + fmt.Printf("versionData.Dist.Tarball => %s\n", versionData.Dist.Tarball) + registryPath, _ := storage.GetRegistryPath() + tarBallFile := strings.Split(tarballFileName, "/")[1] + packageFilePath := path.Join(registryPath, packageName, tarBallFile) + + // Try to get the index.json from the store + fileToServe, found, err := storage.GetIndexJsonFromStore(packageName) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + var jsonFile serviceidos.IndexJson + if !found { + // new package + jsonFile = cr.Request + jsonFile.DistTags["latest"] = version + } else { + // old package + err = storage.ReadIndexJson(fileToServe, &jsonFile) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Checking that you are not publishing over a pervious published version + if jsonFile.Versions[version].Version == version { + fmt.Printf("Version %s of package %s already exists!!\n", version, packageName) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Rewrite attachments + jsonFile.DistTags[tag] = version + nAttachments := make(map[string]serviceidos.IndexJsonAttachments) + nAttachments[fmt.Sprintf("%s-%s.tgz", packageName, version)] = cr.Request.Attachments[fmt.Sprintf("%s-%s.tgz", packageName, version)] + jsonFile.Attachments = nAttachments + + // Merge in the new version data + jsonFile.Versions[version] = versionData + } + + fmt.Println("FiletoServe ==> ", fileToServe) + + // Write index.json + err = storage.WriteIndexJson(fileToServe, &jsonFile) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + fmt.Println("Package path => ", packageFilePath) + // Write bundled package + packageData := jsonFile.Attachments[fmt.Sprintf("%s-%s.tgz", packageName, version)].Data + err = storage.WritePackageToStore(packageFilePath, packageData) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } } diff --git a/handler/tar.go b/handler/tar.go index 534c602..90e533f 100644 --- a/handler/tar.go +++ b/handler/tar.go @@ -1,9 +1,43 @@ package handler import ( + "bytes" + "fmt" + "gosimplenpm/storage" + "io" "net/http" + "net/url" + "strconv" + "strings" + + "github.com/gorilla/mux" ) func Tar(w http.ResponseWriter, r *http.Request) { + // Sample output of npm view + // Public + // dist + // .tarball: https://registry.npmjs.org/react/-/react-18.2.0.tgz + // LocalHost + // dist + // .tarball: http://localhost:4000/@ookusanya/package1/-/package1-0.2.0.tgz + escapedName := mux.Vars(r)["name"] + packageName, _ := url.PathUnescape(escapedName) + fmt.Printf("Package name => %s\n", packageName) + escapedName = mux.Vars(r)["tar"] + tarFileName, _ := url.PathUnescape(escapedName) + fmt.Printf("Tarfile name => %s\n", tarFileName) + + versionName := strings.Split(strings.Split(tarFileName, "-")[1], ".tgz")[0] + fileAsString, err := storage.GetTarFromStore(packageName, versionName) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Sending the tar as a base64 string + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Length", strconv.Itoa(len([]byte(fileAsString)))) + io.Copy(w, bytes.NewReader([]byte(fileAsString))) } diff --git a/main.go b/main.go index 3ac3e91..f004808 100644 --- a/main.go +++ b/main.go @@ -1,13 +1,45 @@ package main import ( + "gosimplenpm/config" "log" "net/http" ) func main() { - app := new(application) + var cfg config.Config + err := config.LoadConfiguration("userdata/config.json", &cfg) + if err != nil { + log.Fatalf("Config is not loaded: %+v\n", err) + } + + app := &application{ + conf: cfg, + } log.Print("Starting server on port 4000") - err := http.ListenAndServe(":4000", app.Routes()) + err = http.ListenAndServe(":4000", app.Routes()) log.Fatal(err) } + +// func main() { +// router := mux.NewRouter() +// router.NewRoute().Path("/{name}").Methods("PUT") +// router.NewRoute().Path("/{name}").Methods("GET") + +// rMatch := &mux.RouteMatch{} + +// u := url.URL{Path: "/@ookusanya%2fsimplepackone"} +// req := http.Request{Method: "GET", URL: &u} + +// x := router.Match(&req, rMatch) +// fmt.Println("Is Matched: ", x) + +// reqt := http.Request{Method: "PUT", URL: &u} +// g := router.Match(&reqt, rMatch) +// fmt.Println("Is Matched: ", g) + +// ut := url.URL{Path: "/@ookusanya%2fsimplepackone/-/simplepackone-1.0.0.tgz"} +// rt := http.Request{Method: "PUT", URL: &ut} +// gt := router.Match(&rt, rMatch) +// fmt.Println("Is Matched: ", gt) +// } diff --git a/middlewares/auth.go b/middlewares/auth.go new file mode 100644 index 0000000..dcd4efb --- /dev/null +++ b/middlewares/auth.go @@ -0,0 +1,55 @@ +package middlewares + +import ( + "gosimplenpm/config" + "net/http" + "strings" +) + +func AuthMiddleware(cfg config.Config) Middleware { + return func(next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + // get headers + authHeader := r.Header.Get("Authorization") + authFields := strings.Fields(authHeader) + if len(authFields) != 2 || strings.ToLower(authFields[0]) != "bearer" { + http.Error(w, "Authentication Error", http.StatusForbidden) + return + } + token := authFields[1] + if token != cfg.Token { + http.Error(w, "Authentication Error", http.StatusForbidden) + return + } + + next(w, r) + } + } +} + +// func AuthMiddleware(next http.HandlerFunc) http.HandlerFunc { +// return func(w http.ResponseWriter, r *http.Request) { +// if cfg == nil { +// log.Println("Config load error") +// http.Error(w, "Config load error", http.StatusInternalServerError) +// return +// } + +// log.Println("Config was loaded") + +// // get headers +// authHeader := r.Header.Get("Authorization") +// authFields := strings.Fields(authHeader) +// if len(authFields) != 2 || strings.ToLower(authFields[0]) != "bearer" { +// http.Error(w, "Authentication Error", http.StatusForbidden) +// return +// } +// token := authFields[1] +// if token != cfg.Token { +// http.Error(w, "Authentication Error", http.StatusForbidden) +// return +// } +// fmt.Println("Authorized") +// next(w, r) +// } +// } diff --git a/middlewares/logRequest.go b/middlewares/logRequest.go new file mode 100644 index 0000000..52e1cb5 --- /dev/null +++ b/middlewares/logRequest.go @@ -0,0 +1,33 @@ +package middlewares + +import ( + "fmt" + "log" + "net/http" + "net/http/httputil" +) + +func LogMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + log.Printf("%s - %s - %s", r.Method, r.URL, r.Host) + + hasBody := false + if r.Method == "PUT" { + hasBody = true + } + + requestDump, err := httputil.DumpRequest(r, hasBody) + if err != nil { + fmt.Println(err) + } + fmt.Println("RequestDump: ", string(requestDump)) + + // fmt.Println("Printing headers") + // for name, values := range r.Header { + // for _, value := range values { + // fmt.Printf("%s:%s\n", name, value) + // } + // } + next.ServeHTTP(w, r) + }) +} diff --git a/middlewares/middlewaretype.go b/middlewares/middlewaretype.go new file mode 100644 index 0000000..a160084 --- /dev/null +++ b/middlewares/middlewaretype.go @@ -0,0 +1,7 @@ +package middlewares + +import ( + "net/http" +) + +type Middleware func(http.HandlerFunc) http.HandlerFunc diff --git a/serviceidos/responseidos.go b/serviceidos/responseidos.go new file mode 100644 index 0000000..08b82e7 --- /dev/null +++ b/serviceidos/responseidos.go @@ -0,0 +1,42 @@ +package serviceidos + +type IndexJsonAttachments struct { + ContentType string `json:"content_type"` + Data string `json:"data"` + Length int `json:"length"` +} + +type IndexJsonDist struct { + Integrity string `json:"integrity"` + Shasum string `json:"shasum"` + Tarball string `json:"tarball"` +} + +type IndexJsonVersions struct { + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + Main string `json:"main,omitempty"` + Scripts map[string]string `json:"scripts,omitempty"` + Author string `json:"author,omitempty"` + License string `json:"license"` + Files []string `json:"files"` + Readme string `json:"readme,omitempty"` + ID string `json:"_id"` + NodeVersion string `json:"_nodeVersion"` + NpmVersion string `json:"_npmVersion"` + Dist IndexJsonDist `json:"dist"` + Dependencies map[string]string `json:"dependencies,omitempty"` + DevDependencies map[string]string `json:"devDependencies,omitempty"` + Resolutions map[string]string `json:"resolutions,omitempty"` +} + +type IndexJson struct { + ID string `json:"_id"` + Name string `json:"name"` + Description string `json:"description"` + DistTags map[string]string `json:"dist-tags"` + Versions map[string]IndexJsonVersions `json:"versions"` + Access string `json:"access"` + Attachments map[string]IndexJsonAttachments `json:"_attachments"` +} diff --git a/storage/fs.go b/storage/fs.go index 7b5d53f..8641535 100644 --- a/storage/fs.go +++ b/storage/fs.go @@ -1,33 +1,174 @@ package storage import ( + "archive/tar" + "compress/gzip" + "encoding/base64" + "encoding/json" "fmt" + "io" "io/fs" + "os" "path" "strings" + "gosimplenpm/serviceidos" "path/filepath" ) -func GetPackageFromStore(packageName string) (string, error) { - fileToServe := "" - searchDir, err := filepath.Abs("./examples") +func GetRegistryPath() (string, error) { + registryPath, err := filepath.Abs("./examples") if err != nil { - fmt.Printf("File repo not found: +%v/n", err) + fmt.Printf("File repo not found: +%v\n", err) return "", err } + return registryPath, err +} + +func GetIndexJsonFromStore(packageName string) (string, bool, error) { + fileToServe := "" + found := false + searchDir, err := GetRegistryPath() + if err != nil { + return searchDir, found, err + } err = filepath.WalkDir(searchDir, func(fp string, info fs.DirEntry, e error) error { if strings.Contains(fp, path.Join(packageName, "index.json")) { fileToServe = fp + found = true + } + return e + }) + + if err != nil { + fmt.Printf("List files error: +%v\n", err) + return fileToServe, found, err + } + + if fileToServe == "" && !found { + fileToServe = path.Join(searchDir, packageName, "index.json") + } + + return fileToServe, found, nil +} + +func GetTarFromStore(packageName string, tarFileName string) (string, error) { + fileToServe := "" + searchDir, err := GetRegistryPath() + if err != nil { + return searchDir, err + } + + err = filepath.WalkDir(searchDir, func(fp string, info fs.DirEntry, e error) error { + if strings.Contains(fp, path.Join(packageName, tarFileName)) { + fileToServe = fp } return e }) if err != nil { - fmt.Printf("List files error: +%v/n", err) + fmt.Printf("List files error: +%v\n", err) + return fileToServe, err + } + + if fileToServe == "" { + return fileToServe, fmt.Errorf("file %s is not found for package %s", tarFileName, packageName) + } + + file, err := os.Open(fileToServe) + if err != nil { + fmt.Printf("Open error: %s\n", fileToServe) return "", err } - return fileToServe, nil + archive, err := gzip.NewReader(file) + if err != nil { + fmt.Printf("Archive Open error: %s\n", fileToServe) + return "", err + } + + tr := tar.NewReader(archive) + bs, err := io.ReadAll(tr) + if err != nil { + fmt.Printf("Archive Read error: %s\n", fileToServe) + return "", err + } + return base64.StdEncoding.EncodeToString(bs), err +} + +func ReadIndexJson(fPath string, res *serviceidos.IndexJson) error { + jsonFile, err := os.Open(fPath) + if err != nil { + fmt.Printf("File Not found: %s\n", fPath) + return err + } + + defer jsonFile.Close() + + err = json.NewDecoder(jsonFile).Decode(res) + if err != nil { + fmt.Printf("Unmarshalerror: %+v\n", err) + return err + } + + return nil +} + +func WriteIndexJson(fPath string, res *serviceidos.IndexJson) error { + // Need to create the directory first + parent := path.Dir(fPath) + err := os.MkdirAll(parent, os.ModePerm) + if err != nil { + fmt.Printf("Folder (%s) creation failed.\n", fPath) + return err + } + + // Create the file + jsonFile, err := os.Create(fPath) + + if err != nil { + fmt.Printf("Creation error for path(%s): %+v\n ", fPath, err) + return err + } + + defer jsonFile.Close() + + err = json.NewEncoder(jsonFile).Encode(res) + if err != nil { + fmt.Printf("Marshalerror: %+v\n", err) + return err + } + + return nil +} + +func WritePackageToStore(fPath string, data string) error { + dec, err := base64.StdEncoding.DecodeString(data) + if err != nil { + fmt.Printf("Base64 Decode error: %+v\n", err) + return err + } + + dataFile, err := os.Create(fPath) + if err != nil { + fmt.Printf("Creation error: %s\n", fPath) + return err + } + + defer dataFile.Close() + + _, err = dataFile.Write(dec) + if err != nil { + fmt.Printf("Write error: %s\n", fPath) + return err + } + + err = dataFile.Sync() + if err != nil { + fmt.Printf("Sync error: %s\n", fPath) + return err + } + + return nil }