/* * Copyright (c) 2016-2017 Samsung Electronics Co., Ltd All Rights Reserved * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License */ package main import ( "bufio" "bytes" "flag" "fmt" "io/ioutil" "log" "net/url" "os" "os/exec" "runtime" "strings" "sync" "time" "unicode" "git.tizen.org/tools/snapsync/config" gitMgr "git.tizen.org/tools/snapsync/git" ) const ( defaultRepoURL = "http://download.tizen.org/snapshots/tizen/" defaultSSHURL = "ssh://review.tizen.org:29418/" ) var ( // profile which will be synchronized target string // profile to which will be base for synchronization source string // URL to place where target profile resides targetURL string // URL to place where target profile resides sourceURL string // snapshot to be used as base for synchronization sourceSnapshot string // ssh address of remote git repository sshURL *url.URL // SR tag tagStr string // root folder, under which all git repos reside localMirror string // path to configuration path configFile string // path to file with list of ignored repositories (regexp) blacklistFile string // editor which will be used to edit report editor string // username for ssh git username string // number of Git Job workers numWorkers int // timeout for Git Manager to acquire lock timeout time.Duration // print version and exit printVersion bool ) // version prints snapsync and go versions and exits. func version() { v := "1.0.1" fmt.Println("snapsync version: " + v + " (built with Go: " + runtime.Version() + ").") os.Exit(0) } // generateTag takes profile name and generates SR tag (as a string). func generateTag(project string) (tag string) { timestamp := time.Now().UTC().Format("20060102.150405") tag = "submit/tizen_" + strings.Replace(project, "-", "_", -1) + "/" + timestamp return } // report prints git repositories summary (if given) and asks user what to do. Returns answer. func report(repos *map[string]*gitMgr.GitRepo) (bool, error) { var ans rune start: // print summary of git repositories if those were given. if repos != nil { for k, v := range *repos { fmt.Printf("%s: %s - %s\n", k, v.SourceCommit, v.Status) } } for { fmt.Println("What would you like to do? ") // empty map means that there's no list to edit if repos == nil { fmt.Println("([c]ontinue / [a]bort)") } else { fmt.Println("([c]ontinue / [e]dit / [a]bort)") } _, err := fmt.Scanf("%c\n", &ans) if err != nil { return false, err } ans = unicode.ToLower(ans) if repos == nil && ans == 'e' { continue } switch ans { case 'c': return true, nil case 'a': return false, nil case 'e': r, err := editReport(*repos) if err != nil { return false, err } *repos = r // after editing ask show new summary and ask again what to do goto start } } } // editReport runs external text editor, so user can remove not needed repositories. func editReport(repos map[string]*gitMgr.GitRepo) (map[string]*gitMgr.GitRepo, error) { // create temporary file for list tmpfile, err := ioutil.TempFile(os.TempDir(), "snapsync") if err != nil { return nil, err } fname := tmpfile.Name() defer os.Remove(fname) defer tmpfile.Close() writer := bufio.NewWriter(tmpfile) for k, v := range repos { // only new and updated repositories are used by snapsync if v.Status == gitMgr.New || v.Status == gitMgr.OK { _, err = writer.WriteString(fmt.Sprintf("%s: %v\n", k, v.Status)) if err != nil { return nil, err } } } err = writer.Flush() if err != nil { return nil, err } var stderr bytes.Buffer // Use external editor editorCmd := exec.Command(editor, fname) editorCmd.Stdin = os.Stdin editorCmd.Stdout = os.Stdout // We want to intercept standard error editorCmd.Stderr = &stderr err = editorCmd.Run() if err != nil { if _, ok := err.(*exec.ExitError); ok { // print editor error output if it failed log.Println(&stderr) } return nil, err } _, err = tmpfile.Seek(0, 0) if err != nil { return nil, err } // get contents of file to new map ret := make(map[string]*gitMgr.GitRepo, len(repos)) scanner := bufio.NewScanner(tmpfile) for scanner.Scan() { s := strings.Split(scanner.Text(), ":")[0] ret[s] = repos[s] } if err = scanner.Err(); err != nil { return nil, err } // return new map with git repositories return ret, nil } // exitOnFalse exits to OS if it was passed false value and nil error. It's used // to check if user decided to abort in report() and if there were no errors. func exitOnFalse(val bool, err error) error { if err == nil && !val { log.Println("Aborting.") os.Exit(0) } return err } func init() { flag.BoolVar(&printVersion, "version", false, "print snapsync version and exit") flag.StringVar(&configFile, "config", "", "configuration file") flag.StringVar(&blacklistFile, "blacklist", "", "file with repositories which should be omitted") flag.StringVar(&target, "target", "", "profile to be synchronized") flag.StringVar(&source, "source", "", "profile from which repositories will be synchronized") flag.StringVar(&sourceSnapshot, "snapshot", "latest", "source snapshot") flag.StringVar(&tagStr, "tag", "", "tag to use for group SR") flag.IntVar(&numWorkers, "workers", 0, "maximum number of pararell job workers, 0 for CPUs number") runtime.GOMAXPROCS(runtime.NumCPU()) } // getValue returns val if it's not empty or def otherwise. Returned string may be optionaly lowercased. func getValue(val, def string, lowerCase bool) string { var ret string if val != "" { ret = val } else { ret = def } if lowerCase { ret = strings.ToLower(ret) } return ret } // setup parses commandline options, configuration file and blacklist. // As a result all required variables are set (otherwise non-nil error is returned). func setup() error { var err error flag.Parse() if printVersion { version() } configFile, err = config.FindFPath(configFile, "snapsync.conf") if err != nil { return fmt.Errorf("Unable to read configuration file: %s", err) } settings, err := config.LoadSettings(configFile) if err != nil { return fmt.Errorf("Reading settings failed: %s", err) } // General if numWorkers == 0 { numWorkers = settings.G.Workers if numWorkers == 0 { numWorkers = runtime.NumCPU() } } timeoutStr := getValue(settings.G.Timeout, "5s", true) timeout, err = time.ParseDuration(timeoutStr) if err != nil { return fmt.Errorf("Parsing timeout failed: %s", err) } blacklistFile = getValue(blacklistFile, settings.G.BlacklistFile, false) // if blacklist option wasn't set then check for blacklist in standard location blacklistFile, err = config.FindFPath(blacklistFile, "blacklist") if err != nil { log.Println("Unable to read blacklist file: ", err) if exitOnFalse(report(nil)) != nil { return err } } editor = getValue(settings.G.Editor, os.Getenv("EDITOR"), false) // User username = settings.U.Username if username == "" { return fmt.Errorf("Username must be set") } // Repos target = getValue(target, settings.R.Target, true) if source == "" { source = getValue(settings.R.Source, "common", true) } sourceURL = getValue(settings.R.SourceURL, defaultRepoURL, false) targetURL = getValue(settings.R.TargetURL, defaultRepoURL, false) sshURLStr := getValue(settings.R.SSHURL, defaultSSHURL, false) sshURL, err = url.Parse(sshURLStr) if err != nil { return fmt.Errorf("Parsing SSH URL from config failed: %s", err) } localMirror, err = os.Getwd() localMirror = getValue(settings.R.LocalMirror, localMirror, false) localMirror = getValue(flag.Arg(0), localMirror, false) if localMirror == "" { //if localMirror is empty, then os.Getwd() failed, so err is set return fmt.Errorf("Couldn't get path of local mirror of git repos: %s", err) } fi, err := os.Stat(localMirror) if err != nil { return fmt.Errorf("Couldn't get file information: %s", err.Error()) } if !fi.Mode().IsDir() { return fmt.Errorf("%s is not a directory", localMirror) } if !strings.HasSuffix(localMirror, "/") { localMirror += "/" } // If target profile wasn't set and SR tag was set, then new profile is spun off. if target == "" { if tagStr == "" { return fmt.Errorf("Either -target or -tag must be set") } log.Println("Target wasn't set - probably new profile/project.") if exitOnFalse(report(nil)) != nil { return err } } tagStr = getValue(tagStr, generateTag(target), false) return nil } func main() { err := setup() if err != nil { log.Fatalln("setup failed: ", err) } blacklisted, err := config.GetBlacklist(blacklistFile) if err != nil { log.Fatalln("Reading blacklist failed: ", err) } var submitted map[string]string if target != "" { // get & parse target latest snapshot manifests submitted, err = getSnapshotInfo(targetURL+target+"/latest", blacklisted) if err != nil { log.Fatalln("Couldn't get list of target repositories: ", err) } } // get & parse source snapshot manifests repoList, err := getSnapshotInfo(sourceURL+source+"/"+sourceSnapshot, blacklisted) if err != nil { log.Fatalln("Couldn't get list of source repositories: ", err) } // prepare git repositories information map repos := make(map[string]*gitMgr.GitRepo, len(repoList)) status := gitMgr.Unknown if target == "" { status = gitMgr.New } addr := sshURL.Scheme + "://" + username + "@" + sshURL.Host + "/" message := fmt.Sprintf("snapsync: synchronize with profile: %s; snapshot: %s", source, sourceSnapshot) tag, err := gitMgr.PrepareAnnotatedTag(tagStr, message) if err != nil { log.Fatalln("Couldn't prepare annotated tag:", err) } for repoName, commit := range repoList { repos[repoName] = &gitMgr.GitRepo{ Repo: repoName, RemoteName: "snapsync", RemoteURL: addr + repoName, SourceCommit: commit, TargetCommit: submitted[repoName], Status: status, Tag: tag, } } var mtx sync.RWMutex var resch chan *gitMgr.GitRepo var errch chan error mgr := gitMgr.NewJobManager(numWorkers) // closures to dispatch jobs and gather results dispatchJobs := func(jtype gitMgr.JobType) { var keys []string for k := range repos { keys = append(keys, k) } for _, k := range keys { mtx.RLock() r := *repos[k] mtx.RUnlock() mgr.ScheduleJob(&gitMgr.GitJob{jtype, &r}) } mgr.JobsDone() } gatherResults := func(jtype gitMgr.JobType) { for inProgress := true; inProgress; { select { case err, open := <-errch: if open { mgr.CancelJobs() log.Fatalln("job failed: ", err) } case resp, open := <-resch: if !open { inProgress = false } else { mtx.Lock() repos[resp.Repo] = resp if jtype == gitMgr.Push { log.Println("pushed: ", resp.Repo) } mtx.Unlock() } } } } dispatchAndGather := func(jtype gitMgr.JobType) { resch, errch = mgr.SpawnWorkers(localMirror, timeout) if resch == nil { log.Fatalln("Job manager busy - timed out.") } go dispatchJobs(jtype) gatherResults(jtype) } if target != "" { dispatchAndGather(gitMgr.Compare) } err = exitOnFalse(report(&repos)) if err != nil { log.Fatalln("Report error: ", err) } dispatchAndGather(gitMgr.AddRemote) dispatchAndGather(gitMgr.Tag) dispatchAndGather(gitMgr.Push) log.Println("SR tag:", tagStr) }