utils/cmd/slackware-sync/main.go

package main

import (
	"flag"
	"fmt"
	"log"
	"net/url"
	"os"
	"os/exec"
	"path"
	"sync"

	"github.com/BurntSushi/toml"
)

var (
	flSyncDir    = flag.String("dir", "", "directory to sync to (this flag overrides the syncDir in the configuration file)")
	flConfigFile = flag.String("c", path.Join(os.Getenv("HOME"), ".slackware-sync.toml"), "config file for the sync")
	flThreads    = flag.Int("t", 1, "threads to fetch with")
	flQuiet      = flag.Bool("q", false, "less output")
)

func main() {
	flag.Parse()
	var config GeneralConfig
	if _, err := toml.DecodeFile(*flConfigFile, &config); err != nil {
		log.Fatal(err)
	}

	if len(*flSyncDir) > 0 {
		config.SyncDir = *flSyncDir
	}
	if *flThreads > 1 {
		config.Threads = *flThreads
	}

	if _, err := EnsureDirExists(config.SyncDir); err != nil {
		log.Fatal(err)
	}

	workers := make(chan int, config.Threads)
	wg := sync.WaitGroup{}
	for name, mirror := range config.Mirrors {
		if !mirror.Enabled {
			continue
		}
		uri, err := url.Parse(mirror.URL)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			continue
		}
		dest := path.Join(config.SyncDir, uri.Host, uri.Path)
		if _, err = EnsureDirExists(dest); err != nil {
			fmt.Fprintln(os.Stderr, err)
			continue
		}

		wg.Add(1)
		rsyncFunc := func() {
			if config.Threads > 1 {
				workers <- 1
			}
			defer func() {
				if config.Threads > 1 {
					<-workers
				}
				wg.Done()
			}()
			args := []string{
				"-avPHS",
				"--delete",
			}
			if mirror.LinkDest != "" {
				if linkDestMirror, ok := config.Mirrors[mirror.LinkDest]; ok {
					if linkDestURL, err := url.Parse(linkDestMirror.URL); err == nil {
						linkDest := path.Join(config.SyncDir, linkDestURL.Host, linkDestURL.Path)
						args = append(args, "--link-dest="+linkDest+"/")
					} else {
						fmt.Fprintf(os.Stderr, "linkDest URL %q does not parse: %s", mirror.LinkDest, err)
					}
				}
			}
			args = append(args, uri.String())
			args = append(args, dest+"/")
			//fmt.Println("rsync", strings.Join(args, " "))
			cmd := exec.Command("rsync", args...)
			cmd.Stderr = os.Stderr // we'll want to see errors, regardless
			if !*flQuiet {
				cmd.Stdout = os.Stdout
			}

			if err = cmd.Run(); err != nil {
				fmt.Fprintf(os.Stderr, "%q: %s", name, err)
			}
		}
		if config.Threads > 1 {
			go rsyncFunc()
		} else {
			rsyncFunc()
		}
	}
	wg.Wait()
}

func EnsureDirExists(path string) (os.FileInfo, error) {
	stat, err := os.Stat(path)
	if err != nil {
		if !os.IsNotExist(err) {
			return stat, err
		}
		if err = os.MkdirAll(path, 0755); err != nil {
			return stat, err
		}
		if stat, err = os.Stat(path); err != nil {
			return stat, err
		}
	}
	return stat, nil
}

type GeneralConfig struct {
	Threads int    `toml:"threads"`
	SyncDir string `toml:"syncDir"`
	Mirrors map[string]Mirror
}

type Mirror struct {
	Title    string `toml:"title"`
	URL      string `toml:"url"`
	Enabled  bool   `toml:"enabled"`
	LinkDest string `toml:"linkDest"`
}