default/src/main.go

package main

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"errors"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path"
	"strconv"
	"strings"
	"text/template"
)

type Config struct {
	XMLName     xml.Name   `xml:"config"`
	BuildEnv    []BuildEnv `xml:"buildenv"`
	BuildEnvMap map[string]BuildEnv

	Repository []Repository `xml:"repository"`
}

type BuildEnv struct {
	Name string `xml:"name,attr"`
	Host string `xml:"host"`
	User string `xml:"user"`
	OS   string `xml:"os"`
}

type Build struct {
	Name        string `xml:"name,attr"`
	IsPlatform  string `xml:"isplatform,attr"`
	NotPlatform string `xml:"not,attr"`
	NotList     []string
	Test        []string `xml:"test"`
	Var         []struct {
		Name  string `xml:"name,attr"`
		Value string `xml:",chardata"`
	}
	Configure string `xml:"configure"`
	Compile   string `xml:"compile"`
	Check     string `xml:"check"`
}

type Repository struct {
	Path      string  `xml:"path"`
	BuildEnvs string  `xml:"buildenvs"`
	Build     []Build `xml:"build"`
}

type Result struct {
	Build     *Build
	Id        string `json:"build"`
	Name      string `json:"name"`
	Configure string `json:"configure"`
	Compile   string `json:"compile"`
	Check     string `json:"check"`
}

type BuildEnvResult struct {
	Env     *BuildEnv
	Results []Result
}

type Commit struct {
	Rev       string
	NodeShort string
	Node      string
	Desc      string
	Author    string
	Date      string
}

type CommitResult struct {
	Commit  *Commit
	Results []BuildEnvResult
}

func main() {
	data, err := os.ReadFile("testconfig.xml")
	if err != nil {
		log.Fatal(err)
	}

	config := &Config{}
	if err := xml.Unmarshal(data, &config); err != nil {
		log.Fatal(err)
	}
	config.BuildEnvMap = make(map[string]BuildEnv)
	for _, env := range config.BuildEnv {
		config.BuildEnvMap[env.Name] = env
	}

	for ri := range config.Repository {
		for bi := range config.Repository[ri].Build {
			build := &config.Repository[ri].Build[bi]
			build.NotList = strings.Split(build.NotPlatform, ",")
		}
	}

	templateStr, err := os.ReadFile("build.template")
	if err != nil {
		log.Fatal(err)
	}

	templateStr2, err := os.ReadFile("build.ps1.template")
	if err != nil {
		log.Fatal(err)
	}

	funcs := template.FuncMap{
		"add": func(i int, n int) int {
			return i + n
		},
		"sub": func(i int, n int) int {
			return i - n
		},
	}

	tpl, err := template.New("").Funcs(funcs).Parse(string(templateStr))
	if err != nil {
		log.Fatal(err)
	}

	tpl2, err := template.New("").Funcs(funcs).Parse(string(templateStr2))
	if err != nil {
		log.Fatal(err)
	}

	err = os.Mkdir("tmp", 0755)
	if err != nil && !os.IsExist(err) {
		log.Fatal(err)
	}

	tmp := "tmp"
	out := "out"

	for _, repo := range config.Repository {
		commit := get_commit_info(repo.Path)
		if commit == nil {
			log.Print("cannot get current commit info for repo ", repo.Path)
			continue
		}
		result_dir := path.Join(out, commit.Node)
		_, err := os.Stat(result_dir)
		if err == nil {
			log.Printf("repository %s commit %s already built", repo.Path, commit.NodeShort)
			continue
		}

		if err := create_repo_archive(&repo, tpl, tpl2, tmp); err != nil {
			log.Fatal(err)
		}

		results := exec_buildenvs(config, &repo, tmp)

		err = save_result(&repo, commit, results, tmp)
		if err != nil {
			log.Print("cannot save results for commit ", commit.NodeShort)
		}
	}
}

func create_repo_archive(repo *Repository, tpl *template.Template, tpl2 *template.Template, tmp_dir string) error {
	// create a build directory, that contains the repository source
	// and build scripts
	buildPath := path.Join(tmp_dir, "build")

	if err := os.RemoveAll(buildPath); err != nil {
		log.Print("remove build dir failed: ", err)
		return err
	}
	if err := os.RemoveAll(path.Join(tmp_dir, "build.tar")); err != nil {
		log.Print("remove build.tar failed")
		return err
	}
	if err := os.RemoveAll(path.Join(tmp_dir, "build.tar.gz")); err != nil {
		log.Print("remove build.tar.gz failed")
		return err
	}
	if err := os.RemoveAll(path.Join(tmp_dir, "result")); err != nil {
		log.Print("remove result failed")
		return err
	}

	err := os.Mkdir(buildPath, 0755)
	if err != nil {
		log.Print("mkdir build dir failed: ", err)
		return err
	}

	// update repository and copy it to the build directory
	cmd := exec.Command("hg", "pull")
	var outb, errb bytes.Buffer
	cmd.Dir = repo.Path
	cmd.Stdout = &outb
	cmd.Stderr = &errb
	err = cmd.Run()
	if err != nil {
		log.Print("hg: ", errb.String())
		return err
	}

	outb.Reset()
	errb.Reset()
	cmd = exec.Command("hg", "update")
	cmd.Dir = repo.Path
	cmd.Stdout = &outb
	cmd.Stderr = &errb
	err = cmd.Run()
	if err != nil {
		log.Print("hg: ", errb.String())
		return err
	}

	// copy repository to the tmp_dir build directory
	cmd = exec.Command("cp", "-r", repo.Path, path.Join(buildPath, "src"))
	err = cmd.Run()
	if err != nil {
		log.Print("cannot copy the repository to the build dir")
		return err
	}

	// create build script
	file, err := os.Create(path.Join(buildPath, "build.sh"))
	if err != nil {
		log.Print("cannot create build.sh")
		return err
	}
	defer file.Close()

	err = tpl.Execute(file, repo)
	if err != nil {
		log.Print("cannot execute build.sh template")
		return err
	}
	file.Chmod(0755)

	file2, err := os.Create(path.Join(buildPath, "build.ps1"))
	if err != nil {
		log.Print("cannot create build.ps1")
		return err
	}
	defer file2.Close()
	err = tpl2.Execute(file2, repo)
	if err != nil {
		log.Print("cannot execute build.ps1 template")
		return err
	}

	// create tarball
	cmd = exec.Command("tar", "cvf", "../build.tar", "src", "build.sh", "build.ps1")
	cmd.Dir = buildPath
	if err := cmd.Run(); err != nil {
		log.Print("tar error")
		return err
	}

	cmd = exec.Command("gzip", path.Join(tmp_dir, "build.tar"))
	if err := cmd.Run(); err != nil {
		log.Print("gzip error")
		return err
	}

	return nil
}

func host_is_available(host string) bool {
	cmd := exec.Command("ping", "-c", "1", host)
	if err := cmd.Run(); err != nil {
		return false
	}
	return true
}

func get_build(builds []Build, id string) *Build {
	i, err := strconv.Atoi(id)
	if err != nil {
		return nil
	}
	if i < 0 || i >= len(builds) {
		return nil
	}
	return &builds[i]
}

func exec_buildenvs(config *Config, repo *Repository, tmp_dir string) []BuildEnvResult {
	var buildResults []BuildEnvResult

	if len(repo.BuildEnvs) == 0 {
		log.Print("repo %s has no buildenvs", repo.Path)
		return buildResults
	}

	buildenvs := strings.Split(repo.BuildEnvs, ",")
	for _, buildenv := range buildenvs {
		env, ok := config.BuildEnvMap[buildenv]
		if !ok {
			log.Print("unknown build env ", buildenv)
			continue
		}
		if !host_is_available(env.Host) {
			log.Print("skip unavailable host ", env.Host)
			continue
		}

		// exec posix or windows build script
		var result BuildEnvResult
		var err error
		if env.OS == "windows" {
			result, err = exec_win_buildenv(config, repo, &env, tmp_dir)
		} else {
			result, err = exec_posix_buildenv(config, repo, &env, tmp_dir)
		}
		if err != nil {
			log.Print("buildenv exec error: ", err)
			continue
		}

		buildResults = append(buildResults, result)
	}

	return buildResults
}

func exec_posix_buildenv(config *Config, repo *Repository, env *BuildEnv, tmp_dir string) (BuildEnvResult, error) {
	result := BuildEnvResult{}

	// steps:
	// upload build.tar.gz
	// extract build.tar.gz on the buildenv host and run build.sh

	// upload using scp
	cmd := exec.Command("scp", path.Join(tmp_dir, "build.tar.gz"), env.Host+":")
	log.Print("scp: ", cmd.Args)
	if err := cmd.Run(); err != nil {
		return result, errors.New("cannot copy build env to " + env.Host)
	}

	// run build.sh
	build_dir := "build"
	remote_command := fmt.Sprintf("sh -c 'rm -Rf %[1]s; mkdir -p %[1]s; gzip -dc build.tar.gz | tar xf - -C %[1]s ; (cd %[1]s; ./build.sh)'", build_dir)
	log.Printf("host: %s: command: %s", env.Host, remote_command)

	var outb, errb bytes.Buffer
	cmd = exec.Command("ssh", "-t", env.Host, remote_command)
	cmd.Stdout = &outb
	cmd.Stderr = &errb
	err := cmd.Run()
	log.Printf("host: %s: stdout: %s", env.Host, outb.String())
	if err != nil {
		log.Print("stderr: ", errb.String())
		return result, errors.New("cannot execute build.sh on buildenv host " + env.Host)
	}

	// download result
	cmd.Stdout = nil
	cmd.Stderr = nil
	cmd = exec.Command("scp", "-r", env.Host+":"+build_dir+"/result", tmp_dir)
	log.Print("scp: ", cmd.Args)
	if err := cmd.Run(); err != nil {
		return result, errors.New("cannot get result from buildenv")
	}

	// parse result
	resultData, err := os.ReadFile(path.Join(tmp_dir, "result/result.json"))
	if err != nil {
		return result, errors.New("cannot read result from buildenv")
	}

	var results []Result
	if err := json.Unmarshal(resultData, &results); err != nil {
		return result, errors.New("cannot parse result.json")
	}

	// store the actual Build pointer in the result
	for i := range results {
		results[i].Build = get_build(repo.Build, results[i].Id)
	}
	log.Print("buildenv finished ", env.Name)

	return BuildEnvResult{Env: env, Results: results}, nil
}

func exec_win_buildenv(config *Config, repo *Repository, env *BuildEnv, tmp_dir string) (BuildEnvResult, error) {
	result := BuildEnvResult{}
	return result, nil
}

func get_commit_info(repo_dir string) *Commit {
	var outb, errb bytes.Buffer

	commit := &Commit{}

	cmd := exec.Command("hg", "log", "-r", ".", "-T", "{rev}\\t{node|short}\\t{node}\\t{date|isodate}")
	cmd.Dir = repo_dir
	cmd.Stdout = &outb
	cmd.Stderr = &errb
	if err := cmd.Run(); err != nil {
		log.Print(outb.String())
		log.Print(errb.String())
		return nil
	}

	info := strings.Split(outb.String(), "\t")
	if len(info) != 4 {
		log.Print(outb.String())
		log.Print(errb.String())
		return nil
	}

	commit.Rev = info[0]
	commit.NodeShort = info[1]
	commit.Node = info[2]
	commit.Date = info[3]

	outb.Reset()
	errb.Reset()

	cmd = exec.Command("hg", "log", "-r", ".", "-T", "{author}")
	cmd.Dir = repo_dir
	cmd.Stdout = &outb
	cmd.Stderr = &errb
	if err := cmd.Run(); err != nil {
		log.Print(outb.String())
		log.Print(errb.String())
		return nil
	}
	commit.Author = outb.String()

	cmd = exec.Command("hg", "log", "-r", ".", "-T", "{desc}")
	cmd.Dir = repo_dir
	cmd.Stdout = &outb
	cmd.Stderr = &errb
	if err := cmd.Run(); err != nil {
		log.Print(outb.String())
		log.Print(errb.String())
		return nil
	}
	commit.Desc = outb.String()

	return commit
}

func save_result(repo *Repository, commit *Commit, result []BuildEnvResult, tmp_dir string) error {
	out_dir := "out/"
	src := path.Join(tmp_dir, "result")
	dst := path.Join(out_dir, commit.Node)

	// move result
	if err := os.Rename(src, dst); err != nil {
		return err
	}

	return nil
}