<commit_before>package server

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"strconv"
	"strings"
	"time"

	"resenje.org/httputils"
	"resenje.org/logging"

	"gopherpit.com/gopherpit/services/packages"
)

func (s Server) packageResolverHandler(w http.ResponseWriter, r *http.Request) {
	var code int
	defer func(startTime time.Time) {
		referrer := r.Referer()
		if referrer == "" {
			referrer = "-"
		}
		userAgent := r.UserAgent()
		if userAgent == "" {
			userAgent = "-"
		}
		ips := []string{}
		xfr := r.Header.Get("X-Forwarded-For")
		if xfr != "" {
			ips = append(ips, xfr)
		}
		xri := r.Header.Get("X-Real-Ip")
		if xri != "" {
			ips = append(ips, xri)
		}
		xips := "-"
		if len(ips) > 0 {
			xips = strings.Join(ips, ", ")
		}
		var level logging.Level
		switch {
		case code >= 500:
			level = logging.ERROR
		case code >= 400:
			level = logging.WARNING
		case code >= 300:
			level = logging.INFO
		case code >= 200:
			level = logging.INFO
		default:
			level = logging.DEBUG
		}
		s.packageAccessLogger.Logf(level, "%s \"%s\" %s %s %s %d %f \"%s\" \"%s\"", r.RemoteAddr, xips, r.Method, httputils.GetRequestEndpoint(r)+r.URL.String(), r.Proto, code, time.Since(startTime).Seconds(), referrer, userAgent)
	}(time.Now())

	domain, _, err := net.SplitHostPort(r.Host)
	if err != nil {
		domain = r.Host
	}
	path := domain + r.URL.Path
	resolution, err := s.PackagesService.ResolvePackage(path)
	if err != nil {
		if err == packages.DomainNotFound || err == packages.PackageNotFound {
			code = http.StatusNotFound
			w.Header().Set("Content-Type", "text/plain; charset=utf-8")
			w.WriteHeader(code)
			fmt.Fprintln(w, fmt.Sprintf("%s: package %s", http.StatusText(code), path))
			return
		}
		s.logger.Errorf("package resolver: resolve package: %s", err)
		code = http.StatusInternalServerError
		textServerError(w, err)
		return
	}

	if resolution.Disabled {
		code = http.StatusNotFound
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.WriteHeader(code)
		fmt.Fprintln(w, fmt.Sprintf("%s: package %s", http.StatusText(code), path))
		return
	}

	code = 200

	repoRoot := resolution.RepoRoot
	if resolution.RefType != "" {
		// If Reference is changed, repo root does not have scheme,
		// so it is set here.
		repoRoot = "http://" + resolution.ImportPrefix
		if s.RedirectToHTTPS {
			repoRoot = "https://" + resolution.ImportPrefix
		}
	}

	s.respond(w, "PackageResolution", map[string]interface{}{
		"GoImport":    fmt.Sprintf("%s %s %s", resolution.ImportPrefix, resolution.VCS, repoRoot),
		"GoSource":    resolution.GoSource,
		"RedirectURL": resolution.RedirectURL,
	})
}

func (s Server) packageGitUploadPackHandler(w http.ResponseWriter, r *http.Request) (notFound bool) {
	var code int
	defer func(startTime time.Time) {
		if !notFound {
			referrer := r.Referer()
			if referrer == "" {
				referrer = "-"
			}
			userAgent := r.UserAgent()
			if userAgent == "" {
				userAgent = "-"
			}
			ips := []string{}
			xfr := r.Header.Get("X-Forwarded-For")
			if xfr != "" {
				ips = append(ips, xfr)
			}
			xri := r.Header.Get("X-Real-Ip")
			if xri != "" {
				ips = append(ips, xri)
			}
			xips := "-"
			if len(ips) > 0 {
				xips = strings.Join(ips, ", ")
			}
			var level logging.Level
			switch {
			case code >= 500:
				level = logging.ERROR
			case code >= 400:
				level = logging.WARNING
			case code >= 300:
				level = logging.INFO
			case code >= 200:
				level = logging.INFO
			default:
				level = logging.DEBUG
			}
			s.packageAccessLogger.Logf(level, "%s \"%s\" %s %s %s %d %f \"%s\" \"%s\"", r.RemoteAddr, xips, r.Method, httputils.GetRequestEndpoint(r)+r.URL.String(), r.Proto, code, time.Since(startTime).Seconds(), referrer, userAgent)
		}
	}(time.Now())

	domain, _, err := net.SplitHostPort(r.Host)
	if err != nil {
		domain = r.Host
	}
	path := domain + strings.TrimSuffix(r.URL.Path, "/git-upload-pack")
	resolution, err := s.PackagesService.ResolvePackage(path)
	if err != nil {
		if err == packages.DomainNotFound || err == packages.PackageNotFound {
			notFound = true
			return
		}
		s.logger.Errorf("package git upload pack: resolve package: %s", err)
		code = 500
		textServerError(w, err)
		return
	}

	if resolution.Disabled {
		notFound = true
		return
	}

	code = 200
	w.Header().Set("Location", strings.TrimSuffix(resolution.RepoRoot, ".git")+".git/git-upload-pack")
	w.WriteHeader(http.StatusMovedPermanently)
	return
}

var (
	errRefNotFound = errors.New("reference not found")
	httpClient     = &http.Client{
		Timeout: 15 * time.Second,
	}
)

func (s Server) packageGitInfoRefsHandler(w http.ResponseWriter, r *http.Request) (notFound bool) {
	var code int
	defer func(startTime time.Time) {
		if !notFound {
			referrer := r.Referer()
			if referrer == "" {
				referrer = "-"
			}
			userAgent := r.UserAgent()
			if userAgent == "" {
				userAgent = "-"
			}
			ips := []string{}
			xfr := r.Header.Get("X-Forwarded-For")
			if xfr != "" {
				ips = append(ips, xfr)
			}
			xri := r.Header.Get("X-Real-Ip")
			if xri != "" {
				ips = append(ips, xri)
			}
			xips := "-"
			if len(ips) > 0 {
				xips = strings.Join(ips, ", ")
			}
			var level logging.Level
			switch {
			case code >= 500:
				level = logging.ERROR
			case code >= 400:
				level = logging.WARNING
			case code >= 300:
				level = logging.INFO
			case code >= 200:
				level = logging.INFO
			default:
				level = logging.DEBUG
			}
			s.packageAccessLogger.Logf(level, "%s \"%s\" %s %s %s %d %f \"%s\" \"%s\"", r.RemoteAddr, xips, r.Method, httputils.GetRequestEndpoint(r)+r.URL.String(), r.Proto, code, time.Since(startTime).Seconds(), referrer, userAgent)
		}
	}(time.Now())

	domain, _, err := net.SplitHostPort(r.Host)
	if err != nil {
		domain = r.Host
	}
	path := domain + strings.TrimSuffix(r.URL.Path, "/info/refs")
	resolution, err := s.PackagesService.ResolvePackage(path)
	if err != nil {
		if err == packages.DomainNotFound || err == packages.PackageNotFound {
			notFound = true
			return
		}
		s.logger.Errorf("package git info refs: resolve package: %s", err)
		code = 500
		textServerError(w, err)
		return
	}
	refsURL := resolution.RepoRoot
	refsURL = strings.TrimRight(refsURL, "/")
	refsURL = strings.TrimSuffix(refsURL, ".git")
	refsURL += ".git/info/refs?service=git-upload-pack"

	resp, err := httpClient.Get(refsURL)
	if err != nil {
		s.logger.Errorf("package git info refs: http get: %s", err)
		code = 500
		textServerError(w, err)
		return
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		s.logger.Warningf("package git info refs: http get: %s: status code %v", refsURL, resp.StatusCode)
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.WriteHeader(resp.StatusCode)
		fmt.Fprintln(w, resp.Status)
		code = resp.StatusCode
		return
	}

	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		s.logger.Errorf("package git info refs: http read body: %s", err)
		code = 500
		textServerError(w, err)
		return
	}

	w.Header().Set("Content-Type", "application/x-git-upload-pack-advertisement")

	if err = writeAlteredGitInfoRef(data, w, resolution.RefType, resolution.RefName); err != nil {
		if err == errRefNotFound {
			s.logger.Warningf("package git info refs: alter refs: %s", err)
			w.Header().Set("Content-Type", "text/plain; charset=utf-8")
			w.WriteHeader(http.StatusNotFound)
			fmt.Fprintln(w, fmt.Sprintf("%s: %s %s", http.StatusText(http.StatusNotFound), resolution.RefType, resolution.RefName))
			code = 404
			return
		}
		s.logger.Errorf("package git info refs: alter refs: %s", err)
		code = 500
		textServerError(w, err)
		return
	}
	code = 200
	return
}

func writeAlteredGitInfoRef(data []byte, w io.Writer, refType, refName string) error {
	var ref string
	switch refType {
	case packages.RefTypeBranch:
		ref = "refs/heads/" + refName
	case packages.RefTypeTag:
		ref = "refs/tags/" + refName
	default:
		return fmt.Errorf("invalid reference type: %s", refType)
	}
	var (
		headIndexStart   int
		headIndexEnd     int
		masterIndexStart int
		masterIndexEnd   int
	)
	var refHash []byte
	for stat, end := 0, 0; stat < len(data); stat = end {
		size, err := strconv.ParseInt(string(data[stat:stat+4]), 16, 32)
		if err != nil {
			return fmt.Errorf("parse line size: %s", string(data[stat:stat+4]))
		}
		if size == 0 {
			size = 4
		}
		end = stat + int(size)
		if end > len(data) {
			return fmt.Errorf("incomplete data")
		}
		if len(data) > stat+4 && data[stat+4] == '#' {
			continue
		}

		hashIndexStart := stat + 4
		hashIndexEnd := bytes.IndexByte(data[hashIndexStart:end], ' ')
		if hashIndexEnd < 0 || hashIndexEnd != 40 {
			continue
		}
		hashIndexEnd += hashIndexStart

		nameIndexStart := hashIndexEnd + 1
		nameIndexEnd := bytes.IndexAny(data[nameIndexStart:end], "\n\x00")
		if nameIndexEnd < 0 {
			nameIndexEnd = end
		} else {
			nameIndexEnd += nameIndexStart
		}

		name := string(data[nameIndexStart:nameIndexEnd])
		if name == "HEAD" {
			headIndexStart = stat
			headIndexEnd = end
			continue
		}
		if name == "refs/heads/master" {
			masterIndexStart = stat
			masterIndexEnd = end
			continue
		}
		if name == ref {
			refHash = data[hashIndexStart:hashIndexEnd]
			switch refType {
			case packages.RefTypeBranch:
				break
			case packages.RefTypeTag:
				continue
			}
		}
		// Anotated tags
		if refType == packages.RefTypeTag && name == ref+"^{}" {
			refHash = data[hashIndexStart:hashIndexEnd]
			break
		}
	}

	if headIndexStart == 0 || len(refHash) == 0 {
		return errRefNotFound
	}

	w.Write(data[:headIndexStart])

	capabilities := ""
	if i := bytes.Index(data[headIndexStart:headIndexEnd], []byte{'\x00'}); i > 0 {
		capabilities = strings.Replace(string(data[headIndexStart+i+1:headIndexEnd-1]), "symref=", "oldref=", -1)
	}

	var line string
	if refType == packages.RefTypeBranch {
		// Always reference master branch in symref to be able to change branch in the future, and
		// to have go get -u working.
		if capabilities == "" {
			line = fmt.Sprintf("%s HEAD\x00symref=HEAD:refs/heads/master\n", refHash)
		} else {
			line = fmt.Sprintf("%s HEAD\x00symref=HEAD:refs/heads/master %s\n", refHash, capabilities)
		}
	} else {
		if capabilities == "" {
			line = fmt.Sprintf("%s HEAD\n", refHash)
		} else {
			line = fmt.Sprintf("%s HEAD\x00%s\n", refHash, capabilities)
		}
	}
	fmt.Fprintf(w, "%04x%s", 4+len(line), line)

	line = fmt.Sprintf("%s refs/heads/master\n", refHash)
	fmt.Fprintf(w, "%04x%s", 4+len(line), line)

	if masterIndexStart > 0 {
		w.Write(data[headIndexEnd:masterIndexStart])
		w.Write(data[masterIndexEnd:])
	} else {
		w.Write(data[headIndexEnd:])
	}

	return nil
}
<commit_msg>Proxy git-upload-data requests to be compatible with Git >=2.11.1<commit_after>package server

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"strconv"
	"strings"
	"time"

	"resenje.org/httputils"
	"resenje.org/logging"

	"gopherpit.com/gopherpit/services/packages"
)

func (s Server) packageResolverHandler(w http.ResponseWriter, r *http.Request) {
	var code int
	defer func(startTime time.Time) {
		referrer := r.Referer()
		if referrer == "" {
			referrer = "-"
		}
		userAgent := r.UserAgent()
		if userAgent == "" {
			userAgent = "-"
		}
		ips := []string{}
		xfr := r.Header.Get("X-Forwarded-For")
		if xfr != "" {
			ips = append(ips, xfr)
		}
		xri := r.Header.Get("X-Real-Ip")
		if xri != "" {
			ips = append(ips, xri)
		}
		xips := "-"
		if len(ips) > 0 {
			xips = strings.Join(ips, ", ")
		}
		var level logging.Level
		switch {
		case code >= 500:
			level = logging.ERROR
		case code >= 400:
			level = logging.WARNING
		case code >= 300:
			level = logging.INFO
		case code >= 200:
			level = logging.INFO
		default:
			level = logging.DEBUG
		}
		s.packageAccessLogger.Logf(level, "%s \"%s\" %s %s %s %d %f \"%s\" \"%s\"", r.RemoteAddr, xips, r.Method, httputils.GetRequestEndpoint(r)+r.URL.String(), r.Proto, code, time.Since(startTime).Seconds(), referrer, userAgent)
	}(time.Now())

	domain, _, err := net.SplitHostPort(r.Host)
	if err != nil {
		domain = r.Host
	}
	path := domain + r.URL.Path
	resolution, err := s.PackagesService.ResolvePackage(path)
	if err != nil {
		if err == packages.DomainNotFound || err == packages.PackageNotFound {
			code = http.StatusNotFound
			w.Header().Set("Content-Type", "text/plain; charset=utf-8")
			w.WriteHeader(code)
			fmt.Fprintln(w, fmt.Sprintf("%s: package %s", http.StatusText(code), path))
			return
		}
		s.logger.Errorf("package resolver: resolve package: %s", err)
		code = http.StatusInternalServerError
		textServerError(w, err)
		return
	}

	if resolution.Disabled {
		code = http.StatusNotFound
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.WriteHeader(code)
		fmt.Fprintln(w, fmt.Sprintf("%s: package %s", http.StatusText(code), path))
		return
	}

	code = 200

	repoRoot := resolution.RepoRoot
	if resolution.RefType != "" {
		// If Reference is changed, repo root does not have scheme,
		// so it is set here.
		repoRoot = "http://" + resolution.ImportPrefix
		if s.RedirectToHTTPS {
			repoRoot = "https://" + resolution.ImportPrefix
		}
	}

	s.respond(w, "PackageResolution", map[string]interface{}{
		"GoImport":    fmt.Sprintf("%s %s %s", resolution.ImportPrefix, resolution.VCS, repoRoot),
		"GoSource":    resolution.GoSource,
		"RedirectURL": resolution.RedirectURL,
	})
}

func (s Server) packageGitUploadPackHandler(w http.ResponseWriter, r *http.Request) (notFound bool) {
	var code int
	defer func(startTime time.Time) {
		if !notFound {
			referrer := r.Referer()
			if referrer == "" {
				referrer = "-"
			}
			userAgent := r.UserAgent()
			if userAgent == "" {
				userAgent = "-"
			}
			ips := []string{}
			xfr := r.Header.Get("X-Forwarded-For")
			if xfr != "" {
				ips = append(ips, xfr)
			}
			xri := r.Header.Get("X-Real-Ip")
			if xri != "" {
				ips = append(ips, xri)
			}
			xips := "-"
			if len(ips) > 0 {
				xips = strings.Join(ips, ", ")
			}
			var level logging.Level
			switch {
			case code >= 500:
				level = logging.ERROR
			case code >= 400:
				level = logging.WARNING
			case code >= 300:
				level = logging.INFO
			case code >= 200:
				level = logging.INFO
			default:
				level = logging.DEBUG
			}
			s.packageAccessLogger.Logf(level, "%s \"%s\" %s %s %s %d %f \"%s\" \"%s\"", r.RemoteAddr, xips, r.Method, httputils.GetRequestEndpoint(r)+r.URL.String(), r.Proto, code, time.Since(startTime).Seconds(), referrer, userAgent)
		}
	}(time.Now())

	domain, _, err := net.SplitHostPort(r.Host)
	if err != nil {
		domain = r.Host
	}
	path := domain + strings.TrimSuffix(r.URL.Path, "/git-upload-pack")
	resolution, err := s.PackagesService.ResolvePackage(path)
	if err != nil {
		if err == packages.DomainNotFound || err == packages.PackageNotFound {
			notFound = true
			return
		}
		s.logger.Errorf("package git upload pack: resolve package: %s", err)
		code = 500
		textServerError(w, err)
		return
	}

	if resolution.Disabled {
		notFound = true
		return
	}

	req, err := http.NewRequest(r.Method, strings.TrimSuffix(resolution.RepoRoot, ".git")+".git/git-upload-pack", r.Body)
	if err != nil {
		s.logger.Errorf("package git upload pack: new request: %s", err)
		code = 500
		textServerError(w, err)
		return
	}
	defer r.Body.Close()

	req.Header.Set("User-Agent", r.Header.Get("User-Agent"))
	req.Header.Set("Accept", r.Header.Get("Accept"))
	req.Header.Set("Content-Type", r.Header.Get("Content-Type"))

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		s.logger.Errorf("package git upload pack: make request: %s", err)
		code = 500
		textServerError(w, err)
		return
	}
	defer resp.Body.Close()

	if _, err = io.Copy(w, resp.Body); err != nil {
		s.logger.Errorf("package git upload pack: copy request data: %s", err)
		code = 500
		textServerError(w, err)
		return
	}

	code = 200
	return
}

var (
	errRefNotFound = errors.New("reference not found")
	httpClient     = &http.Client{
		Timeout: 15 * time.Second,
	}
)

func (s Server) packageGitInfoRefsHandler(w http.ResponseWriter, r *http.Request) (notFound bool) {
	var code int
	defer func(startTime time.Time) {
		if !notFound {
			referrer := r.Referer()
			if referrer == "" {
				referrer = "-"
			}
			userAgent := r.UserAgent()
			if userAgent == "" {
				userAgent = "-"
			}
			ips := []string{}
			xfr := r.Header.Get("X-Forwarded-For")
			if xfr != "" {
				ips = append(ips, xfr)
			}
			xri := r.Header.Get("X-Real-Ip")
			if xri != "" {
				ips = append(ips, xri)
			}
			xips := "-"
			if len(ips) > 0 {
				xips = strings.Join(ips, ", ")
			}
			var level logging.Level
			switch {
			case code >= 500:
				level = logging.ERROR
			case code >= 400:
				level = logging.WARNING
			case code >= 300:
				level = logging.INFO
			case code >= 200:
				level = logging.INFO
			default:
				level = logging.DEBUG
			}
			s.packageAccessLogger.Logf(level, "%s \"%s\" %s %s %s %d %f \"%s\" \"%s\"", r.RemoteAddr, xips, r.Method, httputils.GetRequestEndpoint(r)+r.URL.String(), r.Proto, code, time.Since(startTime).Seconds(), referrer, userAgent)
		}
	}(time.Now())

	domain, _, err := net.SplitHostPort(r.Host)
	if err != nil {
		domain = r.Host
	}
	path := domain + strings.TrimSuffix(r.URL.Path, "/info/refs")
	resolution, err := s.PackagesService.ResolvePackage(path)
	if err != nil {
		if err == packages.DomainNotFound || err == packages.PackageNotFound {
			notFound = true
			return
		}
		s.logger.Errorf("package git info refs: resolve package: %s", err)
		code = 500
		textServerError(w, err)
		return
	}
	refsURL := resolution.RepoRoot
	refsURL = strings.TrimRight(refsURL, "/")
	refsURL = strings.TrimSuffix(refsURL, ".git")
	refsURL += ".git/info/refs?service=git-upload-pack"

	resp, err := httpClient.Get(refsURL)
	if err != nil {
		s.logger.Errorf("package git info refs: http get: %s", err)
		code = 500
		textServerError(w, err)
		return
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		s.logger.Warningf("package git info refs: http get: %s: status code %v", refsURL, resp.StatusCode)
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.WriteHeader(resp.StatusCode)
		fmt.Fprintln(w, resp.Status)
		code = resp.StatusCode
		return
	}

	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		s.logger.Errorf("package git info refs: http read body: %s", err)
		code = 500
		textServerError(w, err)
		return
	}

	w.Header().Set("Content-Type", "application/x-git-upload-pack-advertisement")

	if err = writeAlteredGitInfoRef(data, w, resolution.RefType, resolution.RefName); err != nil {
		if err == errRefNotFound {
			s.logger.Warningf("package git info refs: alter refs: %s", err)
			w.Header().Set("Content-Type", "text/plain; charset=utf-8")
			w.WriteHeader(http.StatusNotFound)
			fmt.Fprintln(w, fmt.Sprintf("%s: %s %s", http.StatusText(http.StatusNotFound), resolution.RefType, resolution.RefName))
			code = 404
			return
		}
		s.logger.Errorf("package git info refs: alter refs: %s", err)
		code = 500
		textServerError(w, err)
		return
	}
	code = 200
	return
}

func writeAlteredGitInfoRef(data []byte, w io.Writer, refType, refName string) error {
	var ref string
	switch refType {
	case packages.RefTypeBranch:
		ref = "refs/heads/" + refName
	case packages.RefTypeTag:
		ref = "refs/tags/" + refName
	default:
		return fmt.Errorf("invalid reference type: %s", refType)
	}
	var (
		headIndexStart   int
		headIndexEnd     int
		masterIndexStart int
		masterIndexEnd   int
	)
	var refHash []byte
	for stat, end := 0, 0; stat < len(data); stat = end {
		size, err := strconv.ParseInt(string(data[stat:stat+4]), 16, 32)
		if err != nil {
			return fmt.Errorf("parse line size: %s", string(data[stat:stat+4]))
		}
		if size == 0 {
			size = 4
		}
		end = stat + int(size)
		if end > len(data) {
			return fmt.Errorf("incomplete data")
		}
		if len(data) > stat+4 && data[stat+4] == '#' {
			continue
		}

		hashIndexStart := stat + 4
		hashIndexEnd := bytes.IndexByte(data[hashIndexStart:end], ' ')
		if hashIndexEnd < 0 || hashIndexEnd != 40 {
			continue
		}
		hashIndexEnd += hashIndexStart

		nameIndexStart := hashIndexEnd + 1
		nameIndexEnd := bytes.IndexAny(data[nameIndexStart:end], "\n\x00")
		if nameIndexEnd < 0 {
			nameIndexEnd = end
		} else {
			nameIndexEnd += nameIndexStart
		}

		name := string(data[nameIndexStart:nameIndexEnd])
		if name == "HEAD" {
			headIndexStart = stat
			headIndexEnd = end
			continue
		}
		if name == "refs/heads/master" {
			masterIndexStart = stat
			masterIndexEnd = end
			continue
		}
		if name == ref {
			refHash = data[hashIndexStart:hashIndexEnd]
			switch refType {
			case packages.RefTypeBranch:
				break
			case packages.RefTypeTag:
				continue
			}
		}
		// Anotated tags
		if refType == packages.RefTypeTag && name == ref+"^{}" {
			refHash = data[hashIndexStart:hashIndexEnd]
			break
		}
	}

	if headIndexStart == 0 || len(refHash) == 0 {
		return errRefNotFound
	}

	w.Write(data[:headIndexStart])

	capabilities := ""
	if i := bytes.Index(data[headIndexStart:headIndexEnd], []byte{'\x00'}); i > 0 {
		capabilities = strings.Replace(string(data[headIndexStart+i+1:headIndexEnd-1]), "symref=", "oldref=", -1)
	}

	var line string
	if refType == packages.RefTypeBranch {
		// Always reference master branch in symref to be able to change branch in the future, and
		// to have go get -u working.
		if capabilities == "" {
			line = fmt.Sprintf("%s HEAD\x00symref=HEAD:refs/heads/master\n", refHash)
		} else {
			line = fmt.Sprintf("%s HEAD\x00symref=HEAD:refs/heads/master %s\n", refHash, capabilities)
		}
	} else {
		if capabilities == "" {
			line = fmt.Sprintf("%s HEAD\n", refHash)
		} else {
			line = fmt.Sprintf("%s HEAD\x00%s\n", refHash, capabilities)
		}
	}
	fmt.Fprintf(w, "%04x%s", 4+len(line), line)

	line = fmt.Sprintf("%s refs/heads/master\n", refHash)
	fmt.Fprintf(w, "%04x%s", 4+len(line), line)

	if masterIndexStart > 0 {
		w.Write(data[headIndexEnd:masterIndexStart])
		w.Write(data[masterIndexEnd:])
	} else {
		w.Write(data[headIndexEnd:])
	}

	return nil
}
<|endoftext|>


<commit_before>/* Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements.  See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

package avro

import "encoding/json"

type AvroRecord interface {
	Schema() Schema
}

// GenericRecord is a generic instance of a record schema.
// Fields are accessible by their name.
type GenericRecord struct {
	fields map[string]interface{}
	schema Schema
}

// Creates a new GenericRecord.
func NewGenericRecord(schema Schema) *GenericRecord {
	return &GenericRecord{
		fields: make(map[string]interface{}),
		schema: schema,
	}
}

// Gets a value by its name.
func (this *GenericRecord) Get(name string) interface{} {
	return this.fields[name]
}

// Sets a value for a given name.
func (this *GenericRecord) Set(name string, value interface{}) {
	this.fields[name] = value
}

// Returns a schema for this GenericRecord.
func (this *GenericRecord) Schema() Schema {
	return this.schema
}

// Returns a JSON representation of this GenericRecord.
func (this *GenericRecord) String() (string, error) {
	m := makeMap(this)
	buf, err := json.Marshal(m)
	return string(buf), err
}

func makeMap(rec *GenericRecord) map[string]interface{} {
	m := make(map[string]interface{})
	for k, v := range rec.fields {
		if r, ok := v.(*GenericRecord); ok {
			v = makeMap(r)
		}
		if a, ok := v.([]interface{}); ok {
			slice := make([]interface{}, len(a))
			for i, elem := range a {
				if rec, ok := elem.(*GenericRecord); ok {
					elem = makeMap(rec)
				}
				slice[i] = elem
			}
			v = slice
		}
		m[k] = v
	}
	return m
}
<commit_msg>Export Map() method on GenericRecord<commit_after>/* Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements.  See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

package avro

import "encoding/json"

type AvroRecord interface {
	Schema() Schema
}

// GenericRecord is a generic instance of a record schema.
// Fields are accessible by their name.
type GenericRecord struct {
	fields map[string]interface{}
	schema Schema
}

// Creates a new GenericRecord.
func NewGenericRecord(schema Schema) *GenericRecord {
	return &GenericRecord{
		fields: make(map[string]interface{}),
		schema: schema,
	}
}

// Gets a value by its name.
func (this *GenericRecord) Get(name string) interface{} {
	return this.fields[name]
}

// Sets a value for a given name.
func (this *GenericRecord) Set(name string, value interface{}) {
	this.fields[name] = value
}

// Returns a schema for this GenericRecord.
func (this *GenericRecord) Schema() Schema {
	return this.schema
}

// Returns a JSON representation of this GenericRecord.
func (this *GenericRecord) String() string {
	m := this.Map()
	buf, err := json.Marshal(m)
	if err != nil {
		panic(err)
	}
	return string(buf)
}

// Returns a map representation of this GenericRecord.
func (this *GenericRecord) Map() map[string]interface{} {
	m := make(map[string]interface{})
	for k, v := range this.fields {
		if r, ok := v.(*GenericRecord); ok {
			v = (r)
		}
		if a, ok := v.([]interface{}); ok {
			slice := make([]interface{}, len(a))
			for i, elem := range a {
				if rec, ok := elem.(*GenericRecord); ok {
					elem = rec.Map()
				}
				slice[i] = elem
			}
			v = slice
		}
		m[k] = v
	}
	return m
}
<|endoftext|>


<commit_before>package node

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	gethcommon "github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/core"
	"github.com/ethereum/go-ethereum/crypto"
	"github.com/ethereum/go-ethereum/eth"
	"github.com/ethereum/go-ethereum/eth/downloader"
	"github.com/ethereum/go-ethereum/les"
	"github.com/ethereum/go-ethereum/light"
	"github.com/ethereum/go-ethereum/log"
	"github.com/ethereum/go-ethereum/node"
	"github.com/ethereum/go-ethereum/p2p"
	"github.com/ethereum/go-ethereum/p2p/discover"
	"github.com/ethereum/go-ethereum/p2p/discv5"
	"github.com/ethereum/go-ethereum/p2p/nat"
	"github.com/ethereum/go-ethereum/whisper/mailserver"
	"github.com/ethereum/go-ethereum/whisper/notifications"
	whisper "github.com/ethereum/go-ethereum/whisper/whisperv5"
	"github.com/status-im/status-go/geth/common"
	"github.com/status-im/status-go/geth/params"
)

// node-related errors
var (
	ErrEthServiceRegistrationFailure     = errors.New("failed to register the Ethereum service")
	ErrWhisperServiceRegistrationFailure = errors.New("failed to register the Whisper service")
	ErrLightEthRegistrationFailure       = errors.New("failed to register the LES service")
	ErrNodeMakeFailure                   = errors.New("error creating p2p node")
	ErrNodeRunFailure                    = errors.New("error running p2p node")
	ErrNodeStartFailure                  = errors.New("error starting p2p node")
)

// MakeNode create a geth node entity
func MakeNode(config *params.NodeConfig) (*node.Node, error) {
	// make sure data directory exists
	if err := os.MkdirAll(filepath.Join(config.DataDir), os.ModePerm); err != nil {
		return nil, err
	}

	// make sure keys directory exists
	if err := os.MkdirAll(filepath.Join(config.KeyStoreDir), os.ModePerm); err != nil {
		return nil, err
	}

	// setup logging
	if _, err := common.SetupLogger(config); err != nil {
		return nil, err
	}

	// configure required node (should you need to update node's config, e.g. add bootstrap nodes, see node.Config)
	stackConfig := defaultEmbeddedNodeConfig(config)

	if len(config.NodeKeyFile) > 0 {
		log.Info("Loading private key file", "file", config.NodeKeyFile)
		pk, err := crypto.LoadECDSA(config.NodeKeyFile)
		if err != nil {
			log.Warn(fmt.Sprintf("Failed loading private key file '%s': %v", config.NodeKeyFile, err))
		}

		// override node's private key
		stackConfig.P2P.PrivateKey = pk
	}

	if len(config.NodeKeyFile) > 0 {
		log.Info("Loading private key file", "file", config.NodeKeyFile)
		pk, err := crypto.LoadECDSA(config.NodeKeyFile)
		if err != nil {
			log.Info("Failed loading private key file", "file", config.NodeKeyFile, "err", err)
		}

		// override node's private key
		stackConfig.P2P.PrivateKey = pk
	}

	stack, err := node.New(stackConfig)
	if err != nil {
		return nil, ErrNodeMakeFailure
	}

	// start Ethereum service
	if err := activateEthService(stack, config); err != nil {
		return nil, fmt.Errorf("%v: %v", ErrEthServiceRegistrationFailure, err)
	}

	// start Whisper service
	if err := activateShhService(stack, config); err != nil {
		return nil, fmt.Errorf("%v: %v", ErrWhisperServiceRegistrationFailure, err)
	}

	return stack, nil
}

// defaultEmbeddedNodeConfig returns default stack configuration for mobile client node
func defaultEmbeddedNodeConfig(config *params.NodeConfig) *node.Config {
	return &node.Config{
		DataDir:           config.DataDir,
		KeyStoreDir:       config.KeyStoreDir,
		UseLightweightKDF: true,
		NoUSB:             true,
		Name:              config.Name,
		Version:           config.Version,
		P2P: p2p.Config{
			NoDiscovery:      true,
			DiscoveryV5:      false,
			DiscoveryV5Addr:  ":0",
			BootstrapNodes:   makeBootstrapNodes(),
			BootstrapNodesV5: makeBootstrapNodesV5(),
			ListenAddr:       ":0",
			NAT:              nat.Any(),
			MaxPeers:         config.MaxPeers,
			MaxPendingPeers:  config.MaxPendingPeers,
		},
		IPCPath:     makeIPCPath(config),
		HTTPHost:    config.HTTPHost,
		HTTPPort:    config.HTTPPort,
		HTTPCors:    []string{"*"},
		HTTPModules: strings.Split(config.APIModules, ","),
		WSHost:      makeWSHost(config),
		WSPort:      config.WSPort,
		WSOrigins:   []string{"*"},
		WSModules:   strings.Split(config.APIModules, ","),
	}
}

// updateCHT changes trusted canonical hash trie root
func updateCHT(eth *les.LightEthereum, config *params.NodeConfig) {
	bc := eth.BlockChain()

	// TODO: Remove this thing as this is an ugly hack.
	// Once CHT sync sub-protocol is working in LES, we will rely on it, as it provides
	// decentralized solution. For now, in order to avoid forcing users to long sync times
	// we use central static resource
	type MsgCHTRoot struct {
		GenesisHash string `json:"net"`
		Number      uint64 `json:"number"`
		Prod        string `json:"prod"`
		Dev         string `json:"dev"`
	}
	loadCHTLists := func() ([]MsgCHTRoot, error) {
		url := config.LightEthConfig.CHTRootConfigURL + "?u=" + strconv.Itoa(int(time.Now().Unix()))
		client := &http.Client{Timeout: 5 * time.Second}
		r, err := client.Get(url)
		if err != nil {
			return nil, err
		}
		defer r.Body.Close()

		var roots []MsgCHTRoot
		err = json.NewDecoder(r.Body).Decode(&roots)
		if err != nil {
			return nil, err
		}

		return roots, nil
	}
	if roots, err := loadCHTLists(); err == nil {
		for _, root := range roots {
			if bc.Genesis().Hash().Hex() == root.GenesisHash {
				log.Info("Loaded root", "root", root)
				if root.Number == 0 {
					continue
				}

				chtRoot := root.Prod
				if config.DevMode {
					chtRoot = root.Dev
				}
				eth.WriteTrustedCht(light.TrustedCht{
					Number: root.Number,
					Root:   gethcommon.HexToHash(chtRoot),
				})
				log.Info("Loaded CHT from net", "CHT", chtRoot, "number", root.Number, "dev", config.DevMode)
				return
			}
		}
	}

	// resort to manually updated
	log.Info("Loading CHT from net failed, setting manually")
	if bc.Genesis().Hash() == params.MainNetGenesisHash {
		eth.WriteTrustedCht(light.TrustedCht{
			Number: 805,
			Root:   gethcommon.HexToHash("85e4286fe0a730390245c49de8476977afdae0eb5530b277f62a52b12313d50f"),
		})
		log.Info("Added trusted CHT for mainnet")
	}

	if bc.Genesis().Hash() == params.RopstenNetGenesisHash {
		root := "fa851b5252cc48ab55f375833b0344cc5c7cacea69be7e2a57976c38d3bb3aef"
		if config.DevMode {
			root = "f2f862314509b22a773eedaaa7fa6452474eb71a3b72525a97dbf5060cbea88f"
		}
		eth.WriteTrustedCht(light.TrustedCht{
			Number: 239,
			Root:   gethcommon.HexToHash(root),
		})
		log.Info("Added trusted CHT for Ropsten", "CHT", root)
	}

	//if bc.Genesis().Hash() == params.RinkebyNetGenesisHash {
	//	root := "0xb100882d00a09292f15e712707649b24d019a47a509e83a00530ac542425c3bd"
	//	if config.DevMode {
	//		root = "0xb100882d00a09292f15e712707649b24d019a47a509e83a00530ac542425c3bd"
	//	}
	//	eth.WriteTrustedCht(light.TrustedCht{
	//		Number: 55,
	//		Root:   gethcommon.HexToHash(root),
	//	})
	//	log.Info("Added trusted CHT for Rinkeby", "CHT", root)
	//}
}

// activateEthService configures and registers the eth.Ethereum service with a given node.
func activateEthService(stack *node.Node, config *params.NodeConfig) error {
	if !config.LightEthConfig.Enabled {
		log.Info("LES protocol is disabled")
		return nil
	}

	var genesis *core.Genesis
	if config.LightEthConfig.Genesis != "" {
		genesis = new(core.Genesis)
		if err := json.Unmarshal([]byte(config.LightEthConfig.Genesis), genesis); err != nil {
			return fmt.Errorf("invalid genesis spec: %v", err)
		}
	}

	ethConf := eth.DefaultConfig
	ethConf.Genesis = genesis
	ethConf.SyncMode = downloader.LightSync
	ethConf.NetworkId = config.NetworkID
	ethConf.DatabaseCache = config.LightEthConfig.DatabaseCache
	ethConf.MaxPeers = config.MaxPeers
	if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
		lightEth, err := les.New(ctx, &ethConf)
		if err == nil {
			updateCHT(lightEth, config)
		}
		return lightEth, err
	}); err != nil {
		return fmt.Errorf("%v: %v", ErrLightEthRegistrationFailure, err)
	}

	return nil
}

// activateShhService configures Whisper and adds it to the given node.
func activateShhService(stack *node.Node, config *params.NodeConfig) error {
	if !config.WhisperConfig.Enabled {
		log.Info("SHH protocol is disabled")
		return nil
	}
	serviceConstructor := func(*node.ServiceContext) (node.Service, error) {
		whisperConfig := config.WhisperConfig
		whisperService := whisper.New()

		// enable mail service
		if whisperConfig.MailServerNode {
			password, err := whisperConfig.ReadPasswordFile()
			if err != nil {
				return nil, err
			}

			var mailServer mailserver.WMailServer
			whisperService.RegisterServer(&mailServer)
			mailServer.Init(whisperService, whisperConfig.DataDir, string(password), whisperConfig.MinimumPoW)
		}

		// enable notification service
		if whisperConfig.NotificationServerNode {
			var notificationServer notifications.NotificationServer
			whisperService.RegisterNotificationServer(&notificationServer)

			notificationServer.Init(whisperService, whisperConfig)
		}

		return whisperService, nil
	}

	return stack.Register(serviceConstructor)
}

// makeIPCPath returns IPC-RPC filename
func makeIPCPath(config *params.NodeConfig) string {
	if !config.IPCEnabled {
		return ""
	}

	return path.Join(config.DataDir, config.IPCFile)
}

// makeWSHost returns WS-RPC Server host, given enabled/disabled flag
func makeWSHost(config *params.NodeConfig) string {
	if !config.WSEnabled {
		return ""
	}

	return config.WSHost
}

// makeBootstrapNodes returns default (hence bootstrap) list of peers
func makeBootstrapNodes() []*discover.Node {
	// on desktops params.TestnetBootnodes and params.MainBootnodes,
	// on mobile client we deliberately keep this list empty
	enodes := []string{}

	var bootstapNodes []*discover.Node
	for _, enode := range enodes {
		bootstapNodes = append(bootstapNodes, discover.MustParseNode(enode))
	}

	return bootstapNodes
}

// makeBootstrapNodesV5 returns default (hence bootstrap) list of peers
func makeBootstrapNodesV5() []*discv5.Node {
	// on desktops params.DiscoveryV5Bootnodes,
	// on mobile client we deliberately keep this list empty
	enodes := []string{}

	var bootstapNodes []*discv5.Node
	for _, enode := range enodes {
		bootstapNodes = append(bootstapNodes, discv5.MustParseNode(enode))
	}

	return bootstapNodes
}
<commit_msg>node: topic discovery boot nodes updated<commit_after>package node

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	gethcommon "github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/core"
	"github.com/ethereum/go-ethereum/crypto"
	"github.com/ethereum/go-ethereum/eth"
	"github.com/ethereum/go-ethereum/eth/downloader"
	"github.com/ethereum/go-ethereum/les"
	"github.com/ethereum/go-ethereum/light"
	"github.com/ethereum/go-ethereum/log"
	"github.com/ethereum/go-ethereum/node"
	"github.com/ethereum/go-ethereum/p2p"
	"github.com/ethereum/go-ethereum/p2p/discover"
	"github.com/ethereum/go-ethereum/p2p/discv5"
	"github.com/ethereum/go-ethereum/p2p/nat"
	gethparams "github.com/ethereum/go-ethereum/params"
	"github.com/ethereum/go-ethereum/whisper/mailserver"
	"github.com/ethereum/go-ethereum/whisper/notifications"
	whisper "github.com/ethereum/go-ethereum/whisper/whisperv5"
	"github.com/status-im/status-go/geth/common"
	"github.com/status-im/status-go/geth/params"
)

// node-related errors
var (
	ErrEthServiceRegistrationFailure     = errors.New("failed to register the Ethereum service")
	ErrWhisperServiceRegistrationFailure = errors.New("failed to register the Whisper service")
	ErrLightEthRegistrationFailure       = errors.New("failed to register the LES service")
	ErrNodeMakeFailure                   = errors.New("error creating p2p node")
	ErrNodeRunFailure                    = errors.New("error running p2p node")
	ErrNodeStartFailure                  = errors.New("error starting p2p node")
)

// MakeNode create a geth node entity
func MakeNode(config *params.NodeConfig) (*node.Node, error) {
	// make sure data directory exists
	if err := os.MkdirAll(filepath.Join(config.DataDir), os.ModePerm); err != nil {
		return nil, err
	}

	// make sure keys directory exists
	if err := os.MkdirAll(filepath.Join(config.KeyStoreDir), os.ModePerm); err != nil {
		return nil, err
	}

	// setup logging
	if _, err := common.SetupLogger(config); err != nil {
		return nil, err
	}

	// configure required node (should you need to update node's config, e.g. add bootstrap nodes, see node.Config)
	stackConfig := defaultEmbeddedNodeConfig(config)

	if len(config.NodeKeyFile) > 0 {
		log.Info("Loading private key file", "file", config.NodeKeyFile)
		pk, err := crypto.LoadECDSA(config.NodeKeyFile)
		if err != nil {
			log.Warn(fmt.Sprintf("Failed loading private key file '%s': %v", config.NodeKeyFile, err))
		}

		// override node's private key
		stackConfig.P2P.PrivateKey = pk
	}

	if len(config.NodeKeyFile) > 0 {
		log.Info("Loading private key file", "file", config.NodeKeyFile)
		pk, err := crypto.LoadECDSA(config.NodeKeyFile)
		if err != nil {
			log.Info("Failed loading private key file", "file", config.NodeKeyFile, "err", err)
		}

		// override node's private key
		stackConfig.P2P.PrivateKey = pk
	}

	stack, err := node.New(stackConfig)
	if err != nil {
		return nil, ErrNodeMakeFailure
	}

	// start Ethereum service
	if err := activateEthService(stack, config); err != nil {
		return nil, fmt.Errorf("%v: %v", ErrEthServiceRegistrationFailure, err)
	}

	// start Whisper service
	if err := activateShhService(stack, config); err != nil {
		return nil, fmt.Errorf("%v: %v", ErrWhisperServiceRegistrationFailure, err)
	}

	return stack, nil
}

// defaultEmbeddedNodeConfig returns default stack configuration for mobile client node
func defaultEmbeddedNodeConfig(config *params.NodeConfig) *node.Config {
	return &node.Config{
		DataDir:           config.DataDir,
		KeyStoreDir:       config.KeyStoreDir,
		UseLightweightKDF: true,
		NoUSB:             true,
		Name:              config.Name,
		Version:           config.Version,
		P2P: p2p.Config{
			NoDiscovery:      true,
			DiscoveryV5:      true,
			DiscoveryV5Addr:  ":0",
			BootstrapNodes:   makeBootstrapNodes(),
			BootstrapNodesV5: makeBootstrapNodesV5(),
			ListenAddr:       ":0",
			NAT:              nat.Any(),
			MaxPeers:         config.MaxPeers,
			MaxPendingPeers:  config.MaxPendingPeers,
		},
		IPCPath:     makeIPCPath(config),
		HTTPHost:    config.HTTPHost,
		HTTPPort:    config.HTTPPort,
		HTTPCors:    []string{"*"},
		HTTPModules: strings.Split(config.APIModules, ","),
		WSHost:      makeWSHost(config),
		WSPort:      config.WSPort,
		WSOrigins:   []string{"*"},
		WSModules:   strings.Split(config.APIModules, ","),
	}
}

// updateCHT changes trusted canonical hash trie root
func updateCHT(eth *les.LightEthereum, config *params.NodeConfig) {
	bc := eth.BlockChain()

	// TODO: Remove this thing as this is an ugly hack.
	// Once CHT sync sub-protocol is working in LES, we will rely on it, as it provides
	// decentralized solution. For now, in order to avoid forcing users to long sync times
	// we use central static resource
	type MsgCHTRoot struct {
		GenesisHash string `json:"net"`
		Number      uint64 `json:"number"`
		Prod        string `json:"prod"`
		Dev         string `json:"dev"`
	}
	loadCHTLists := func() ([]MsgCHTRoot, error) {
		url := config.LightEthConfig.CHTRootConfigURL + "?u=" + strconv.Itoa(int(time.Now().Unix()))
		client := &http.Client{Timeout: 5 * time.Second}
		r, err := client.Get(url)
		if err != nil {
			return nil, err
		}
		defer r.Body.Close()

		var roots []MsgCHTRoot
		err = json.NewDecoder(r.Body).Decode(&roots)
		if err != nil {
			return nil, err
		}

		return roots, nil
	}
	if roots, err := loadCHTLists(); err == nil {
		for _, root := range roots {
			if bc.Genesis().Hash().Hex() == root.GenesisHash {
				log.Info("Loaded root", "root", root)
				if root.Number == 0 {
					continue
				}

				chtRoot := root.Prod
				if config.DevMode {
					chtRoot = root.Dev
				}
				eth.WriteTrustedCht(light.TrustedCht{
					Number: root.Number,
					Root:   gethcommon.HexToHash(chtRoot),
				})
				log.Info("Loaded CHT from net", "CHT", chtRoot, "number", root.Number, "dev", config.DevMode)
				return
			}
		}
	}

	// resort to manually updated
	log.Info("Loading CHT from net failed, setting manually")
	if bc.Genesis().Hash() == params.MainNetGenesisHash {
		eth.WriteTrustedCht(light.TrustedCht{
			Number: 805,
			Root:   gethcommon.HexToHash("85e4286fe0a730390245c49de8476977afdae0eb5530b277f62a52b12313d50f"),
		})
		log.Info("Added trusted CHT for mainnet")
	}

	if bc.Genesis().Hash() == params.RopstenNetGenesisHash {
		root := "fa851b5252cc48ab55f375833b0344cc5c7cacea69be7e2a57976c38d3bb3aef"
		if config.DevMode {
			root = "f2f862314509b22a773eedaaa7fa6452474eb71a3b72525a97dbf5060cbea88f"
		}
		eth.WriteTrustedCht(light.TrustedCht{
			Number: 239,
			Root:   gethcommon.HexToHash(root),
		})
		log.Info("Added trusted CHT for Ropsten", "CHT", root)
	}

	//if bc.Genesis().Hash() == params.RinkebyNetGenesisHash {
	//	root := "0xb100882d00a09292f15e712707649b24d019a47a509e83a00530ac542425c3bd"
	//	if config.DevMode {
	//		root = "0xb100882d00a09292f15e712707649b24d019a47a509e83a00530ac542425c3bd"
	//	}
	//	eth.WriteTrustedCht(light.TrustedCht{
	//		Number: 55,
	//		Root:   gethcommon.HexToHash(root),
	//	})
	//	log.Info("Added trusted CHT for Rinkeby", "CHT", root)
	//}
}

// activateEthService configures and registers the eth.Ethereum service with a given node.
func activateEthService(stack *node.Node, config *params.NodeConfig) error {
	if !config.LightEthConfig.Enabled {
		log.Info("LES protocol is disabled")
		return nil
	}

	var genesis *core.Genesis
	if config.LightEthConfig.Genesis != "" {
		genesis = new(core.Genesis)
		if err := json.Unmarshal([]byte(config.LightEthConfig.Genesis), genesis); err != nil {
			return fmt.Errorf("invalid genesis spec: %v", err)
		}
	}

	ethConf := eth.DefaultConfig
	ethConf.Genesis = genesis
	ethConf.SyncMode = downloader.LightSync
	ethConf.NetworkId = config.NetworkID
	ethConf.DatabaseCache = config.LightEthConfig.DatabaseCache
	ethConf.MaxPeers = config.MaxPeers
	if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
		lightEth, err := les.New(ctx, &ethConf)
		if err == nil {
			updateCHT(lightEth, config)
		}
		return lightEth, err
	}); err != nil {
		return fmt.Errorf("%v: %v", ErrLightEthRegistrationFailure, err)
	}

	return nil
}

// activateShhService configures Whisper and adds it to the given node.
func activateShhService(stack *node.Node, config *params.NodeConfig) error {
	if !config.WhisperConfig.Enabled {
		log.Info("SHH protocol is disabled")
		return nil
	}
	serviceConstructor := func(*node.ServiceContext) (node.Service, error) {
		whisperConfig := config.WhisperConfig
		whisperService := whisper.New()

		// enable mail service
		if whisperConfig.MailServerNode {
			password, err := whisperConfig.ReadPasswordFile()
			if err != nil {
				return nil, err
			}

			var mailServer mailserver.WMailServer
			whisperService.RegisterServer(&mailServer)
			mailServer.Init(whisperService, whisperConfig.DataDir, string(password), whisperConfig.MinimumPoW)
		}

		// enable notification service
		if whisperConfig.NotificationServerNode {
			var notificationServer notifications.NotificationServer
			whisperService.RegisterNotificationServer(&notificationServer)

			notificationServer.Init(whisperService, whisperConfig)
		}

		return whisperService, nil
	}

	return stack.Register(serviceConstructor)
}

// makeIPCPath returns IPC-RPC filename
func makeIPCPath(config *params.NodeConfig) string {
	if !config.IPCEnabled {
		return ""
	}

	return path.Join(config.DataDir, config.IPCFile)
}

// makeWSHost returns WS-RPC Server host, given enabled/disabled flag
func makeWSHost(config *params.NodeConfig) string {
	if !config.WSEnabled {
		return ""
	}

	return config.WSHost
}

// makeBootstrapNodes returns default (hence bootstrap) list of peers
func makeBootstrapNodes() []*discover.Node {
	// on desktops params.TestnetBootnodes and params.MainBootnodes,
	// on mobile client we deliberately keep this list empty
	enodes := []string{}

	var bootstapNodes []*discover.Node
	for _, enode := range enodes {
		bootstapNodes = append(bootstapNodes, discover.MustParseNode(enode))
	}

	return bootstapNodes
}

// makeBootstrapNodesV5 returns default (hence bootstrap) list of peers
func makeBootstrapNodesV5() []*discv5.Node {
	enodes := gethparams.DiscoveryV5Bootnodes

	var bootstapNodes []*discv5.Node
	for _, enode := range enodes {
		bootstapNodes = append(bootstapNodes, discv5.MustParseNode(enode))
	}

	return bootstapNodes
}
<|endoftext|>


<commit_before>package ggk

import (
	"fmt"
)

// Alpha types
// Describe how to interpret the alpha compoent of a pixel.
type AlphaType int

const (
	AlphaType_Unknown AlphaType = iota

	// All pixels are stored as opaque. This differs slightly from kIgnore in
	// that kOpaque has correct "Opaque" values stored in the pixels, while
	// kIgnore may not, but in both cases the caller should treat the pixels
	// as opaque.
	AlphaType_Opaque

	// All pixels have their alpha premultiplied in their color components.
	// This is the natural format for the rendering target pixels.
	AlphaType_Premul

	// All pixels have their color components stroed without any regard to the
	// alpha. e.g. this is the default configuration for PNG images.
	//
	// This alpha-type is ONLY supported for input images. Rendering cannot
	// generate this on output.
	AlphaType_Unpremul

	AlphaType_LastEnum = AlphaType_Unpremul
)

func (at AlphaType) IsOpaque() bool {
	return at == AlphaType_Opaque
}

func (at AlphaType) IsValid() bool {
	return at >= 0 && at <= AlphaType_LastEnum
}

// Color types
// Describes how to interpret the components of a pixel.
// ColorTypeN32 is an alias for whichever 32bit ARGB format is the "native"
// form for blitters. Use this if you don't hava a swizzle preference
// for 32bit pixels.
type ColorType int

const (
	ColorType_Unknown ColorType = iota
	ColorType_Alpha8
	ColorType_RGB565
	ColorType_ARGB4444
	ColorType_RGBA8888
	ColorType_BGRA8888
	ColorType_Index8
	ColorType_Gray8

	ColorTypeLastEnum = ColorType_Gray8
)

func (ct ColorType) BytesPerPixel() int {
	// TODO: const?
	var bytesPerPixel = []int{
		0, // Unknown
		1, // Alpha8
		2, // RGB565
		2, // ARGB4444
		4, // RGBA8888
		4, // BGRA8888
		1, // Index8
		1, // Gray8
	}

	if ct < 0 || int(ct) >= len(bytesPerPixel) {
		return 0
	}

	return bytesPerPixel[ct]
}

func (ct ColorType) MinRowBytes(width int) int {
	return width * ct.BytesPerPixel()
}

func (ct ColorType) IsVaild() bool {
	return ct >= 0 && ct <= ColorTypeLastEnum
}

func (ct ColorType) ComputeOffset(x, y int, rowBytes uint) uint {
	var shift uint = 0

	switch ct.BytesPerPixel() {
	case 4:
		shift = 2
	case 2:
		shift = 1
	case 1:
		shift = 0
	default:
		return 0
	}

	return uint(y)*rowBytes + uint(x)<<shift
}

// Return true if alphaType is supported by colorType. If there is a canonical
// alphaType for this colorType, return it in canonical.
func (ct ColorType) ValidateAlphaType(alphaType AlphaType) AlphaType {
	return 0
}

func ColorTypeN32() ColorType {
	return ColorType_RGBA8888
}

// YUV color space
// Describes the color space a YUV pixel
type YUVColorSpace int

const (
	// Standard JPEG color space.
	YUVColorSpace_JPEG YUVColorSpace = iota
	// SDTV standard Rec. 601 color space. Uses "studio swing" [16, 245] color
	// range. See http://en.wikipedia.org/wiki/Rec._601 for details.
	YUVColorSpace_Rec601
	// HDTV standard Rec. 709 color space. Uses "studio swing" [16, 235] color
	// range. See http://en.wikipedia.org/wiki/Rec._709 for details.
	YUVColorSpace_Rec709

	YUVColorSpace_LastEnum = YUVColorSpace_Rec709
)

// Color profile type
type ColorProfileType int

const (
	ColorProfileType_Linear ColorProfileType = iota
	ColorProfileType_SRGB
	ColorProfileType_LastEnum = ColorProfileType_SRGB
)

func (pt ColorProfileType) IsValid() bool {
	return pt >= 0 && pt <= ColorProfileType_LastEnum
}

// Describe an image's dimensions and pixel type.
// Used for both src images and render-targets (surfaces).
type ImageInfo struct {
	width  int
	height int

	colorType   ColorType
	alphaType   AlphaType
	profileType ColorProfileType
}

func NewImageInfo(width, height int, colorType ColorType, alphaType AlphaType, profileType ColorProfileType) *ImageInfo {
	var imageInfo = &ImageInfo{
		width:       width,
		height:      height,
		colorType:   colorType,
		alphaType:   alphaType,
		profileType: profileType,
	}

	return imageInfo
}

func NewImageInfoN32(width, height int, alphaType AlphaType, profileType ColorProfileType) *ImageInfo {
	return NewImageInfo(width, height, ColorTypeN32(), alphaType, profileType)
}

func NewImageInfoN32Premul(width, height int, profileType ColorProfileType) *ImageInfo {
	return NewImageInfo(width, height, ColorTypeN32(), AlphaType_Premul, profileType)
}

func NewImageInfoA8(width, height int) *ImageInfo {
	return NewImageInfo(width, height, ColorType_Alpha8, AlphaType_Premul, ColorProfileType_Linear)
}

func NewImageInfoUnknown(width, height int) *ImageInfo {
	return NewImageInfo(width, height, ColorType_Unknown, AlphaType_Unknown, ColorProfileType_Linear)
}

func (imageInfo *ImageInfo) Width() int {
	return imageInfo.width
}

func (imageInfo *ImageInfo) Height() int {
	return imageInfo.height
}

func (imageInfo *ImageInfo) ColorType() ColorType {
	return imageInfo.colorType
}

func (imageInfo *ImageInfo) AlphaType() AlphaType {
	return imageInfo.alphaType
}

func (imageInfo *ImageInfo) ProfileType() ColorProfileType {
	return imageInfo.profileType
}

func (imageInfo *ImageInfo) IsEmpty() bool {
	return imageInfo.width <= 0 || imageInfo.height <= 0
}

func (imageInfo *ImageInfo) IsOpaque() bool {
	return imageInfo.alphaType.IsOpaque()
}

func (imageInfo *ImageInfo) IsLinear() bool {
	return imageInfo.profileType == ColorProfileType_Linear
}

func (imageInfo *ImageInfo) IsSRGB() bool {
	return imageInfo.profileType == ColorProfileType_SRGB
}

func (imageInfo *ImageInfo) ComputeOffset(x, y int, rowBytes uint) (uint, error) {
	if uint(x) >= uint(imageInfo.width) || uint(y) >= uint(imageInfo.height) {
		return 0, fmt.Errorf("OOR: ggk.ImageInfo(0x%x).ComputeOffset(%d, %d, %d)", imageInfo, x, y, rowBytes)
	}

	return imageInfo.colorType.ComputeOffset(x, y, rowBytes), nil
}

func (imageInfo *ImageInfo) Equal(other *ImageInfo) bool {
	var equal = false

	equal = (imageInfo.colorType == other.colorType)
	equal = equal && (imageInfo.alphaType == other.alphaType)
	equal = equal && (imageInfo.profileType == other.profileType)
	equal = equal && (imageInfo.width == other.width)
	equal = equal && (imageInfo.height == other.height)

	return equal
}

func (imageInfo *ImageInfo) BytesPerPixel() int {
	return imageInfo.colorType.BytesPerPixel()
}

func (imageInfo *ImageInfo) MinRowBytes64() uint64 {
	var minRowBytes64 uint64 = uint64(imageInfo.width) * uint64(imageInfo.BytesPerPixel())
	return minRowBytes64
}

func (imageInfo *ImageInfo) MinRowBytes() uint {
	return uint(imageInfo.MinRowBytes64())
}

func (imageInfo *ImageInfo) ValidRowBytes(rowBytes uint) bool {
	return uint64(rowBytes) >= imageInfo.MinRowBytes64()
}

func (imageInfo *ImageInfo) SafeSize64(rowBytes uint) uint64 {
	if imageInfo.height == 0 {
		return 0
	}

	return uint64(imageInfo.height-1)*uint64(rowBytes) + uint64(imageInfo.width*imageInfo.BytesPerPixel())
}

func (imageInfo *ImageInfo) SafeSize(rowBytes uint) uint {
	var size uint64 = imageInfo.SafeSize64(rowBytes)
	if size != uint64(uint(size)) {
		return 0
	}
	return uint(size)
}
<commit_msg>rule to 80 column<commit_after>package ggk

import (
	"fmt"
)

// Alpha types
// Describe how to interpret the alpha compoent of a pixel.
type AlphaType int

const (
	AlphaType_Unknown AlphaType = iota

	// All pixels are stored as opaque. This differs slightly from kIgnore in
	// that kOpaque has correct "Opaque" values stored in the pixels, while
	// kIgnore may not, but in both cases the caller should treat the pixels
	// as opaque.
	AlphaType_Opaque

	// All pixels have their alpha premultiplied in their color components.
	// This is the natural format for the rendering target pixels.
	AlphaType_Premul

	// All pixels have their color components stroed without any regard to the
	// alpha. e.g. this is the default configuration for PNG images.
	//
	// This alpha-type is ONLY supported for input images. Rendering cannot
	// generate this on output.
	AlphaType_Unpremul

	AlphaType_LastEnum = AlphaType_Unpremul
)

func (at AlphaType) IsOpaque() bool {
	return at == AlphaType_Opaque
}

func (at AlphaType) IsValid() bool {
	return at >= 0 && at <= AlphaType_LastEnum
}

// Color types
// Describes how to interpret the components of a pixel.
// ColorTypeN32 is an alias for whichever 32bit ARGB format is the "native"
// form for blitters. Use this if you don't hava a swizzle preference
// for 32bit pixels.
type ColorType int

const (
	ColorType_Unknown ColorType = iota
	ColorType_Alpha8
	ColorType_RGB565
	ColorType_ARGB4444
	ColorType_RGBA8888
	ColorType_BGRA8888
	ColorType_Index8
	ColorType_Gray8

	ColorTypeLastEnum = ColorType_Gray8
)

func (ct ColorType) BytesPerPixel() int {
	// TODO: const?
	var bytesPerPixel = []int{
		0, // Unknown
		1, // Alpha8
		2, // RGB565
		2, // ARGB4444
		4, // RGBA8888
		4, // BGRA8888
		1, // Index8
		1, // Gray8
	}

	if ct < 0 || int(ct) >= len(bytesPerPixel) {
		return 0
	}

	return bytesPerPixel[ct]
}

func (ct ColorType) MinRowBytes(width int) int {
	return width * ct.BytesPerPixel()
}

func (ct ColorType) IsVaild() bool {
	return ct >= 0 && ct <= ColorTypeLastEnum
}

func (ct ColorType) ComputeOffset(x, y int, rowBytes uint) uint {
	var shift uint = 0

	switch ct.BytesPerPixel() {
	case 4:
		shift = 2
	case 2:
		shift = 1
	case 1:
		shift = 0
	default:
		return 0
	}

	return uint(y)*rowBytes + uint(x)<<shift
}

// Return true if alphaType is supported by colorType. If there is a canonical
// alphaType for this colorType, return it in canonical.
func (ct ColorType) ValidateAlphaType(alphaType AlphaType) AlphaType {
	return 0
}

func ColorTypeN32() ColorType {
	return ColorType_RGBA8888
}

// YUV color space
// Describes the color space a YUV pixel
type YUVColorSpace int

const (
	// Standard JPEG color space.
	YUVColorSpace_JPEG YUVColorSpace = iota
	// SDTV standard Rec. 601 color space. Uses "studio swing" [16, 245] color
	// range. See http://en.wikipedia.org/wiki/Rec._601 for details.
	YUVColorSpace_Rec601
	// HDTV standard Rec. 709 color space. Uses "studio swing" [16, 235] color
	// range. See http://en.wikipedia.org/wiki/Rec._709 for details.
	YUVColorSpace_Rec709

	YUVColorSpace_LastEnum = YUVColorSpace_Rec709
)

// Color profile type
type ColorProfileType int

const (
	ColorProfileType_Linear ColorProfileType = iota
	ColorProfileType_SRGB
	ColorProfileType_LastEnum = ColorProfileType_SRGB
)

func (pt ColorProfileType) IsValid() bool {
	return pt >= 0 && pt <= ColorProfileType_LastEnum
}

// Describe an image's dimensions and pixel type.
// Used for both src images and render-targets (surfaces).
type ImageInfo struct {
	width  int
	height int

	colorType   ColorType
	alphaType   AlphaType
	profileType ColorProfileType
}

func NewImageInfo(width, height int, colorType ColorType, alphaType AlphaType,
	profileType ColorProfileType) *ImageInfo {
	var imageInfo = &ImageInfo{
		width:       width,
		height:      height,
		colorType:   colorType,
		alphaType:   alphaType,
		profileType: profileType,
	}

	return imageInfo
}

func NewImageInfoN32(width, height int, alphaType AlphaType, profileType
	ColorProfileType) *ImageInfo {
	return NewImageInfo(width, height, ColorTypeN32(), alphaType, profileType)
}

func NewImageInfoN32Premul(width, height int, profileType ColorProfileType) *ImageInfo {
	return NewImageInfo(width, height, ColorTypeN32(), AlphaType_Premul,
		profileType)
}

func NewImageInfoA8(width, height int) *ImageInfo {
	return NewImageInfo(width, height, ColorType_Alpha8, AlphaType_Premul,
		ColorProfileType_Linear)
}

func NewImageInfoUnknown(width, height int) *ImageInfo {
	return NewImageInfo(width, height, ColorType_Unknown, AlphaType_Unknown,
		ColorProfileType_Linear)
}

func (imageInfo *ImageInfo) Width() int {
	return imageInfo.width
}

func (imageInfo *ImageInfo) Height() int {
	return imageInfo.height
}

func (imageInfo *ImageInfo) ColorType() ColorType {
	return imageInfo.colorType
}

func (imageInfo *ImageInfo) AlphaType() AlphaType {
	return imageInfo.alphaType
}

func (imageInfo *ImageInfo) ProfileType() ColorProfileType {
	return imageInfo.profileType
}

func (imageInfo *ImageInfo) IsEmpty() bool {
	return imageInfo.width <= 0 || imageInfo.height <= 0
}

func (imageInfo *ImageInfo) IsOpaque() bool {
	return imageInfo.alphaType.IsOpaque()
}

func (imageInfo *ImageInfo) IsLinear() bool {
	return imageInfo.profileType == ColorProfileType_Linear
}

func (imageInfo *ImageInfo) IsSRGB() bool {
	return imageInfo.profileType == ColorProfileType_SRGB
}

func (imageInfo *ImageInfo) ComputeOffset(x, y int, rowBytes uint) (uint, error) {
	if uint(x) >= uint(imageInfo.width) || uint(y) >= uint(imageInfo.height) {
		return 0, fmt.Errorf("OOR: ggk.ImageInfo(0x%x).ComputeOffset(%d, %d, %d)",
			imageInfo, x, y, rowBytes)
	}

	return imageInfo.colorType.ComputeOffset(x, y, rowBytes), nil
}

func (imageInfo *ImageInfo) Equal(other *ImageInfo) bool {
	var equal = false

	equal = (imageInfo.colorType == other.colorType)
	equal = equal && (imageInfo.alphaType == other.alphaType)
	equal = equal && (imageInfo.profileType == other.profileType)
	equal = equal && (imageInfo.width == other.width)
	equal = equal && (imageInfo.height == other.height)

	return equal
}

func (imageInfo *ImageInfo) BytesPerPixel() int {
	return imageInfo.colorType.BytesPerPixel()
}

func (imageInfo *ImageInfo) MinRowBytes64() uint64 {
	var minRowBytes64 uint64 =
		uint64(imageInfo.width) * uint64(imageInfo.BytesPerPixel())
	return minRowBytes64
}

func (imageInfo *ImageInfo) MinRowBytes() uint {
	return uint(imageInfo.MinRowBytes64())
}

func (imageInfo *ImageInfo) ValidRowBytes(rowBytes uint) bool {
	return uint64(rowBytes) >= imageInfo.MinRowBytes64()
}

func (imageInfo *ImageInfo) SafeSize64(rowBytes uint) uint64 {
	if imageInfo.height == 0 {
		return 0
	}

	return uint64(imageInfo.height-1)*uint64(rowBytes) +
		uint64(imageInfo.width*imageInfo.BytesPerPixel())
}

func (imageInfo *ImageInfo) SafeSize(rowBytes uint) uint {
	var size uint64 = imageInfo.SafeSize64(rowBytes)
	if size != uint64(uint(size)) {
		return 0
	}
	return uint(size)
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	kube_flag "k8s.io/apiserver/pkg/util/flag"
	"k8s.io/contrib/cluster-autoscaler/config"
	"k8s.io/contrib/cluster-autoscaler/config/dynamic"
	"k8s.io/contrib/cluster-autoscaler/core"
	"k8s.io/contrib/cluster-autoscaler/estimator"
	"k8s.io/contrib/cluster-autoscaler/expander"
	"k8s.io/contrib/cluster-autoscaler/metrics"
	"k8s.io/contrib/cluster-autoscaler/simulator"
	kube_util "k8s.io/contrib/cluster-autoscaler/utils/kubernetes"
	kube_client "k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
	kube_leaderelection "k8s.io/kubernetes/pkg/client/leaderelection"
	"k8s.io/kubernetes/pkg/client/leaderelection/resourcelock"

	"github.com/golang/glog"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/spf13/pflag"
)

// MultiStringFlag is a flag for passing multiple parameters using same flag
type MultiStringFlag []string

// String returns string representation of the node groups.
func (flag *MultiStringFlag) String() string {
	return "[" + strings.Join(*flag, " ") + "]"
}

// Set adds a new configuration.
func (flag *MultiStringFlag) Set(value string) error {
	*flag = append(*flag, value)
	return nil
}

var (
	nodeGroupsFlag          MultiStringFlag
	address                 = flag.String("address", ":8085", "The address to expose prometheus metrics.")
	kubernetes              = flag.String("kubernetes", "", "Kuberentes master location. Leave blank for default")
	cloudConfig             = flag.String("cloud-config", "", "The path to the cloud provider configuration file.  Empty string for no configuration file.")
	configMapName           = flag.String("configmap", "", "The name of the ConfigMap containing settings used for dynamic reconfiguration. Empty string for no ConfigMap.")
	namespace               = flag.String("namespace", "kube-system", "Namespace in which cluster-autoscaler run. If a --configmap flag is also provided, ensure that the configmap exists in this namespace before CA runs.")
	verifyUnschedulablePods = flag.Bool("verify-unschedulable-pods", true,
		"If enabled CA will ensure that each pod marked by Scheduler as unschedulable actually can't be scheduled on any node."+
			"This prevents from adding unnecessary nodes in situation when CA and Scheduler have different configuration.")
	scaleDownEnabled = flag.Bool("scale-down-enabled", true, "Should CA scale down the cluster")
	scaleDownDelay   = flag.Duration("scale-down-delay", 10*time.Minute,
		"Duration from the last scale up to the time when CA starts to check scale down options")
	scaleDownUnneededTime = flag.Duration("scale-down-unneeded-time", 10*time.Minute,
		"How long a node should be unneeded before it is eligible for scale down")
	scaleDownUnreadyTime = flag.Duration("scale-down-unready-time", 20*time.Minute,
		"How long an unready node should be unneeded before it is eligible for scale down")
	scaleDownUtilizationThreshold = flag.Float64("scale-down-utilization-threshold", 0.5,
		"Node utilization level, defined as sum of requested resources divided by capacity, below which a node can be considered for scale down")
	scaleDownTrialInterval = flag.Duration("scale-down-trial-interval", 1*time.Minute,
		"How often scale down possiblity is check")
	scanInterval                = flag.Duration("scan-interval", 10*time.Second, "How often cluster is reevaluated for scale up or down")
	maxNodesTotal               = flag.Int("max-nodes-total", 0, "Maximum number of nodes in all node groups. Cluster autoscaler will not grow the cluster beyond this number.")
	cloudProviderFlag           = flag.String("cloud-provider", "gce", "Cloud provider type. Allowed values: gce, aws, azure")
	maxEmptyBulkDeleteFlag      = flag.Int("max-empty-bulk-delete", 10, "Maximum number of empty nodes that can be deleted at the same time.")
	maxGratefulTerminationFlag  = flag.Int("max-grateful-termination-sec", 60, "Maximum number of seconds CA waints for pod termination when trying to scale down a node.")
	maxTotalUnreadyPercentage   = flag.Float64("max-total-unready-percentage", 33, "Maximum percentage of unready nodes after which CA halts operations")
	okTotalUnreadyCount         = flag.Int("ok-total-unready-count", 3, "Number of allowed unready nodes, irrespective of max-total-unready-percentage")
	maxNodeProvisionTime        = flag.Duration("max-node-provision-time", 15*time.Minute, "Maximum time CA waits for node to be provisioned")
	unregisteredNodeRemovalTime = flag.Duration("unregistered-node-removal-time", 15*time.Minute, "Time that CA waits before removing nodes that are not registered in Kubernetes")

	estimatorFlag = flag.String("estimator", estimator.BinpackingEstimatorName,
		"Type of resource estimator to be used in scale up. Available values: ["+strings.Join(estimator.AvailableEstimators, ",")+"]")

	expanderFlag = flag.String("expander", expander.RandomExpanderName,
		"Type of node group expander to be used in scale up. Available values: ["+strings.Join(expander.AvailableExpanders, ",")+"]")

	writeStatusConfigMapFlag = flag.Bool("write-status-configmap", true, "Should CA write status information to a configmap")
)

func createAutoscalerOptions() core.AutoscalerOptions {
	autoscalingOpts := core.AutoscalingOptions{
		CloudConfig:                   *cloudConfig,
		CloudProviderName:             *cloudProviderFlag,
		MaxTotalUnreadyPercentage:     *maxTotalUnreadyPercentage,
		OkTotalUnreadyCount:           *okTotalUnreadyCount,
		EstimatorName:                 *estimatorFlag,
		ExpanderName:                  *expanderFlag,
		MaxEmptyBulkDelete:            *maxEmptyBulkDeleteFlag,
		MaxGratefulTerminationSec:     *maxGratefulTerminationFlag,
		MaxNodeProvisionTime:          *maxNodeProvisionTime,
		MaxNodesTotal:                 *maxNodesTotal,
		UnregisteredNodeRemovalTime:   *unregisteredNodeRemovalTime,
		ScaleDownDelay:                *scaleDownDelay,
		ScaleDownEnabled:              *scaleDownEnabled,
		ScaleDownTrialInterval:        *scaleDownTrialInterval,
		ScaleDownUnneededTime:         *scaleDownUnneededTime,
		ScaleDownUnreadyTime:          *scaleDownUnreadyTime,
		ScaleDownUtilizationThreshold: *scaleDownUtilizationThreshold,
		VerifyUnschedulablePods:       *verifyUnschedulablePods,
		WriteStatusConfigMap:          *writeStatusConfigMapFlag,
	}

	configFetcherOpts := dynamic.ConfigFetcherOptions{
		ConfigMapName: *configMapName,
		Namespace:     *namespace,
	}

	return core.AutoscalerOptions{
		AutoscalingOptions:   autoscalingOpts,
		ConfigFetcherOptions: configFetcherOpts,
	}
}

func createKubeClient() kube_client.Interface {
	url, err := url.Parse(*kubernetes)
	if err != nil {
		glog.Fatalf("Failed to parse Kuberentes url: %v", err)
	}

	kubeConfig, err := config.GetKubeClientConfig(url)
	if err != nil {
		glog.Fatalf("Failed to build Kuberentes client configuration: %v", err)
	}

	return kube_client.NewForConfigOrDie(kubeConfig)
}

func registerSignalHandlers(autoscaler core.Autoscaler) {
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, os.Interrupt, os.Kill, syscall.SIGTERM, syscall.SIGQUIT)
	glog.Info("Registered cleanup signal handler")

	go func() {
		<-sigs
		glog.Info("Receieved signal, attempting cleanup")
		autoscaler.ExitCleanUp()
		glog.Info("Cleaned up, exiting...")
		glog.Flush()
		os.Exit(0)
	}()
}

// In order to meet interface criteria for LeaderElectionConfig we need to
// take stop channel as an argument. However, since we are committing a suicide
// after loosing mastership we can safely ignore it.
func run(_ <-chan struct{}) {
	kubeClient := createKubeClient()
	kubeEventRecorder := kube_util.CreateEventRecorder(kubeClient)
	opts := createAutoscalerOptions()
	predicateChecker, err := simulator.NewPredicateChecker(kubeClient)
	if err != nil {
		glog.Fatalf("Failed to create predicate checker: %v", err)
	}
	stopChannel := make(chan struct{})
	listerRegistry := kube_util.NewListerRegistryWithDefaultListers(kubeClient, stopChannel)
	autoscaler := core.NewAutoscaler(opts, predicateChecker, kubeClient, kubeEventRecorder, listerRegistry)

	autoscaler.CleanUp()
	registerSignalHandlers(autoscaler)

	for {
		select {
		case <-time.After(*scanInterval):
			{
				loopStart := time.Now()
				metrics.UpdateLastTime("main")

				autoscaler.RunOnce(loopStart)

				metrics.UpdateDuration("main", loopStart)
			}
		}
	}
}

func main() {
	leaderElection := kube_leaderelection.DefaultLeaderElectionConfiguration()
	leaderElection.LeaderElect = true

	kube_leaderelection.BindFlags(&leaderElection, pflag.CommandLine)
	flag.Var(&nodeGroupsFlag, "nodes", "sets min,max size and other configuration data for a node group in a format accepted by cloud provider."+
		"Can be used multiple times. Format: <min>:<max>:<other...>")
	kube_flag.InitFlags()

	glog.Infof("Cluster Autoscaler %s", ClusterAutoscalerVersion)

	correctEstimator := false
	for _, availableEstimator := range estimator.AvailableEstimators {
		if *estimatorFlag == availableEstimator {
			correctEstimator = true
		}
	}
	if !correctEstimator {
		glog.Fatalf("Unrecognized estimator: %v", *estimatorFlag)
	}

	go func() {
		http.Handle("/metrics", prometheus.Handler())
		err := http.ListenAndServe(*address, nil)
		glog.Fatalf("Failed to start metrics: %v", err)
	}()

	if !leaderElection.LeaderElect {
		run(nil)
	} else {
		id, err := os.Hostname()
		if err != nil {
			glog.Fatalf("Unable to get hostname: %v", err)
		}

		kubeClient := createKubeClient()

		// Validate that the client is ok.
		_, err = kubeClient.Core().Nodes().List(metav1.ListOptions{})
		if err != nil {
			glog.Fatalf("Failed to get nodes from apiserver: %v", err)
		}

		kube_leaderelection.RunOrDie(kube_leaderelection.LeaderElectionConfig{
			Lock: &resourcelock.EndpointsLock{
				EndpointsMeta: metav1.ObjectMeta{
					Namespace: *namespace,
					Name:      "cluster-autoscaler",
				},
				Client: kubeClient,
				LockConfig: resourcelock.ResourceLockConfig{
					Identity:      id,
					EventRecorder: kube_util.CreateEventRecorder(kubeClient),
				},
			},
			LeaseDuration: leaderElection.LeaseDuration.Duration,
			RenewDeadline: leaderElection.RenewDeadline.Duration,
			RetryPeriod:   leaderElection.RetryPeriod.Duration,
			Callbacks: kube_leaderelection.LeaderCallbacks{
				OnStartedLeading: run,
				OnStoppedLeading: func() {
					glog.Fatalf("lost master")
				},
			},
		})
	}
}
<commit_msg>Cluster-Autoscaler: fix ignoring node groups config<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	kube_flag "k8s.io/apiserver/pkg/util/flag"
	"k8s.io/contrib/cluster-autoscaler/config"
	"k8s.io/contrib/cluster-autoscaler/config/dynamic"
	"k8s.io/contrib/cluster-autoscaler/core"
	"k8s.io/contrib/cluster-autoscaler/estimator"
	"k8s.io/contrib/cluster-autoscaler/expander"
	"k8s.io/contrib/cluster-autoscaler/metrics"
	"k8s.io/contrib/cluster-autoscaler/simulator"
	kube_util "k8s.io/contrib/cluster-autoscaler/utils/kubernetes"
	kube_client "k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
	kube_leaderelection "k8s.io/kubernetes/pkg/client/leaderelection"
	"k8s.io/kubernetes/pkg/client/leaderelection/resourcelock"

	"github.com/golang/glog"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/spf13/pflag"
)

// MultiStringFlag is a flag for passing multiple parameters using same flag
type MultiStringFlag []string

// String returns string representation of the node groups.
func (flag *MultiStringFlag) String() string {
	return "[" + strings.Join(*flag, " ") + "]"
}

// Set adds a new configuration.
func (flag *MultiStringFlag) Set(value string) error {
	*flag = append(*flag, value)
	return nil
}

var (
	nodeGroupsFlag          MultiStringFlag
	address                 = flag.String("address", ":8085", "The address to expose prometheus metrics.")
	kubernetes              = flag.String("kubernetes", "", "Kuberentes master location. Leave blank for default")
	cloudConfig             = flag.String("cloud-config", "", "The path to the cloud provider configuration file.  Empty string for no configuration file.")
	configMapName           = flag.String("configmap", "", "The name of the ConfigMap containing settings used for dynamic reconfiguration. Empty string for no ConfigMap.")
	namespace               = flag.String("namespace", "kube-system", "Namespace in which cluster-autoscaler run. If a --configmap flag is also provided, ensure that the configmap exists in this namespace before CA runs.")
	verifyUnschedulablePods = flag.Bool("verify-unschedulable-pods", true,
		"If enabled CA will ensure that each pod marked by Scheduler as unschedulable actually can't be scheduled on any node."+
			"This prevents from adding unnecessary nodes in situation when CA and Scheduler have different configuration.")
	scaleDownEnabled = flag.Bool("scale-down-enabled", true, "Should CA scale down the cluster")
	scaleDownDelay   = flag.Duration("scale-down-delay", 10*time.Minute,
		"Duration from the last scale up to the time when CA starts to check scale down options")
	scaleDownUnneededTime = flag.Duration("scale-down-unneeded-time", 10*time.Minute,
		"How long a node should be unneeded before it is eligible for scale down")
	scaleDownUnreadyTime = flag.Duration("scale-down-unready-time", 20*time.Minute,
		"How long an unready node should be unneeded before it is eligible for scale down")
	scaleDownUtilizationThreshold = flag.Float64("scale-down-utilization-threshold", 0.5,
		"Node utilization level, defined as sum of requested resources divided by capacity, below which a node can be considered for scale down")
	scaleDownTrialInterval = flag.Duration("scale-down-trial-interval", 1*time.Minute,
		"How often scale down possiblity is check")
	scanInterval                = flag.Duration("scan-interval", 10*time.Second, "How often cluster is reevaluated for scale up or down")
	maxNodesTotal               = flag.Int("max-nodes-total", 0, "Maximum number of nodes in all node groups. Cluster autoscaler will not grow the cluster beyond this number.")
	cloudProviderFlag           = flag.String("cloud-provider", "gce", "Cloud provider type. Allowed values: gce, aws, azure")
	maxEmptyBulkDeleteFlag      = flag.Int("max-empty-bulk-delete", 10, "Maximum number of empty nodes that can be deleted at the same time.")
	maxGratefulTerminationFlag  = flag.Int("max-grateful-termination-sec", 60, "Maximum number of seconds CA waints for pod termination when trying to scale down a node.")
	maxTotalUnreadyPercentage   = flag.Float64("max-total-unready-percentage", 33, "Maximum percentage of unready nodes after which CA halts operations")
	okTotalUnreadyCount         = flag.Int("ok-total-unready-count", 3, "Number of allowed unready nodes, irrespective of max-total-unready-percentage")
	maxNodeProvisionTime        = flag.Duration("max-node-provision-time", 15*time.Minute, "Maximum time CA waits for node to be provisioned")
	unregisteredNodeRemovalTime = flag.Duration("unregistered-node-removal-time", 15*time.Minute, "Time that CA waits before removing nodes that are not registered in Kubernetes")

	estimatorFlag = flag.String("estimator", estimator.BinpackingEstimatorName,
		"Type of resource estimator to be used in scale up. Available values: ["+strings.Join(estimator.AvailableEstimators, ",")+"]")

	expanderFlag = flag.String("expander", expander.RandomExpanderName,
		"Type of node group expander to be used in scale up. Available values: ["+strings.Join(expander.AvailableExpanders, ",")+"]")

	writeStatusConfigMapFlag = flag.Bool("write-status-configmap", true, "Should CA write status information to a configmap")
)

func createAutoscalerOptions() core.AutoscalerOptions {
	autoscalingOpts := core.AutoscalingOptions{
		CloudConfig:                   *cloudConfig,
		CloudProviderName:             *cloudProviderFlag,
		MaxTotalUnreadyPercentage:     *maxTotalUnreadyPercentage,
		OkTotalUnreadyCount:           *okTotalUnreadyCount,
		EstimatorName:                 *estimatorFlag,
		ExpanderName:                  *expanderFlag,
		MaxEmptyBulkDelete:            *maxEmptyBulkDeleteFlag,
		MaxGratefulTerminationSec:     *maxGratefulTerminationFlag,
		MaxNodeProvisionTime:          *maxNodeProvisionTime,
		MaxNodesTotal:                 *maxNodesTotal,
		NodeGroups:                    nodeGroupsFlag,
		UnregisteredNodeRemovalTime:   *unregisteredNodeRemovalTime,
		ScaleDownDelay:                *scaleDownDelay,
		ScaleDownEnabled:              *scaleDownEnabled,
		ScaleDownTrialInterval:        *scaleDownTrialInterval,
		ScaleDownUnneededTime:         *scaleDownUnneededTime,
		ScaleDownUnreadyTime:          *scaleDownUnreadyTime,
		ScaleDownUtilizationThreshold: *scaleDownUtilizationThreshold,
		VerifyUnschedulablePods:       *verifyUnschedulablePods,
		WriteStatusConfigMap:          *writeStatusConfigMapFlag,
	}

	configFetcherOpts := dynamic.ConfigFetcherOptions{
		ConfigMapName: *configMapName,
		Namespace:     *namespace,
	}

	return core.AutoscalerOptions{
		AutoscalingOptions:   autoscalingOpts,
		ConfigFetcherOptions: configFetcherOpts,
	}
}

func createKubeClient() kube_client.Interface {
	url, err := url.Parse(*kubernetes)
	if err != nil {
		glog.Fatalf("Failed to parse Kuberentes url: %v", err)
	}

	kubeConfig, err := config.GetKubeClientConfig(url)
	if err != nil {
		glog.Fatalf("Failed to build Kuberentes client configuration: %v", err)
	}

	return kube_client.NewForConfigOrDie(kubeConfig)
}

func registerSignalHandlers(autoscaler core.Autoscaler) {
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, os.Interrupt, os.Kill, syscall.SIGTERM, syscall.SIGQUIT)
	glog.Info("Registered cleanup signal handler")

	go func() {
		<-sigs
		glog.Info("Receieved signal, attempting cleanup")
		autoscaler.ExitCleanUp()
		glog.Info("Cleaned up, exiting...")
		glog.Flush()
		os.Exit(0)
	}()
}

// In order to meet interface criteria for LeaderElectionConfig we need to
// take stop channel as an argument. However, since we are committing a suicide
// after loosing mastership we can safely ignore it.
func run(_ <-chan struct{}) {
	kubeClient := createKubeClient()
	kubeEventRecorder := kube_util.CreateEventRecorder(kubeClient)
	opts := createAutoscalerOptions()
	predicateChecker, err := simulator.NewPredicateChecker(kubeClient)
	if err != nil {
		glog.Fatalf("Failed to create predicate checker: %v", err)
	}
	stopChannel := make(chan struct{})
	listerRegistry := kube_util.NewListerRegistryWithDefaultListers(kubeClient, stopChannel)
	autoscaler := core.NewAutoscaler(opts, predicateChecker, kubeClient, kubeEventRecorder, listerRegistry)

	autoscaler.CleanUp()
	registerSignalHandlers(autoscaler)

	for {
		select {
		case <-time.After(*scanInterval):
			{
				loopStart := time.Now()
				metrics.UpdateLastTime("main")

				autoscaler.RunOnce(loopStart)

				metrics.UpdateDuration("main", loopStart)
			}
		}
	}
}

func main() {
	leaderElection := kube_leaderelection.DefaultLeaderElectionConfiguration()
	leaderElection.LeaderElect = true

	kube_leaderelection.BindFlags(&leaderElection, pflag.CommandLine)
	flag.Var(&nodeGroupsFlag, "nodes", "sets min,max size and other configuration data for a node group in a format accepted by cloud provider."+
		"Can be used multiple times. Format: <min>:<max>:<other...>")
	kube_flag.InitFlags()

	glog.Infof("Cluster Autoscaler %s", ClusterAutoscalerVersion)

	correctEstimator := false
	for _, availableEstimator := range estimator.AvailableEstimators {
		if *estimatorFlag == availableEstimator {
			correctEstimator = true
		}
	}
	if !correctEstimator {
		glog.Fatalf("Unrecognized estimator: %v", *estimatorFlag)
	}

	go func() {
		http.Handle("/metrics", prometheus.Handler())
		err := http.ListenAndServe(*address, nil)
		glog.Fatalf("Failed to start metrics: %v", err)
	}()

	if !leaderElection.LeaderElect {
		run(nil)
	} else {
		id, err := os.Hostname()
		if err != nil {
			glog.Fatalf("Unable to get hostname: %v", err)
		}

		kubeClient := createKubeClient()

		// Validate that the client is ok.
		_, err = kubeClient.Core().Nodes().List(metav1.ListOptions{})
		if err != nil {
			glog.Fatalf("Failed to get nodes from apiserver: %v", err)
		}

		kube_leaderelection.RunOrDie(kube_leaderelection.LeaderElectionConfig{
			Lock: &resourcelock.EndpointsLock{
				EndpointsMeta: metav1.ObjectMeta{
					Namespace: *namespace,
					Name:      "cluster-autoscaler",
				},
				Client: kubeClient,
				LockConfig: resourcelock.ResourceLockConfig{
					Identity:      id,
					EventRecorder: kube_util.CreateEventRecorder(kubeClient),
				},
			},
			LeaseDuration: leaderElection.LeaseDuration.Duration,
			RenewDeadline: leaderElection.RenewDeadline.Duration,
			RetryPeriod:   leaderElection.RetryPeriod.Duration,
			Callbacks: kube_leaderelection.LeaderCallbacks{
				OnStartedLeading: run,
				OnStoppedLeading: func() {
					glog.Fatalf("lost master")
				},
			},
		})
	}
}
<|endoftext|>