<commit_before>/**********************************************************************************
* Copyright (c) 2009-2017 Misakai Ltd.
* This program is free software: you can redistribute it and/or modify it under the
* terms of the GNU Affero General Public License as published by the  Free Software
* Foundation, either version 3 of the License, or(at your option) any later version.
*
* This program is distributed  in the hope that it  will be useful, but WITHOUT ANY
* WARRANTY;  without even  the implied warranty of MERCHANTABILITY or FITNESS FOR A
* PARTICULAR PURPOSE.  See the GNU Affero General Public License  for  more details.
*
* You should have  received a copy  of the  GNU Affero General Public License along
* with this program. If not, see<http://www.gnu.org/licenses/>.
************************************************************************************/

package broker

import (
	"bufio"
	"fmt"
	"net"
	"runtime/debug"
	"sync"
	"sync/atomic"
	"time"

	"github.com/emitter-io/emitter/broker/message"
	"github.com/emitter-io/emitter/logging"
	"github.com/emitter-io/emitter/network/address"
	"github.com/emitter-io/emitter/network/mqtt"
	"github.com/emitter-io/emitter/security"
)

// Conn represents an incoming connection.
type Conn struct {
	sync.Mutex
	tracked  uint32            // Whether the connection was already tracked or not.
	socket   net.Conn          // The transport used to read and write messages.
	username string            // The username provided by the client during MQTT connect.
	luid     security.ID       // The locally unique id of the connection.
	guid     string            // The globally unique id of the connection.
	service  *Service          // The service for this connection.
	subs     *message.Counters // The subscriptions for this connection.
}

// NewConn creates a new connection.
func (s *Service) newConn(t net.Conn) *Conn {
	c := &Conn{
		tracked: 0,
		luid:    security.NewID(),
		service: s,
		socket:  t,
		subs:    message.NewCounters(),
	}

	// Generate a globally unique id as well
	c.guid = c.luid.Unique(uint64(address.Hardware()), "emitter")
	logging.LogTarget("conn", "created", c.luid)

	// Increment the connection counter
	atomic.AddInt64(&s.connections, 1)
	return c
}

// ID returns the unique identifier of the subsriber.
func (c *Conn) ID() string {
	return c.guid
}

// Type returns the type of the subscriber
func (c *Conn) Type() message.SubscriberType {
	return message.SubscriberDirect
}

// track tracks the connection by adding it to the metering.
func (c *Conn) track(contract security.Contract) {
	if atomic.LoadUint32(&c.tracked) == 0 {

		// We keep only the IP address for fair tracking
		addr := c.socket.RemoteAddr().String()
		if tcp, ok := c.socket.RemoteAddr().(*net.TCPAddr); ok {
			addr = tcp.IP.String()
		}

		// Add the device to the stats and mark as done
		contract.Stats().AddDevice(addr)
		atomic.StoreUint32(&c.tracked, 1)
	}
}

// Process processes the messages.
func (c *Conn) Process() error {
	defer c.Close()
	reader := bufio.NewReaderSize(c.socket, 65536)

	for {
		// Set read/write deadlines so we can close dangling connections
		c.socket.SetDeadline(time.Now().Add(time.Second * 120))

		// Decode an incoming MQTT packet
		msg, err := mqtt.DecodePacket(reader)
		if err != nil {
			return err
		}

		switch msg.Type() {

		// We got an attempt to connect to MQTT.
		case mqtt.TypeOfConnect:
			packet := msg.(*mqtt.Connect)
			c.username = string(packet.Username)

			// Write the ack
			ack := mqtt.Connack{ReturnCode: 0x00}
			if _, err := ack.EncodeTo(c.socket); err != nil {
				return err
			}

		// We got an attempt to subscribe to a channel.
		case mqtt.TypeOfSubscribe:
			packet := msg.(*mqtt.Subscribe)
			ack := mqtt.Suback{
				MessageID: packet.MessageID,
				Qos:       make([]uint8, 0, len(packet.Subscriptions)),
			}

			// Subscribe for each subscription
			for _, sub := range packet.Subscriptions {
				if err := c.onSubscribe(sub.Topic); err != nil {
					logging.LogError("conn", "subscibe received", err)
					// TODO: Handle Error
				}

				// Append the QoS
				ack.Qos = append(ack.Qos, sub.Qos)
			}

			// Acknowledge the subscription
			if _, err := ack.EncodeTo(c.socket); err != nil {
				return err
			}

		// We got an attempt to unsubscribe from a channel.
		case mqtt.TypeOfUnsubscribe:
			packet := msg.(*mqtt.Unsubscribe)
			ack := mqtt.Unsuback{MessageID: packet.MessageID}

			// Unsubscribe from each subscription
			for _, sub := range packet.Topics {
				c.onUnsubscribe(sub.Topic) // TODO: Handle error or just ignore?
			}

			// Acknowledge the unsubscription
			if _, err := ack.EncodeTo(c.socket); err != nil {
				return err
			}

		// We got an MQTT ping response, respond appropriately.
		case mqtt.TypeOfPingreq:
			ack := mqtt.Pingresp{}
			if _, err := ack.EncodeTo(c.socket); err != nil {
				return err
			}

		case mqtt.TypeOfDisconnect:
			return nil

		case mqtt.TypeOfPublish:
			packet := msg.(*mqtt.Publish)

			if err := c.onPublish(packet.Topic, packet.Payload); err != nil {
				logging.LogError("conn", "publish received", err)
				// TODO: Handle Error
			}

			// Acknowledge the publication
			if packet.Header.QOS > 0 {
				ack := mqtt.Puback{MessageID: packet.MessageID}
				if _, err := ack.EncodeTo(c.socket); err != nil {
					return err
				}
			}
		}
	}
}

// Send forwards the message to the underlying client.
func (c *Conn) Send(m *message.Message) error {
	packet := mqtt.Publish{
		Header: &mqtt.StaticHeader{
			QOS: 0, // TODO when we'll support more QoS
		},
		MessageID: 0,         // TODO
		Topic:     m.Channel, // The channel for this message.
		Payload:   m.Payload, // The payload for this message.
	}

	// Acknowledge the publication
	_, err := packet.EncodeTo(c.socket)
	if err != nil {
		logging.LogError("conn", "message send", err)
		return err
	}

	return nil
}

// Subscribe subscribes to a particular channel.
func (c *Conn) Subscribe(ssid message.Ssid, channel []byte) {
	c.Lock()
	defer c.Unlock()

	// Add the subscription
	if first := c.subs.Increment(ssid, channel); first {

		// Subscribe the subscriber
		c.service.onSubscribe(ssid, c)

		// Broadcast the subscription within our cluster
		c.service.notifySubscribe(c, ssid, channel)
	}
}

// Unsubscribe unsubscribes this client from a particular channel.
func (c *Conn) Unsubscribe(ssid message.Ssid, channel []byte) {
	c.Lock()
	defer c.Unlock()

	// Decrement the counter and if there's no more subscriptions, notify everyone.
	if last := c.subs.Decrement(ssid); last {

		// Unsubscribe the subscriber
		c.service.onUnsubscribe(ssid, c)

		// Broadcast the unsubscription within our cluster
		c.service.notifyUnsubscribe(c, ssid, channel)
	}
}

// Close terminates the connection.
func (c *Conn) Close() error {
	logging.LogTarget("conn", "closed", c.guid)

	// Unsubscribe from everything, no need to lock since each Unsubscribe is
	// already locked. Locking the 'Close()' would result in a deadlock.
	for _, counter := range c.subs.All() {
		c.service.onUnsubscribe(counter.Ssid, c)
		c.service.notifyUnsubscribe(c, counter.Ssid, counter.Channel)
	}

	// Attempt to recover a panic
	if r := recover(); r != nil {
		logging.LogAction("closing", fmt.Sprintf("pancic recovered: %s \n %s", r, debug.Stack()))
	}

	// Close the transport and decrement the connection counter
	atomic.AddInt64(&c.service.connections, -1)
	return c.socket.Close()
}
<commit_msg>Handle error when subscription fail.<commit_after>/**********************************************************************************
* Copyright (c) 2009-2017 Misakai Ltd.
* This program is free software: you can redistribute it and/or modify it under the
* terms of the GNU Affero General Public License as published by the  Free Software
* Foundation, either version 3 of the License, or(at your option) any later version.
*
* This program is distributed  in the hope that it  will be useful, but WITHOUT ANY
* WARRANTY;  without even  the implied warranty of MERCHANTABILITY or FITNESS FOR A
* PARTICULAR PURPOSE.  See the GNU Affero General Public License  for  more details.
*
* You should have  received a copy  of the  GNU Affero General Public License along
* with this program. If not, see<http://www.gnu.org/licenses/>.
************************************************************************************/

package broker

import (
	"bufio"
	"fmt"
	"net"
	"runtime/debug"
	"sync"
	"sync/atomic"
	"time"

	"github.com/emitter-io/emitter/broker/message"
	"github.com/emitter-io/emitter/logging"
	"github.com/emitter-io/emitter/network/address"
	"github.com/emitter-io/emitter/network/mqtt"
	"github.com/emitter-io/emitter/security"
)

// Conn represents an incoming connection.
type Conn struct {
	sync.Mutex
	tracked  uint32            // Whether the connection was already tracked or not.
	socket   net.Conn          // The transport used to read and write messages.
	username string            // The username provided by the client during MQTT connect.
	luid     security.ID       // The locally unique id of the connection.
	guid     string            // The globally unique id of the connection.
	service  *Service          // The service for this connection.
	subs     *message.Counters // The subscriptions for this connection.
}

// NewConn creates a new connection.
func (s *Service) newConn(t net.Conn) *Conn {
	c := &Conn{
		tracked: 0,
		luid:    security.NewID(),
		service: s,
		socket:  t,
		subs:    message.NewCounters(),
	}

	// Generate a globally unique id as well
	c.guid = c.luid.Unique(uint64(address.Hardware()), "emitter")
	logging.LogTarget("conn", "created", c.luid)

	// Increment the connection counter
	atomic.AddInt64(&s.connections, 1)
	return c
}

// ID returns the unique identifier of the subsriber.
func (c *Conn) ID() string {
	return c.guid
}

// Type returns the type of the subscriber
func (c *Conn) Type() message.SubscriberType {
	return message.SubscriberDirect
}

// track tracks the connection by adding it to the metering.
func (c *Conn) track(contract security.Contract) {
	if atomic.LoadUint32(&c.tracked) == 0 {

		// We keep only the IP address for fair tracking
		addr := c.socket.RemoteAddr().String()
		if tcp, ok := c.socket.RemoteAddr().(*net.TCPAddr); ok {
			addr = tcp.IP.String()
		}

		// Add the device to the stats and mark as done
		contract.Stats().AddDevice(addr)
		atomic.StoreUint32(&c.tracked, 1)
	}
}

// Process processes the messages.
func (c *Conn) Process() error {
	defer c.Close()
	reader := bufio.NewReaderSize(c.socket, 65536)

	for {
		// Set read/write deadlines so we can close dangling connections
		c.socket.SetDeadline(time.Now().Add(time.Second * 120))

		// Decode an incoming MQTT packet
		msg, err := mqtt.DecodePacket(reader)
		if err != nil {
			return err
		}

		switch msg.Type() {

		// We got an attempt to connect to MQTT.
		case mqtt.TypeOfConnect:
			packet := msg.(*mqtt.Connect)
			c.username = string(packet.Username)

			// Write the ack
			ack := mqtt.Connack{ReturnCode: 0x00}
			if _, err := ack.EncodeTo(c.socket); err != nil {
				return err
			}

		// We got an attempt to subscribe to a channel.
		case mqtt.TypeOfSubscribe:
			packet := msg.(*mqtt.Subscribe)
			ack := mqtt.Suback{
				MessageID: packet.MessageID,
				Qos:       make([]uint8, 0, len(packet.Subscriptions)),
			}

			// Subscribe for each subscription
			for _, sub := range packet.Subscriptions {
				if err := c.onSubscribe(sub.Topic); err != nil {
					logging.LogError("conn", "subscibe received", err)
					ack.Qos = append(ack.Qos, 0x80) // 0x80 indicate subscription failure
				} else {
					// Append the QoS
					ack.Qos = append(ack.Qos, sub.Qos)
				}
			}

			// Acknowledge the subscription
			if _, err := ack.EncodeTo(c.socket); err != nil {
				return err
			}

		// We got an attempt to unsubscribe from a channel.
		case mqtt.TypeOfUnsubscribe:
			packet := msg.(*mqtt.Unsubscribe)
			ack := mqtt.Unsuback{MessageID: packet.MessageID}

			// Unsubscribe from each subscription
			for _, sub := range packet.Topics {
				c.onUnsubscribe(sub.Topic) // TODO: Handle error or just ignore?
			}

			// Acknowledge the unsubscription
			if _, err := ack.EncodeTo(c.socket); err != nil {
				return err
			}

		// We got an MQTT ping response, respond appropriately.
		case mqtt.TypeOfPingreq:
			ack := mqtt.Pingresp{}
			if _, err := ack.EncodeTo(c.socket); err != nil {
				return err
			}

		case mqtt.TypeOfDisconnect:
			return nil

		case mqtt.TypeOfPublish:
			packet := msg.(*mqtt.Publish)

			if err := c.onPublish(packet.Topic, packet.Payload); err != nil {
				logging.LogError("conn", "publish received", err)
				// TODO: Handle Error
			}

			// Acknowledge the publication
			if packet.Header.QOS > 0 {
				ack := mqtt.Puback{MessageID: packet.MessageID}
				if _, err := ack.EncodeTo(c.socket); err != nil {
					return err
				}
			}
		}
	}
}

// Send forwards the message to the underlying client.
func (c *Conn) Send(m *message.Message) error {
	packet := mqtt.Publish{
		Header: &mqtt.StaticHeader{
			QOS: 0, // TODO when we'll support more QoS
		},
		MessageID: 0,         // TODO
		Topic:     m.Channel, // The channel for this message.
		Payload:   m.Payload, // The payload for this message.
	}

	// Acknowledge the publication
	_, err := packet.EncodeTo(c.socket)
	if err != nil {
		logging.LogError("conn", "message send", err)
		return err
	}

	return nil
}

// Subscribe subscribes to a particular channel.
func (c *Conn) Subscribe(ssid message.Ssid, channel []byte) {
	c.Lock()
	defer c.Unlock()

	// Add the subscription
	if first := c.subs.Increment(ssid, channel); first {

		// Subscribe the subscriber
		c.service.onSubscribe(ssid, c)

		// Broadcast the subscription within our cluster
		c.service.notifySubscribe(c, ssid, channel)
	}
}

// Unsubscribe unsubscribes this client from a particular channel.
func (c *Conn) Unsubscribe(ssid message.Ssid, channel []byte) {
	c.Lock()
	defer c.Unlock()

	// Decrement the counter and if there's no more subscriptions, notify everyone.
	if last := c.subs.Decrement(ssid); last {

		// Unsubscribe the subscriber
		c.service.onUnsubscribe(ssid, c)

		// Broadcast the unsubscription within our cluster
		c.service.notifyUnsubscribe(c, ssid, channel)
	}
}

// Close terminates the connection.
func (c *Conn) Close() error {
	logging.LogTarget("conn", "closed", c.guid)

	// Unsubscribe from everything, no need to lock since each Unsubscribe is
	// already locked. Locking the 'Close()' would result in a deadlock.
	for _, counter := range c.subs.All() {
		c.service.onUnsubscribe(counter.Ssid, c)
		c.service.notifyUnsubscribe(c, counter.Ssid, counter.Channel)
	}

	// Attempt to recover a panic
	if r := recover(); r != nil {
		logging.LogAction("closing", fmt.Sprintf("pancic recovered: %s \n %s", r, debug.Stack()))
	}

	// Close the transport and decrement the connection counter
	atomic.AddInt64(&c.service.connections, -1)
	return c.socket.Close()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/getgauge/gauge/gauge_messages"
	"runtime"
)

type parallelSpecExecution struct {
	manifest             *manifest
	runner               *testRunner
	specifications       []*specification
	pluginHandler        *pluginHandler
	currentExecutionInfo *gauge_messages.ExecutionInfo
	aggregateResult      *suiteResult
}

type specCollection struct {
	specs []*specification
}

func (e *parallelSpecExecution) start() *suiteResult {
	specCollections := e.distributeSpecs(numberOfCores())
	suiteResultChannel := make(chan *suiteResult, len(specCollections))

	for _, specCollection := range specCollections {
		go e.startSpecsExecution(specCollection, suiteResultChannel)
	}

	suiteResults := make([]*suiteResult, 0)
	for _, _ = range specCollections {
		suiteResults = append(suiteResults, <-suiteResultChannel)
	}

	e.aggregateResult = e.aggregateResults(suiteResults)
	return e.aggregateResult
}

func (e *parallelSpecExecution) startSpecsExecution(specCollection *specCollection, suiteResults chan *suiteResult) {
	runner, err := startRunnerAndMakeConnection(e.manifest)
	if err != nil {
		fmt.Println("Failed: " + err.Error())
		suiteResults <- &suiteResult{}
		return
	}
	execution := newExecution(e.manifest, specCollection.specs, runner, e.pluginHandler, false)
	result := execution.start()
	runner.kill()
	suiteResults <- result
}

func (e *parallelSpecExecution) distributeSpecs(distributions int) []*specCollection {
	if distributions > len(e.specifications) {
		distributions = len(e.specifications)
	}
	specCollections := make([]*specCollection, distributions)
	for i := 0; i < len(e.specifications); i++ {
		mod := i % distributions
		if specCollections[mod] == nil {
			specCollections[mod] = &specCollection{specs: make([]*specification, 0)}
		}
		specCollections[mod].specs = append(specCollections[mod].specs, e.specifications[i])
	}
	return specCollections
}

func (e *parallelSpecExecution) finish() {
	message := &gauge_messages.Message{MessageType: gauge_messages.Message_SuiteExecutionResult.Enum(),
		SuiteExecutionResult: &gauge_messages.SuiteExecutionResult{SuiteResult: convertToProtoSuiteResult(e.aggregateResult)}}
	e.pluginHandler.notifyPlugins(message)
	e.pluginHandler.gracefullyKillPlugins()
}

func (e *parallelSpecExecution) aggregateResults(suiteResults []*suiteResult) *suiteResult {
	aggregateResult := &suiteResult{isFailed: false, specResults: make([]*specResult, 0)}
	for _, result := range suiteResults {
		aggregateResult.executionTime += result.executionTime
		aggregateResult.specsFailedCount += result.specsFailedCount
		aggregateResult.specResults = append(aggregateResult.specResults, result.specResults...)
		if result.isFailed {
			aggregateResult.isFailed = true
		}
		if result.preSuite != nil {
			aggregateResult.preSuite = result.preSuite
		}
		if result.postSuite != nil {
			aggregateResult.postSuite = result.postSuite
		}
	}
	return aggregateResult
}

func numberOfCores() int {
	return runtime.NumCPU()
}
<commit_msg>Reusing existing existing runner while executing in parallel<commit_after>package main

import (
	"fmt"
	"github.com/getgauge/gauge/gauge_messages"
	"runtime"
)

type parallelSpecExecution struct {
	manifest             *manifest
	runner               *testRunner
	specifications       []*specification
	pluginHandler        *pluginHandler
	currentExecutionInfo *gauge_messages.ExecutionInfo
	aggregateResult      *suiteResult
}

type specCollection struct {
	specs []*specification
}

func (e *parallelSpecExecution) start() *suiteResult {
	specCollections := e.distributeSpecs(numberOfCores())
	suiteResultChannel := make(chan *suiteResult, len(specCollections))

	for i, specCollection := range specCollections {
		if i == 0 {
			go e.startSpecsExecution(specCollection, suiteResultChannel, e.runner)
		} else {
			go e.startSpecsExecution(specCollection, suiteResultChannel, nil)
		}
	}

	suiteResults := make([]*suiteResult, 0)
	for _, _ = range specCollections {
		suiteResults = append(suiteResults, <-suiteResultChannel)
	}

	e.aggregateResult = e.aggregateResults(suiteResults)
	return e.aggregateResult
}

func (e *parallelSpecExecution) startSpecsExecution(specCollection *specCollection, suiteResults chan *suiteResult, runner *testRunner) {
	if runner == nil {
		var err error
		runner, err = startRunnerAndMakeConnection(e.manifest)
		if err != nil {
			fmt.Println("Failed: " + err.Error())
			suiteResults <- &suiteResult{}
			return
		}
	}
	e.startSpecsExecutionWithRunner(specCollection, suiteResults, runner)
}

func (e *parallelSpecExecution) startSpecsExecutionWithRunner(specCollection *specCollection, suiteResults chan *suiteResult, runner *testRunner) {
	execution := newExecution(e.manifest, specCollection.specs, runner, e.pluginHandler, false)
	result := execution.start()
	runner.kill()
	suiteResults <- result

}

func (e *parallelSpecExecution) distributeSpecs(distributions int) []*specCollection {
	if distributions > len(e.specifications) {
		distributions = len(e.specifications)
	}
	specCollections := make([]*specCollection, distributions)
	for i := 0; i < len(e.specifications); i++ {
		mod := i % distributions
		if specCollections[mod] == nil {
			specCollections[mod] = &specCollection{specs: make([]*specification, 0)}
		}
		specCollections[mod].specs = append(specCollections[mod].specs, e.specifications[i])
	}
	return specCollections
}

func (e *parallelSpecExecution) finish() {
	message := &gauge_messages.Message{MessageType: gauge_messages.Message_SuiteExecutionResult.Enum(),
		SuiteExecutionResult: &gauge_messages.SuiteExecutionResult{SuiteResult: convertToProtoSuiteResult(e.aggregateResult)}}
	e.pluginHandler.notifyPlugins(message)
	e.pluginHandler.gracefullyKillPlugins()
}

func (e *parallelSpecExecution) aggregateResults(suiteResults []*suiteResult) *suiteResult {
	aggregateResult := &suiteResult{isFailed: false, specResults: make([]*specResult, 0)}
	for _, result := range suiteResults {
		aggregateResult.executionTime += result.executionTime
		aggregateResult.specsFailedCount += result.specsFailedCount
		aggregateResult.specResults = append(aggregateResult.specResults, result.specResults...)
		if result.isFailed {
			aggregateResult.isFailed = true
		}
		if result.preSuite != nil {
			aggregateResult.preSuite = result.preSuite
		}
		if result.postSuite != nil {
			aggregateResult.postSuite = result.postSuite
		}
	}
	return aggregateResult
}

func numberOfCores() int {
	return runtime.NumCPU()
}
<|endoftext|>


<commit_before>package udp

import (
	"expvar"
	"fmt"
	"github.com/davyxu/cellnet"
	"github.com/davyxu/cellnet/peer"
	"github.com/davyxu/cellnet/util"
	"net"
	"time"
)

const MaxUDPRecvBuffer = 2048

type udpAcceptor struct {
	peer.CoreSessionManager
	peer.CorePeerProperty
	peer.CoreContextSet
	peer.CoreRunningTag
	peer.CoreProcBundle

	localAddr *net.UDPAddr

	conn *net.UDPConn

	sesQueue *util.Queue

	sesTimeout time.Duration

	mtSesQueueCount      *expvar.Int
	mtTotalRecvUDPPacket *expvar.Int
}

func (self *udpAcceptor) Start() cellnet.Peer {

	if self.mtSesQueueCount == nil {
		self.mtSesQueueCount = expvar.NewInt(fmt.Sprintf("cellnet.Peer(%s).SessionQueueCount", self.Name()))
	}

	if self.mtTotalRecvUDPPacket == nil {
		self.mtTotalRecvUDPPacket = expvar.NewInt(fmt.Sprintf("cellnet.Peer(%s).TotalRecvUDPPacket", self.Name()))
	}

	var err error
	self.localAddr, err = net.ResolveUDPAddr("udp", self.Address())

	if err != nil {

		log.Errorf("#udp.resolve failed(%s) %v", self.NameOrAddress(), err.Error())
		return self
	}

	self.conn, err = net.ListenUDP("udp", self.localAddr)

	if err != nil {
		log.Errorf("#udp.listen failed(%s) %s", self.NameOrAddress(), err.Error())
		self.SetRunning(false)
		return self
	}

	log.Infof("#udp.listen(%s) %s", self.Name(), self.Address())

	go self.accept()

	return self
}

func (self *udpAcceptor) accept() {

	self.SetRunning(true)

	recvBuff := make([]byte, MaxUDPRecvBuffer)

	for {

		n, remoteAddr, err := self.conn.ReadFromUDP(recvBuff)
		if err != nil {
			break
		}

		if n > 0 {
			self.mtTotalRecvUDPPacket.Add(1)

			ses := self.allocSession(remoteAddr)
			ses.Recv(recvBuff[:n])
		}

	}

	self.SetRunning(false)

}

func (self *udpAcceptor) allocSession(addr *net.UDPAddr) *udpSession {

	var ses *udpSession

	if self.sesQueue.Count() > 0 {
		ses = self.sesQueue.Peek().(*udpSession)

		// 这个session还能用，需要重新new
		if ses.IsAlive() {
			ses = nil
		} else {
			// 可以复用
			ses = self.sesQueue.Dequeue().(*udpSession)
		}

	}

	if ses == nil {
		ses = &udpSession{}
		self.sesQueue.Enqueue(ses)
	}

	self.mtSesQueueCount.Set(int64(self.sesQueue.Count()))

	ses.timeOutTick = time.Now().Add(self.sesTimeout)
	ses.conn = self.conn
	ses.remote = addr
	ses.pInterface = self
	ses.CoreProcBundle = &self.CoreProcBundle

	return ses
}

func (self *udpAcceptor) SetSessionTTL(dur time.Duration) {
	self.sesTimeout = dur
}

func (self *udpAcceptor) Stop() {

	if self.conn != nil {
		self.conn.Close()
	}

	// TODO 等待accept线程结束
	self.SetRunning(false)
}

func (self *udpAcceptor) TypeName() string {
	return "udp.Acceptor"
}

func init() {

	peer.RegisterPeerCreator(func() cellnet.Peer {
		p := &udpAcceptor{
			sesQueue:   util.NewQueue(64),
			sesTimeout: time.Second,
		}

		return p
	})
}
<commit_msg>add udp端口接口<commit_after>package udp

import (
	"expvar"
	"fmt"
	"github.com/davyxu/cellnet"
	"github.com/davyxu/cellnet/peer"
	"github.com/davyxu/cellnet/util"
	"net"
	"time"
)

const MaxUDPRecvBuffer = 2048

type udpAcceptor struct {
	peer.CoreSessionManager
	peer.CorePeerProperty
	peer.CoreContextSet
	peer.CoreRunningTag
	peer.CoreProcBundle

	localAddr *net.UDPAddr

	conn *net.UDPConn

	sesQueue *util.Queue

	sesTimeout time.Duration

	mtSesQueueCount      *expvar.Int
	mtTotalRecvUDPPacket *expvar.Int
}

func (self *udpAcceptor) IsReady() bool {

	return self.IsRunning()
}

func (self *udpAcceptor) Port() int {
	if self.conn == nil {
		return 0
	}

	return self.conn.LocalAddr().(*net.UDPAddr).Port
}

func (self *udpAcceptor) Start() cellnet.Peer {

	if self.mtSesQueueCount == nil {
		self.mtSesQueueCount = expvar.NewInt(fmt.Sprintf("cellnet.Peer(%s).SessionQueueCount", self.Name()))
	}

	if self.mtTotalRecvUDPPacket == nil {
		self.mtTotalRecvUDPPacket = expvar.NewInt(fmt.Sprintf("cellnet.Peer(%s).TotalRecvUDPPacket", self.Name()))
	}

	var err error
	self.localAddr, err = net.ResolveUDPAddr("udp", self.Address())

	if err != nil {

		log.Errorf("#udp.resolve failed(%s) %v", self.NameOrAddress(), err.Error())
		return self
	}

	self.conn, err = net.ListenUDP("udp", self.localAddr)

	if err != nil {
		log.Errorf("#udp.listen failed(%s) %s", self.NameOrAddress(), err.Error())
		self.SetRunning(false)
		return self
	}

	log.Infof("#udp.listen(%s) %s", self.Name(), self.Address())

	go self.accept()

	return self
}

func (self *udpAcceptor) accept() {

	self.SetRunning(true)

	recvBuff := make([]byte, MaxUDPRecvBuffer)

	for {

		n, remoteAddr, err := self.conn.ReadFromUDP(recvBuff)
		if err != nil {
			break
		}

		if n > 0 {
			self.mtTotalRecvUDPPacket.Add(1)

			ses := self.allocSession(remoteAddr)
			ses.Recv(recvBuff[:n])
		}

	}

	self.SetRunning(false)

}

func (self *udpAcceptor) allocSession(addr *net.UDPAddr) *udpSession {

	var ses *udpSession

	if self.sesQueue.Count() > 0 {
		ses = self.sesQueue.Peek().(*udpSession)

		// 这个session还能用，需要重新new
		if ses.IsAlive() {
			ses = nil
		} else {
			// 可以复用
			ses = self.sesQueue.Dequeue().(*udpSession)
		}

	}

	if ses == nil {
		ses = &udpSession{}
		self.sesQueue.Enqueue(ses)
	}

	self.mtSesQueueCount.Set(int64(self.sesQueue.Count()))

	ses.timeOutTick = time.Now().Add(self.sesTimeout)
	ses.conn = self.conn
	ses.remote = addr
	ses.pInterface = self
	ses.CoreProcBundle = &self.CoreProcBundle

	return ses
}

func (self *udpAcceptor) SetSessionTTL(dur time.Duration) {
	self.sesTimeout = dur
}

func (self *udpAcceptor) Stop() {

	if self.conn != nil {
		self.conn.Close()
	}

	// TODO 等待accept线程结束
	self.SetRunning(false)
}

func (self *udpAcceptor) TypeName() string {
	return "udp.Acceptor"
}

func init() {

	peer.RegisterPeerCreator(func() cellnet.Peer {
		p := &udpAcceptor{
			sesQueue:   util.NewQueue(64),
			sesTimeout: time.Second,
		}

		return p
	})
}
<|endoftext|>


<commit_before>// Copyright ©2015 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package topo

import (
	"github.com/gonum/graph"
	"github.com/gonum/graph/internal/set"
)

// VertexOrdering returns the vertex ordering and the k-cores of
// the undirected graph g.
func VertexOrdering(g graph.Undirected) (order []graph.Node, cores [][]graph.Node) {
	nodes := g.Nodes()

	// The algorithm used here is essentially as described at
	// http://en.wikipedia.org/w/index.php?title=Degeneracy_%28graph_theory%29&oldid=640308710

	// Initialize an output list L.
	var l []graph.Node

	// Compute a number d_v for each vertex v in G,
	// the number of neighbors of v that are not already in L.
	// Initially, these numbers are just the degrees of the vertices.
	dv := make(map[int]int, len(nodes))
	var (
		maxDegree  int
		neighbours = make(map[int][]graph.Node)
	)
	for _, n := range nodes {
		adj := g.From(n)
		neighbours[n.ID()] = adj
		dv[n.ID()] = len(adj)
		if len(adj) > maxDegree {
			maxDegree = len(adj)
		}
	}

	// Initialize an array D such that D[i] contains a list of the
	// vertices v that are not already in L for which d_v = i.
	d := make([][]graph.Node, maxDegree+1)
	for _, n := range nodes {
		deg := dv[n.ID()]
		d[deg] = append(d[deg], n)
	}

	// Initialize k to 0.
	k := 0
	// Repeat n times:
	s := []int{0}
	for _ = range nodes { // TODO(kortschak): Remove blank assignment when go1.3.3 is no longer supported.
		// Scan the array cells D[0], D[1], ... until
		// finding an i for which D[i] is nonempty.
		var (
			i  int
			di []graph.Node
		)
		for i, di = range d {
			if len(di) != 0 {
				break
			}
		}

		// Set k to max(k,i).
		if i > k {
			k = i
			s = append(s, make([]int, k-len(s)+1)...)
		}

		// Select a vertex v from D[i]. Add v to the
		// beginning of L and remove it from D[i].
		var v graph.Node
		v, d[i] = di[len(di)-1], di[:len(di)-1]
		l = append(l, v)
		s[k]++
		delete(dv, v.ID())

		// For each neighbor w of v not already in L,
		// subtract one from d_w and move w to the
		// cell of D corresponding to the new value of d_w.
		for _, w := range neighbours[v.ID()] {
			dw, ok := dv[w.ID()]
			if !ok {
				continue
			}
			for i, n := range d[dw] {
				if n.ID() == w.ID() {
					d[dw][i], d[dw] = d[dw][len(d[dw])-1], d[dw][:len(d[dw])-1]
					dw--
					d[dw] = append(d[dw], w)
					break
				}
			}
			dv[w.ID()] = dw
		}
	}

	for i, j := 0, len(l)-1; i < j; i, j = i+1, j-1 {
		l[i], l[j] = l[j], l[i]
	}
	cores = make([][]graph.Node, len(s))
	offset := len(l)
	for i, n := range s {
		cores[i] = l[offset-n : offset]
		offset -= n
	}
	return l, cores
}

// BronKerbosch returns the set of maximal cliques of the undirected graph g.
func BronKerbosch(g graph.Undirected) [][]graph.Node {
	nodes := g.Nodes()

	// The algorithm used here is essentially BronKerbosch3 as described at
	// http://en.wikipedia.org/w/index.php?title=Bron%E2%80%93Kerbosch_algorithm&oldid=656805858

	p := make(set.Nodes, len(nodes))
	for _, n := range nodes {
		p.Add(n)
	}
	x := make(set.Nodes)
	var bk bronKerbosch
	order, _ := VertexOrdering(g)
	for _, v := range order {
		neighbours := g.From(v)
		nv := make(set.Nodes, len(neighbours))
		for _, n := range neighbours {
			nv.Add(n)
		}
		bk.maximalCliquePivot(g, []graph.Node{v}, make(set.Nodes).Intersect(p, nv), make(set.Nodes).Intersect(x, nv))
		p.Remove(v)
		x.Add(v)
	}
	return bk
}

type bronKerbosch [][]graph.Node

func (bk *bronKerbosch) maximalCliquePivot(g graph.Undirected, r []graph.Node, p, x set.Nodes) {
	if len(p) == 0 && len(x) == 0 {
		*bk = append(*bk, r)
		return
	}

	neighbours := bk.choosePivotFrom(g, p, x)
	nu := make(set.Nodes, len(neighbours))
	for _, n := range neighbours {
		nu.Add(n)
	}
	for _, v := range p {
		if nu.Has(v) {
			continue
		}
		neighbours := g.From(v)
		nv := make(set.Nodes, len(neighbours))
		for _, n := range neighbours {
			nv.Add(n)
		}

		var found bool
		for _, n := range r {
			if n.ID() == v.ID() {
				found = true
				break
			}
		}
		var sr []graph.Node
		if !found {
			sr = append(r[:len(r):len(r)], v)
		}

		bk.maximalCliquePivot(g, sr, make(set.Nodes).Intersect(p, nv), make(set.Nodes).Intersect(x, nv))
		p.Remove(v)
		x.Add(v)
	}
}

func (*bronKerbosch) choosePivotFrom(g graph.Undirected, p, x set.Nodes) (neighbors []graph.Node) {
	// TODO(kortschak): Investigate the impact of pivot choice that maximises
	// |p ⋂ neighbours(u)| as a function of input size. Until then, leave as
	// compile time option.
	if !tomitaTanakaTakahashi {
		for _, n := range p {
			return g.From(n)
		}
		for _, n := range x {
			return g.From(n)
		}
		panic("bronKerbosch: empty set")
	}

	var (
		max   = -1
		pivot graph.Node
	)
	maxNeighbors := func(s set.Nodes) {
	outer:
		for _, u := range s {
			nb := g.From(u)
			c := len(nb)
			if c <= max {
				continue
			}
			for n := range nb {
				if _, ok := p[n]; ok {
					continue
				}
				c--
				if c <= max {
					continue outer
				}
			}
			max = c
			pivot = u
			neighbors = nb
		}
	}
	maxNeighbors(p)
	maxNeighbors(x)
	if pivot == nil {
		panic("bronKerbosch: empty set")
	}
	return neighbors
}
<commit_msg>topo: drop blank range variable assignment<commit_after>// Copyright ©2015 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package topo

import (
	"github.com/gonum/graph"
	"github.com/gonum/graph/internal/set"
)

// VertexOrdering returns the vertex ordering and the k-cores of
// the undirected graph g.
func VertexOrdering(g graph.Undirected) (order []graph.Node, cores [][]graph.Node) {
	nodes := g.Nodes()

	// The algorithm used here is essentially as described at
	// http://en.wikipedia.org/w/index.php?title=Degeneracy_%28graph_theory%29&oldid=640308710

	// Initialize an output list L.
	var l []graph.Node

	// Compute a number d_v for each vertex v in G,
	// the number of neighbors of v that are not already in L.
	// Initially, these numbers are just the degrees of the vertices.
	dv := make(map[int]int, len(nodes))
	var (
		maxDegree  int
		neighbours = make(map[int][]graph.Node)
	)
	for _, n := range nodes {
		adj := g.From(n)
		neighbours[n.ID()] = adj
		dv[n.ID()] = len(adj)
		if len(adj) > maxDegree {
			maxDegree = len(adj)
		}
	}

	// Initialize an array D such that D[i] contains a list of the
	// vertices v that are not already in L for which d_v = i.
	d := make([][]graph.Node, maxDegree+1)
	for _, n := range nodes {
		deg := dv[n.ID()]
		d[deg] = append(d[deg], n)
	}

	// Initialize k to 0.
	k := 0
	// Repeat n times:
	s := []int{0}
	for range nodes {
		// Scan the array cells D[0], D[1], ... until
		// finding an i for which D[i] is nonempty.
		var (
			i  int
			di []graph.Node
		)
		for i, di = range d {
			if len(di) != 0 {
				break
			}
		}

		// Set k to max(k,i).
		if i > k {
			k = i
			s = append(s, make([]int, k-len(s)+1)...)
		}

		// Select a vertex v from D[i]. Add v to the
		// beginning of L and remove it from D[i].
		var v graph.Node
		v, d[i] = di[len(di)-1], di[:len(di)-1]
		l = append(l, v)
		s[k]++
		delete(dv, v.ID())

		// For each neighbor w of v not already in L,
		// subtract one from d_w and move w to the
		// cell of D corresponding to the new value of d_w.
		for _, w := range neighbours[v.ID()] {
			dw, ok := dv[w.ID()]
			if !ok {
				continue
			}
			for i, n := range d[dw] {
				if n.ID() == w.ID() {
					d[dw][i], d[dw] = d[dw][len(d[dw])-1], d[dw][:len(d[dw])-1]
					dw--
					d[dw] = append(d[dw], w)
					break
				}
			}
			dv[w.ID()] = dw
		}
	}

	for i, j := 0, len(l)-1; i < j; i, j = i+1, j-1 {
		l[i], l[j] = l[j], l[i]
	}
	cores = make([][]graph.Node, len(s))
	offset := len(l)
	for i, n := range s {
		cores[i] = l[offset-n : offset]
		offset -= n
	}
	return l, cores
}

// BronKerbosch returns the set of maximal cliques of the undirected graph g.
func BronKerbosch(g graph.Undirected) [][]graph.Node {
	nodes := g.Nodes()

	// The algorithm used here is essentially BronKerbosch3 as described at
	// http://en.wikipedia.org/w/index.php?title=Bron%E2%80%93Kerbosch_algorithm&oldid=656805858

	p := make(set.Nodes, len(nodes))
	for _, n := range nodes {
		p.Add(n)
	}
	x := make(set.Nodes)
	var bk bronKerbosch
	order, _ := VertexOrdering(g)
	for _, v := range order {
		neighbours := g.From(v)
		nv := make(set.Nodes, len(neighbours))
		for _, n := range neighbours {
			nv.Add(n)
		}
		bk.maximalCliquePivot(g, []graph.Node{v}, make(set.Nodes).Intersect(p, nv), make(set.Nodes).Intersect(x, nv))
		p.Remove(v)
		x.Add(v)
	}
	return bk
}

type bronKerbosch [][]graph.Node

func (bk *bronKerbosch) maximalCliquePivot(g graph.Undirected, r []graph.Node, p, x set.Nodes) {
	if len(p) == 0 && len(x) == 0 {
		*bk = append(*bk, r)
		return
	}

	neighbours := bk.choosePivotFrom(g, p, x)
	nu := make(set.Nodes, len(neighbours))
	for _, n := range neighbours {
		nu.Add(n)
	}
	for _, v := range p {
		if nu.Has(v) {
			continue
		}
		neighbours := g.From(v)
		nv := make(set.Nodes, len(neighbours))
		for _, n := range neighbours {
			nv.Add(n)
		}

		var found bool
		for _, n := range r {
			if n.ID() == v.ID() {
				found = true
				break
			}
		}
		var sr []graph.Node
		if !found {
			sr = append(r[:len(r):len(r)], v)
		}

		bk.maximalCliquePivot(g, sr, make(set.Nodes).Intersect(p, nv), make(set.Nodes).Intersect(x, nv))
		p.Remove(v)
		x.Add(v)
	}
}

func (*bronKerbosch) choosePivotFrom(g graph.Undirected, p, x set.Nodes) (neighbors []graph.Node) {
	// TODO(kortschak): Investigate the impact of pivot choice that maximises
	// |p ⋂ neighbours(u)| as a function of input size. Until then, leave as
	// compile time option.
	if !tomitaTanakaTakahashi {
		for _, n := range p {
			return g.From(n)
		}
		for _, n := range x {
			return g.From(n)
		}
		panic("bronKerbosch: empty set")
	}

	var (
		max   = -1
		pivot graph.Node
	)
	maxNeighbors := func(s set.Nodes) {
	outer:
		for _, u := range s {
			nb := g.From(u)
			c := len(nb)
			if c <= max {
				continue
			}
			for n := range nb {
				if _, ok := p[n]; ok {
					continue
				}
				c--
				if c <= max {
					continue outer
				}
			}
			max = c
			pivot = u
			neighbors = nb
		}
	}
	maxNeighbors(p)
	maxNeighbors(x)
	if pivot == nil {
		panic("bronKerbosch: empty set")
	}
	return neighbors
}
<|endoftext|>


<commit_before>package net

import (
	"io"
	"log"
	"net"
	"sync"

	"../redlot"
)

var counter struct {
	sync.RWMutex
	ConnCounter uint64
	TotalCalls  uint64
}

func init() {
	// Register commands.
	// system info
	REG("INFO", StatusReply, info)

	// KV type
	REG("GET", BulkReply, redlot.Get)
	REG("SET", StatusReply, redlot.Set)
	REG("DEL", StatusReply, redlot.Del)
	REG("EXISTS", IntReply, redlot.Exists)
	REG("SETX", StatusReply, redlot.Setx)
	REG("SETEX", StatusReply, redlot.Setx) // Alias of SETX
	REG("TTL", IntReply, redlot.TTL)
	REG("EXPIRE", IntReply, redlot.Expire)
	REGL("KEYS", ListReply, redlot.Keys)
	REGL("SCAN", ListReply, redlot.Scan)
	REGL("RSCAN", ListReply, redlot.Rscan)
	REGL("MULTI_GET", ListReply, redlot.MultiGet)
}

func Serve(addr string, options *redlot.Options) {
	// Open LevelDB with options.
	redlot.Open(options)

	// Create sockets listener.
	l, err := net.Listen("tcp4", addr)
	if err != nil {
		log.Fatalf("Listen error: %v\n", err.Error())
	}
	defer l.Close()

	for {
		conn, err := l.Accept()
		if err != nil {
			log.Fatalf("Wait for a connection error: %s\n", err.Error())
		}

		// Count connecion
		counter.Lock()
		counter.ConnCounter++
		counter.Unlock()

		go func(c net.Conn) {
			for {
				req, err := newRequset(c)
				if err == io.EOF {
					break
				}
				if err != nil {
					continue
				}

				counter.Lock()
				counter.TotalCalls++
				counter.Unlock()

				r := run(req.Cmd, req.Args)
				r.WriteTo(c)
			}

			c.Close()
			counter.Lock()
			counter.ConnCounter--
			counter.Unlock()

		}(conn)

	}
}
<commit_msg>Register RKEYS command.<commit_after>package net

import (
	"io"
	"log"
	"net"
	"sync"

	"../redlot"
)

var counter struct {
	sync.RWMutex
	ConnCounter uint64
	TotalCalls  uint64
}

func init() {
	// Register commands.
	// system info
	REG("INFO", StatusReply, info)

	// KV type
	REG("GET", BulkReply, redlot.Get)
	REG("SET", StatusReply, redlot.Set)
	REG("DEL", StatusReply, redlot.Del)
	REG("EXISTS", IntReply, redlot.Exists)
	REG("SETX", StatusReply, redlot.Setx)
	REG("SETEX", StatusReply, redlot.Setx) // Alias of SETX
	REG("TTL", IntReply, redlot.TTL)
	REG("EXPIRE", IntReply, redlot.Expire)
	REGL("KEYS", ListReply, redlot.Keys)
	REGL("RKEYS", ListReply, redlot.Rkeys)
	REGL("SCAN", ListReply, redlot.Scan)
	REGL("RSCAN", ListReply, redlot.Rscan)
	REGL("MULTI_GET", ListReply, redlot.MultiGet)
}

func Serve(addr string, options *redlot.Options) {
	// Open LevelDB with options.
	redlot.Open(options)

	// Create sockets listener.
	l, err := net.Listen("tcp4", addr)
	if err != nil {
		log.Fatalf("Listen error: %v\n", err.Error())
	}
	defer l.Close()

	for {
		conn, err := l.Accept()
		if err != nil {
			log.Fatalf("Wait for a connection error: %s\n", err.Error())
		}

		// Count connecion
		counter.Lock()
		counter.ConnCounter++
		counter.Unlock()

		go func(c net.Conn) {
			for {
				req, err := newRequset(c)
				if err == io.EOF {
					break
				}
				if err != nil {
					continue
				}

				counter.Lock()
				counter.TotalCalls++
				counter.Unlock()

				r := run(req.Cmd, req.Args)
				r.WriteTo(c)
			}

			c.Close()
			counter.Lock()
			counter.ConnCounter--
			counter.Unlock()

		}(conn)

	}
}
<|endoftext|>


<commit_before>/*
A wrapper on top of the Site struct that caches the list of image URLs in memory. Cache retrieval is via channels as
the cache code runs in a go-routine to allow access from multiple go-routines and we wan't to avoid the "thundering herd"
problem by only letting one process refresh the cache.

It also allows us to update the cache in the backend while serving stale copies until a new cache is built.
*/
package main

import (
	"sync/atomic"
	"time"
	"fmt"
	"sync"
)

const CACHE_INTERVAL = 10 * time.Second

type CachedSite struct {
	*Site

	KeyCache        atomic.Value
	LastCacheUpdate time.Time

	CacheUpdateMutex sync.Mutex
}

type GetFromCache struct {
	keys []string
	err  error
}

func NewCachedSiteFromSite(s *Site) *CachedSite {
	cs := &CachedSite{s, atomic.Value{}, time.Time{}, sync.Mutex{}}
	<- cs.GetFromCache()
	return cs
}

func (cs *CachedSite) GetAllImageUrls() []string {
	var imageUrls []string = []string{}

	imageKeys, err := cs.GetAllImageKeys()
	if err != nil {
		fmt.Printf("Unable to get image keys from S3. Error: %s", err.Error())
		return imageUrls
	}

	for _, v := range imageKeys {
		imageUrl, err := cs.GetUrlForImage(v)
		if err != nil {
			fmt.Printf("Unable to create URL for key %s. Error: %s", v, err.Error())
			continue
		}
		imageUrls = append(imageUrls, imageUrl)
	}

	return imageUrls
}

func (cs *CachedSite) GetAllImageKeys() ([]string, error) {
	result := <-cs.GetFromCache()
	if result.err != nil {
		return nil, result.err
	} else {
		return result.keys, result.err
	}
}

func (cs *CachedSite) NeedsUpdate() bool {
	return time.Now().Sub(cs.LastCacheUpdate) > CACHE_INTERVAL
}

func (cs *CachedSite) GetFromCache() chan *GetFromCache {
	cs.CacheUpdateMutex.Lock()

	c := make(chan *GetFromCache)
	go func() {
		var keys []string
		var err error

		if cs.NeedsUpdate() {
			keys, err = cs.Site.GetAllImageKeys()
			if err == nil {
				cs.KeyCache.Store(keys)
				cs.LastCacheUpdate = time.Now()
			}
		} else {
			keys, err = cs.KeyCache.Load().([]string), nil
		}

		c <- &GetFromCache{keys, err}
		cs.CacheUpdateMutex.Unlock()
	}()

	return c
}
<commit_msg>Improved cache read interface<commit_after>/*
A wrapper on top of the Site struct that caches the list of image URLs in memory. Cache retrieval is via channels as
the cache code runs in a go-routine to allow access from multiple go-routines and we wan't to avoid the "thundering herd"
problem by only letting one process refresh the cache.

It also allows us to update the cache in the backend while serving stale copies until a new cache is built.
*/
package main

import (
	"fmt"
	"sync"
	"sync/atomic"
	"time"
)

const CACHE_INTERVAL = 10 * time.Second

type CachedSite struct {
	*Site

	KeyCache        atomic.Value
	LastCacheUpdate time.Time

	CacheUpdateMutex sync.Mutex
}

type GetFromCache struct {
	keys []string
	err  error
}

func NewCachedSiteFromSite(s *Site) *CachedSite {
	cs := &CachedSite{s, atomic.Value{}, time.Time{}, sync.Mutex{}}
	<-cs.GetFromCache()
	return cs
}

func (cs *CachedSite) GetAllImageUrls() []string {
	var imageUrls []string = []string{}

	imageKeys, err := cs.GetAllImageKeys()
	if err != nil {
		fmt.Printf("Unable to get image keys from S3. Error: %s", err.Error())
		return imageUrls
	}

	for _, v := range imageKeys {
		imageUrl, err := cs.GetUrlForImage(v)
		if err != nil {
			fmt.Printf("Unable to create URL for key %s. Error: %s", v, err.Error())
			continue
		}
		imageUrls = append(imageUrls, imageUrl)
	}

	return imageUrls
}

func (cs *CachedSite) GetAllImageKeys() ([]string, error) {
	result := <-cs.GetFromCache()
	if result.err != nil {
		return nil, result.err
	} else {
		return result.keys, result.err
	}
}

func (cs *CachedSite) NeedsUpdate() bool {
	return time.Now().Sub(cs.LastCacheUpdate) > CACHE_INTERVAL
}

func (cs *CachedSite) GetFromCache() chan *GetFromCache {
	cs.CacheUpdateMutex.Lock()

	c := make(chan *GetFromCache)
	go func() {
		var keys []string
		var err error

		if cs.KeyCache.Load() != nil {
			c <- &GetFromCache{cs.KeyCache.Load().([]string), nil}

			if cs.NeedsUpdate() {
				keys, err = cs.Site.GetAllImageKeys()
				if err == nil {
					cs.KeyCache.Store(keys)
					cs.LastCacheUpdate = time.Now()
				}
			}
		} else {
			keys, err = cs.Site.GetAllImageKeys()
			if err == nil {
				cs.KeyCache.Store(keys)
				cs.LastCacheUpdate = time.Now()
			}
			c <- &GetFromCache{keys, err}
		}

		cs.CacheUpdateMutex.Unlock()
	}()

	return c
}
<|endoftext|>


<commit_before>package accounts

import (
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/instance"
)

// This file contains the account_type object as defined in
// docs/konnectors_oauth

// Various grant types
// - AuthorizationCode is the server-side grant type.
// - ImplicitGrant is the implicit grant type
// - ImplicitGrantRedirectURL is the implicit grant type but with redirect_url
//    											  instead of redirect_uri
const (
	AuthorizationCode        = "authorization_code"
	ImplicitGrant            = "token"
	ImplicitGrantRedirectURL = "token_redirect_url"
)

// Token Request authentication modes for AuthorizationCode grant type
// normal is through form parameters
// some services requires it as Basic
const (
	FormTokenAuthMode  = "form"
	BasicTokenAuthMode = "basic"
)

// RefreshToken is the refresh grant type
var RefreshToken = "refresh_token"

// ErrUnrefreshable is the error when an account type or information
// within an account does not allow refreshing it.
var ErrUnrefreshable = errors.New("this account can not be refreshed")

// AccountType holds configuration information for
type AccountType struct {
	DocID                 string            `json:"_id,omitempty"`
	DocRev                string            `json:"_rev,omitempty"`
	GrantMode             string            `json:"grant_mode,omitempty"`
	ClientID              string            `json:"client_id,omitempty"`
	ClientSecret          string            `json:"client_secret,omitempty"`
	AuthEndpoint          string            `json:"auth_endpoint,omitempty"`
	TokenEndpoint         string            `json:"token_endpoint,omitempty"`
	TokenAuthMode         string            `json:"token_mode,omitempty"`
	RegisteredRedirectURI string            `json:"redirect_uri,omitempty"`
	ExtraAuthQuery        map[string]string `json:"extras,omitempty"`
}

// ID is used to implement the couchdb.Doc interface
func (at *AccountType) ID() string { return at.DocID }

// Rev is used to implement the couchdb.Doc interface
func (at *AccountType) Rev() string { return at.DocRev }

// SetID is used to implement the couchdb.Doc interface
func (at *AccountType) SetID(id string) { at.DocID = id }

// SetRev is used to implement the couchdb.Doc interface
func (at *AccountType) SetRev(rev string) { at.DocRev = rev }

// DocType implements couchdb.Doc
func (at *AccountType) DocType() string { return consts.AccountTypes }

// Clone implements couchdb.Doc
func (at *AccountType) Clone() couchdb.Doc {
	cloned := *at
	cloned.ExtraAuthQuery = make(map[string]string)
	for k, v := range at.ExtraAuthQuery {
		cloned.ExtraAuthQuery[k] = v
	}
	return &cloned
}

// ensure AccountType implements couchdb.Doc
var _ couchdb.Doc = (*AccountType)(nil)

type tokenEndpointResponse struct {
	RefreshToken     string `json:"refresh_token"`
	AccessToken      string `json:"access_token"`
	IDToken          string `json:"id_token"` // alternative name for access_token
	ExpiresIn        int    `json:"expires_in"`
	TokenType        string `json:"token_type"`
	Error            string `json:"error"`
	ErrorDescription string `json:"error_description"`
}

// RedirectURI returns the redirectURI for an account,
// it can be either the
func (at *AccountType) RedirectURI(i *instance.Instance) string {
	redirectURI := i.PageURL("/accounts/"+at.ID()+"/redirect", nil)
	if at.RegisteredRedirectURI != "" {
		redirectURI = at.RegisteredRedirectURI
	}
	return redirectURI
}

// MakeOauthStartURL returns the url at which direct the user to start
// the oauth flow
func (at *AccountType) MakeOauthStartURL(i *instance.Instance, scope string, state string) (string, error) {

	u, err := url.Parse(at.AuthEndpoint)
	if err != nil {
		return "", err
	}
	vv := u.Query()

	redirectURI := at.RedirectURI(i)

	switch at.GrantMode {
	case AuthorizationCode:
		vv.Add("scope", scope)
		vv.Add("response_type", "code")
		vv.Add("client_id", at.ClientID)
		vv.Add("state", state)
		vv.Add("redirect_uri", redirectURI)
	case ImplicitGrant:
		vv.Add("scope", scope)
		vv.Add("response_type", "token")
		vv.Add("client_id", at.ClientID)
		vv.Add("state", state)
		vv.Add("redirect_uri", redirectURI)
	case ImplicitGrantRedirectURL:
		vv.Add("scope", scope)
		vv.Add("response_type", "token")
		vv.Add("state", state)
		vv.Add("redirect_url", redirectURI)
	default:
		return "", errors.New("Wrong account type")
	}

	for k, v := range at.ExtraAuthQuery {
		vv.Add(k, v)
	}

	u.RawQuery = vv.Encode()
	return u.String(), nil

}

// RequestAccessToken asks the service an access token
// https://tools.ietf.org/html/rfc6749#section-4
func (at *AccountType) RequestAccessToken(i *instance.Instance, accessCode, stateCode, stateNonce string) (*Account, error) {
	data := url.Values{
		"grant_type":   []string{AuthorizationCode},
		"code":         []string{accessCode},
		"redirect_uri": []string{at.RedirectURI(i)},
		"state":        []string{stateCode},
		"nonce":        []string{stateNonce},
	}

	if at.TokenAuthMode != BasicTokenAuthMode {
		data.Add("client_id", at.ClientID)
		data.Add("client_secret", at.ClientSecret)
	}

	body := data.Encode()
	req, err := http.NewRequest("POST", at.TokenEndpoint, strings.NewReader(body))
	if err != nil {
		return nil, err
	}

	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	req.Header.Add("Accept", "application/json")

	if at.TokenAuthMode == BasicTokenAuthMode {
		auth := []byte(at.ClientID + ":" + at.ClientSecret)
		req.Header.Add("Authorization", "Basic "+base64.StdEncoding.EncodeToString(auth))
	}

	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	resBody, err := ioutil.ReadAll(res.Body)
	if res.StatusCode != 200 {
		return nil, errors.New("oauth services responded with non-200 res: " + string(resBody))
	}
	if err != nil {
		return nil, err
	}

	var out struct {
		RefreshToken     string `json:"refresh_token"`
		AccessToken      string `json:"access_token"`
		IDToken          string `json:"id_token"` // alternative name for access_token
		ExpiresIn        int    `json:"expires_in"`
		TokenType        string `json:"token_type"`
		Error            string `json:"error"`
		ErrorDescription string `json:"error_description"`
	}
	err = json.Unmarshal(resBody, &out)
	if err != nil {
		return nil, err
	}
	if out.Error != "" {
		return nil, fmt.Errorf("OauthError(%s) %s", out.Error, out.ErrorDescription)
	}

	var ExpiresAt time.Time
	if out.ExpiresIn != 0 {
		ExpiresAt = time.Now().Add(time.Duration(out.ExpiresIn) * time.Second)
	}

	account := &Account{
		AccountType: at.ID(),
		Oauth:       &OauthInfo{ExpiresAt: ExpiresAt},
	}

	if out.AccessToken == "" {
		out.AccessToken = out.IDToken
	}

	if out.AccessToken == "" {
		return nil, errors.New("server responded without access token")
	}

	account.Oauth.AccessToken = out.AccessToken
	account.Oauth.RefreshToken = out.RefreshToken
	account.Oauth.TokenType = out.TokenType

	// decode same resBody into a map for non-standard fields
	var extras map[string]interface{}
	json.Unmarshal(resBody, &extras)
	delete(extras, "access_token")
	delete(extras, "refresh_token")
	delete(extras, "token_type")
	delete(extras, "expires_in")

	if len(extras) > 0 {
		account.Extras = extras
	}

	return account, nil
}

// RefreshAccount requires a new AccessToken using the RefreshToken
// as specified in https://tools.ietf.org/html/rfc6749#section-6
func (at *AccountType) RefreshAccount(a Account) error {

	if a.Oauth == nil {
		return ErrUnrefreshable
	}

	// If no endpoint is specified for the account type, the stack just sends
	// the client ID and client secret to the konnector and let it fetch the
	// token its-self.
	if a.Oauth.RefreshToken == "" {
		a.Oauth.ClientID = at.ClientID
		a.Oauth.ClientSecret = at.ClientSecret
		return nil
	}

	res, err := http.PostForm(at.TokenEndpoint, url.Values{
		"grant_type":    []string{RefreshToken},
		"refresh_token": []string{a.Oauth.RefreshToken},
		"client_id":     []string{at.ClientID},
		"client_secret": []string{at.ClientSecret},
	})

	if err != nil {
		return err
	}

	if res.StatusCode != 200 {
		return errors.New("oauth services responded with non-200 res")
	}

	var out tokenEndpointResponse
	err = json.NewDecoder(res.Body).Decode(&out)
	if err != nil {
		return err
	}

	if out.Error != "" {
		return fmt.Errorf("OauthError(%s) %s", out.Error, out.ErrorDescription)
	}

	if out.AccessToken != "" {
		a.Oauth.AccessToken = out.AccessToken
	}

	if out.ExpiresIn != 0 {
		a.Oauth.ExpiresAt = time.Now().Add(time.Duration(out.ExpiresIn) * time.Second)
	}

	if out.RefreshToken != "" {
		a.Oauth.RefreshToken = out.RefreshToken
	}

	return nil
}

// TypeInfo returns the AccountType document for a given id
func TypeInfo(id string) (*AccountType, error) {
	if id == "" {
		return nil, errors.New("no account type id provided")
	}
	var a AccountType
	err := couchdb.GetDoc(couchdb.GlobalSecretsDB, consts.AccountTypes, id, &a)
	if err != nil {
		return nil, err
	}
	return &a, nil
}
<commit_msg>Give more info when OAuth refresh for konnectors fails<commit_after>package accounts

import (
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/instance"
)

// This file contains the account_type object as defined in
// docs/konnectors_oauth

// Various grant types
// - AuthorizationCode is the server-side grant type.
// - ImplicitGrant is the implicit grant type
// - ImplicitGrantRedirectURL is the implicit grant type but with redirect_url
//    											  instead of redirect_uri
const (
	AuthorizationCode        = "authorization_code"
	ImplicitGrant            = "token"
	ImplicitGrantRedirectURL = "token_redirect_url"
)

// Token Request authentication modes for AuthorizationCode grant type
// normal is through form parameters
// some services requires it as Basic
const (
	FormTokenAuthMode  = "form"
	BasicTokenAuthMode = "basic"
)

// RefreshToken is the refresh grant type
var RefreshToken = "refresh_token"

// ErrUnrefreshable is the error when an account type or information
// within an account does not allow refreshing it.
var ErrUnrefreshable = errors.New("this account can not be refreshed")

// AccountType holds configuration information for
type AccountType struct {
	DocID                 string            `json:"_id,omitempty"`
	DocRev                string            `json:"_rev,omitempty"`
	GrantMode             string            `json:"grant_mode,omitempty"`
	ClientID              string            `json:"client_id,omitempty"`
	ClientSecret          string            `json:"client_secret,omitempty"`
	AuthEndpoint          string            `json:"auth_endpoint,omitempty"`
	TokenEndpoint         string            `json:"token_endpoint,omitempty"`
	TokenAuthMode         string            `json:"token_mode,omitempty"`
	RegisteredRedirectURI string            `json:"redirect_uri,omitempty"`
	ExtraAuthQuery        map[string]string `json:"extras,omitempty"`
}

// ID is used to implement the couchdb.Doc interface
func (at *AccountType) ID() string { return at.DocID }

// Rev is used to implement the couchdb.Doc interface
func (at *AccountType) Rev() string { return at.DocRev }

// SetID is used to implement the couchdb.Doc interface
func (at *AccountType) SetID(id string) { at.DocID = id }

// SetRev is used to implement the couchdb.Doc interface
func (at *AccountType) SetRev(rev string) { at.DocRev = rev }

// DocType implements couchdb.Doc
func (at *AccountType) DocType() string { return consts.AccountTypes }

// Clone implements couchdb.Doc
func (at *AccountType) Clone() couchdb.Doc {
	cloned := *at
	cloned.ExtraAuthQuery = make(map[string]string)
	for k, v := range at.ExtraAuthQuery {
		cloned.ExtraAuthQuery[k] = v
	}
	return &cloned
}

// ensure AccountType implements couchdb.Doc
var _ couchdb.Doc = (*AccountType)(nil)

type tokenEndpointResponse struct {
	RefreshToken     string `json:"refresh_token"`
	AccessToken      string `json:"access_token"`
	IDToken          string `json:"id_token"` // alternative name for access_token
	ExpiresIn        int    `json:"expires_in"`
	TokenType        string `json:"token_type"`
	Error            string `json:"error"`
	ErrorDescription string `json:"error_description"`
}

// RedirectURI returns the redirectURI for an account,
// it can be either the
func (at *AccountType) RedirectURI(i *instance.Instance) string {
	redirectURI := i.PageURL("/accounts/"+at.ID()+"/redirect", nil)
	if at.RegisteredRedirectURI != "" {
		redirectURI = at.RegisteredRedirectURI
	}
	return redirectURI
}

// MakeOauthStartURL returns the url at which direct the user to start
// the oauth flow
func (at *AccountType) MakeOauthStartURL(i *instance.Instance, scope string, state string) (string, error) {

	u, err := url.Parse(at.AuthEndpoint)
	if err != nil {
		return "", err
	}
	vv := u.Query()

	redirectURI := at.RedirectURI(i)

	switch at.GrantMode {
	case AuthorizationCode:
		vv.Add("scope", scope)
		vv.Add("response_type", "code")
		vv.Add("client_id", at.ClientID)
		vv.Add("state", state)
		vv.Add("redirect_uri", redirectURI)
	case ImplicitGrant:
		vv.Add("scope", scope)
		vv.Add("response_type", "token")
		vv.Add("client_id", at.ClientID)
		vv.Add("state", state)
		vv.Add("redirect_uri", redirectURI)
	case ImplicitGrantRedirectURL:
		vv.Add("scope", scope)
		vv.Add("response_type", "token")
		vv.Add("state", state)
		vv.Add("redirect_url", redirectURI)
	default:
		return "", errors.New("Wrong account type")
	}

	for k, v := range at.ExtraAuthQuery {
		vv.Add(k, v)
	}

	u.RawQuery = vv.Encode()
	return u.String(), nil

}

// RequestAccessToken asks the service an access token
// https://tools.ietf.org/html/rfc6749#section-4
func (at *AccountType) RequestAccessToken(i *instance.Instance, accessCode, stateCode, stateNonce string) (*Account, error) {
	data := url.Values{
		"grant_type":   []string{AuthorizationCode},
		"code":         []string{accessCode},
		"redirect_uri": []string{at.RedirectURI(i)},
		"state":        []string{stateCode},
		"nonce":        []string{stateNonce},
	}

	if at.TokenAuthMode != BasicTokenAuthMode {
		data.Add("client_id", at.ClientID)
		data.Add("client_secret", at.ClientSecret)
	}

	body := data.Encode()
	req, err := http.NewRequest("POST", at.TokenEndpoint, strings.NewReader(body))
	if err != nil {
		return nil, err
	}

	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	req.Header.Add("Accept", "application/json")

	if at.TokenAuthMode == BasicTokenAuthMode {
		auth := []byte(at.ClientID + ":" + at.ClientSecret)
		req.Header.Add("Authorization", "Basic "+base64.StdEncoding.EncodeToString(auth))
	}

	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	resBody, err := ioutil.ReadAll(res.Body)
	if res.StatusCode != 200 {
		return nil, errors.New("oauth services responded with non-200 res: " + string(resBody))
	}
	if err != nil {
		return nil, err
	}

	var out struct {
		RefreshToken     string `json:"refresh_token"`
		AccessToken      string `json:"access_token"`
		IDToken          string `json:"id_token"` // alternative name for access_token
		ExpiresIn        int    `json:"expires_in"`
		TokenType        string `json:"token_type"`
		Error            string `json:"error"`
		ErrorDescription string `json:"error_description"`
	}
	err = json.Unmarshal(resBody, &out)
	if err != nil {
		return nil, err
	}
	if out.Error != "" {
		return nil, fmt.Errorf("OauthError(%s) %s", out.Error, out.ErrorDescription)
	}

	var ExpiresAt time.Time
	if out.ExpiresIn != 0 {
		ExpiresAt = time.Now().Add(time.Duration(out.ExpiresIn) * time.Second)
	}

	account := &Account{
		AccountType: at.ID(),
		Oauth:       &OauthInfo{ExpiresAt: ExpiresAt},
	}

	if out.AccessToken == "" {
		out.AccessToken = out.IDToken
	}

	if out.AccessToken == "" {
		return nil, errors.New("server responded without access token")
	}

	account.Oauth.AccessToken = out.AccessToken
	account.Oauth.RefreshToken = out.RefreshToken
	account.Oauth.TokenType = out.TokenType

	// decode same resBody into a map for non-standard fields
	var extras map[string]interface{}
	json.Unmarshal(resBody, &extras)
	delete(extras, "access_token")
	delete(extras, "refresh_token")
	delete(extras, "token_type")
	delete(extras, "expires_in")

	if len(extras) > 0 {
		account.Extras = extras
	}

	return account, nil
}

// RefreshAccount requires a new AccessToken using the RefreshToken
// as specified in https://tools.ietf.org/html/rfc6749#section-6
func (at *AccountType) RefreshAccount(a Account) error {

	if a.Oauth == nil {
		return ErrUnrefreshable
	}

	// If no endpoint is specified for the account type, the stack just sends
	// the client ID and client secret to the konnector and let it fetch the
	// token its-self.
	if a.Oauth.RefreshToken == "" {
		a.Oauth.ClientID = at.ClientID
		a.Oauth.ClientSecret = at.ClientSecret
		return nil
	}

	res, err := http.PostForm(at.TokenEndpoint, url.Values{
		"grant_type":    []string{RefreshToken},
		"refresh_token": []string{a.Oauth.RefreshToken},
		"client_id":     []string{at.ClientID},
		"client_secret": []string{at.ClientSecret},
	})

	if err != nil {
		return err
	}

	if res.StatusCode != 200 {
		resBody, _ := ioutil.ReadAll(res.Body)
		return errors.New("oauth services responded with non-200 res: " + string(resBody))
	}

	var out tokenEndpointResponse
	err = json.NewDecoder(res.Body).Decode(&out)
	if err != nil {
		return err
	}

	if out.Error != "" {
		return fmt.Errorf("OauthError(%s) %s", out.Error, out.ErrorDescription)
	}

	if out.AccessToken != "" {
		a.Oauth.AccessToken = out.AccessToken
	}

	if out.ExpiresIn != 0 {
		a.Oauth.ExpiresAt = time.Now().Add(time.Duration(out.ExpiresIn) * time.Second)
	}

	if out.RefreshToken != "" {
		a.Oauth.RefreshToken = out.RefreshToken
	}

	return nil
}

// TypeInfo returns the AccountType document for a given id
func TypeInfo(id string) (*AccountType, error) {
	if id == "" {
		return nil, errors.New("no account type id provided")
	}
	var a AccountType
	err := couchdb.GetDoc(couchdb.GlobalSecretsDB, consts.AccountTypes, id, &a)
	if err != nil {
		return nil, err
	}
	return &a, nil
}
<|endoftext|>


<commit_before>// Copyright 2019 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package build

import (
	"fmt"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/google/syzkaller/pkg/osutil"
)

type freebsd struct{}

func (ctx freebsd) build(params *Params) error {
	confDir := fmt.Sprintf("%v/sys/%v/conf/", params.KernelDir, params.TargetArch)
	confFile := "SYZKALLER"

	config := params.Config
	if config == nil {
		config = []byte(`
include "./GENERIC"

ident		SYZKALLER
options 	COVERAGE
options 	KCOV

options 	DEBUG_VFS_LOCKS
options 	DIAGNOSTIC
`)
	}
	if err := osutil.WriteFile(filepath.Join(confDir, confFile), config); err != nil {
		return err
	}

	objPrefix := filepath.Join(params.KernelDir, "obj")
	if err := ctx.make(params.KernelDir, objPrefix, "kernel-toolchain", "-DNO_CLEAN"); err != nil {
		return err
	}
	if err := ctx.make(params.KernelDir, objPrefix, "buildkernel", fmt.Sprintf("KERNCONF=%v", confFile)); err != nil {
		return err
	}

	kernelObjDir := filepath.Join(objPrefix, params.KernelDir,
		fmt.Sprintf("%v.%v", params.TargetArch, params.TargetArch), "sys", confFile)
	for _, s := range []struct{ dir, src, dst string }{
		{params.UserspaceDir, "image", "image"},
		{params.UserspaceDir, "key", "key"},
		{kernelObjDir, "kernel.full", "obj/kernel.full"},
	} {
		fullSrc := filepath.Join(s.dir, s.src)
		fullDst := filepath.Join(params.OutputDir, s.dst)
		if err := osutil.CopyFile(fullSrc, fullDst); err != nil {
			return fmt.Errorf("failed to copy %v -> %v: %v", fullSrc, fullDst, err)
		}
	}

	script := fmt.Sprintf(`
set -eux
md=$(sudo mdconfig -a -t vnode image)
partn=$(gpart show /dev/${md} | awk '/freebsd-ufs/{print $3}' | head -n 1)
tmpdir=$(mktemp -d)
sudo mount /dev/${md}p${partn} $tmpdir

sudo MAKEOBJDIRPREFIX=%s make -C %s installkernel KERNCONF=%s DESTDIR=$tmpdir

echo 'pf_load="YES"' | sudo tee -a ${tmpdir}/boot/loader.conf

sudo umount $tmpdir
sudo mdconfig -d -u ${md#md}
`, objPrefix, params.KernelDir, confFile)

	if debugOut, err := osutil.RunCmd(10*time.Minute, params.OutputDir, "/bin/sh", "-c", script); err != nil {
		return fmt.Errorf("error copying kernel: %v\n%v", err, debugOut)
	}
	return nil
}

func (ctx freebsd) clean(kernelDir, targetArch string) error {
	objPrefix := filepath.Join(kernelDir, "obj")
	return ctx.make(kernelDir, objPrefix, "cleanworld")
}

func (ctx freebsd) make(kernelDir string, objPrefix string, makeArgs ...string) error {
	args := append([]string{
		fmt.Sprintf("MAKEOBJDIRPREFIX=%v", objPrefix),
		"make",
		"-C", kernelDir,
		"-j", strconv.Itoa(runtime.NumCPU()),
	}, makeArgs...)
	_, err := osutil.RunCmd(3*time.Hour, kernelDir, "sh", "-c", strings.Join(args, " "))
	return err
}
<commit_msg>pkg/build: enable extra TCP stacks on FreeBSD<commit_after>// Copyright 2019 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package build

import (
	"fmt"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/google/syzkaller/pkg/osutil"
)

type freebsd struct{}

func (ctx freebsd) build(params *Params) error {
	confDir := fmt.Sprintf("%v/sys/%v/conf/", params.KernelDir, params.TargetArch)
	confFile := "SYZKALLER"

	config := params.Config
	if config == nil {
		config = []byte(`
include "./GENERIC"

ident		SYZKALLER
options 	COVERAGE
options 	KCOV

options 	KERN_TLS
options 	TCPHPTS

options 	DEBUG_VFS_LOCKS
options 	DIAGNOSTIC
`)
	}
	if err := osutil.WriteFile(filepath.Join(confDir, confFile), config); err != nil {
		return err
	}

	objPrefix := filepath.Join(params.KernelDir, "obj")
	if err := ctx.make(params.KernelDir, objPrefix, "kernel-toolchain", "-DNO_CLEAN"); err != nil {
		return err
	}
	if err := ctx.make(params.KernelDir, objPrefix, "buildkernel", "WITH_EXTRA_TCP_STACKS=",
		fmt.Sprintf("KERNCONF=%v", confFile)); err != nil {
		return err
	}

	kernelObjDir := filepath.Join(objPrefix, params.KernelDir,
		fmt.Sprintf("%v.%v", params.TargetArch, params.TargetArch), "sys", confFile)
	for _, s := range []struct{ dir, src, dst string }{
		{params.UserspaceDir, "image", "image"},
		{params.UserspaceDir, "key", "key"},
		{kernelObjDir, "kernel.full", "obj/kernel.full"},
	} {
		fullSrc := filepath.Join(s.dir, s.src)
		fullDst := filepath.Join(params.OutputDir, s.dst)
		if err := osutil.CopyFile(fullSrc, fullDst); err != nil {
			return fmt.Errorf("failed to copy %v -> %v: %v", fullSrc, fullDst, err)
		}
	}

	script := fmt.Sprintf(`
set -eux
md=$(sudo mdconfig -a -t vnode image)
partn=$(gpart show /dev/${md} | awk '/freebsd-ufs/{print $3}' | head -n 1)
tmpdir=$(mktemp -d)
sudo mount /dev/${md}p${partn} $tmpdir

sudo MAKEOBJDIRPREFIX=%s make -C %s installkernel WITH_EXTRA_TCP_STACKS= KERNCONF=%s DESTDIR=$tmpdir

cat | sudo tee -a ${tmpdir}/boot/loader.conf <<__EOF__
pf_load="YES"
tcp_bbr_load="YES"
tcp_rack_load="YES"
__EOF__

sudo umount $tmpdir
sudo mdconfig -d -u ${md#md}
`, objPrefix, params.KernelDir, confFile)

	if debugOut, err := osutil.RunCmd(10*time.Minute, params.OutputDir, "/bin/sh", "-c", script); err != nil {
		return fmt.Errorf("error copying kernel: %v\n%v", err, debugOut)
	}
	return nil
}

func (ctx freebsd) clean(kernelDir, targetArch string) error {
	objPrefix := filepath.Join(kernelDir, "obj")
	return ctx.make(kernelDir, objPrefix, "cleanworld")
}

func (ctx freebsd) make(kernelDir string, objPrefix string, makeArgs ...string) error {
	args := append([]string{
		fmt.Sprintf("MAKEOBJDIRPREFIX=%v", objPrefix),
		"make",
		"-C", kernelDir,
		"-j", strconv.Itoa(runtime.NumCPU()),
	}, makeArgs...)
	_, err := osutil.RunCmd(3*time.Hour, kernelDir, "sh", "-c", strings.Join(args, " "))
	return err
}
<|endoftext|>


<commit_before>package captainslog

import "errors"

var (
	// ErrTransform is returned by a Transformer when it cannot
	// perform its function
	ErrTransform = errors.New("transform error")
)

// Transformer accept a SyslogMsg, and return a modified SyslogMsg
type Transformer interface {
	Transform(SyslogMsg) (SyslogMsg, error)
}

// Matcher accepts a SyslogMsg and returns true of it matches
type Matcher interface {
	Match(msg *SyslogMsg) bool
}

// OutputAdapter is an interface for adapters that provide
// specific functionality to OutputChannelers. They
// are transport adapters - for instance, TCPOutputAdapter
// converts *Syslog messages received off a channeler
// to RFC3164 []byte encoded syslog messages sent over TCP.
type OutputAdapter interface {
	Output(s *SyslogMsg) (int, error)
	Connect() error
	RetryInterval() int
	Close()
}
<commit_msg>Problem: typo in comment<commit_after>package captainslog

import "errors"

var (
	// ErrTransform is returned by a Transformer when it cannot
	// perform its function
	ErrTransform = errors.New("transform error")
)

// Transformer accept a SyslogMsg, and return a modified SyslogMsg
type Transformer interface {
	Transform(SyslogMsg) (SyslogMsg, error)
}

// Matcher accepts a SyslogMsg and returns true if it matches
type Matcher interface {
	Match(msg *SyslogMsg) bool
}

// OutputAdapter is an interface for adapters that provide
// specific functionality to OutputChannelers. They
// are transport adapters - for instance, TCPOutputAdapter
// converts *Syslog messages received off a channeler
// to RFC3164 []byte encoded syslog messages sent over TCP.
type OutputAdapter interface {
	Output(s *SyslogMsg) (int, error)
	Connect() error
	RetryInterval() int
	Close()
}
<|endoftext|>


<commit_before>package dai

import (
	r "github.com/dancannon/gorethink"
	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/db/model"
	"github.com/materials-commons/mcstore/pkg/db/schema"
)

// rFiles implements the Files interface for RethinkDB
type rFiles struct {
	session *r.Session
}

// NewRFiles creates a new instance of rFiles
func NewRFiles(session *r.Session) rFiles {
	return rFiles{
		session: session,
	}
}

// ByID looks up a file by its primary key. In RethinkDB this is the id field.
func (f rFiles) ByID(id string) (*schema.File, error) {
	var file schema.File
	if err := model.Files.Qs(f.session).ByID(id, &file); err != nil {
		return nil, err
	}
	return &file, nil
}

// ByChecksum looks up a file by its checksum. This routine only returns the original
// root entry, it will not return entries that are duplicates and point at the root.
func (f rFiles) ByChecksum(checksum string) (*schema.File, error) {
	rql := model.Files.T().GetAllByIndex("checksum", checksum).Filter(r.Row.Field("usesid").Eq(""))
	var file schema.File
	if err := model.Files.Qs(f.session).Row(rql, &file); err != nil {
		return nil, err
	}
	return &file, nil
}

// ByPath looks up a file by its name in a specific directory. It only returns the
// current file, not hidden files.
func (f rFiles) ByPath(name, dirID string) (*schema.File, error) {
	rql := r.Table("datadir2datafile").GetAllByIndex("datadir_id", dirID).
		EqJoin("datafile_id", r.Table("datafiles")).
		Zip().
		Filter(r.Row.Field("current").Eq(true).And(r.Row.Field("name").Eq(name)))
	var file schema.File
	if err := model.Files.Qs(f.session).Row(rql, &file); err != nil {
		return nil, err
	}
	return &file, nil
}

// Insert adds a new file to the system.
func (f rFiles) Insert(file *schema.File, dirID string, projectID string) (*schema.File, error) {
	var newFile schema.File
	if err := model.Files.Qs(f.session).Insert(file, &newFile); err != nil {
		return nil, err
	}

	err := f.updateDependencies(newFile.ID, dirID, projectID)
	return &newFile, err
}

// updateDependencies updates the datadir2datafile and project2datafile tables.
func (f rFiles) updateDependencies(fileID, dirID, projectID string) error {
	proj2file := schema.Project2DataFile{
		ProjectID:  projectID,
		DataFileID: fileID,
	}

	dir2file := schema.DataDir2DataFile{
		DataDirID:  dirID,
		DataFileID: fileID,
	}

	// We will return one of the errors, even if both inserts failed. There really isn't anything the
	// system can do about a specific error, so we just want to communicate that a failure occured.
	err := model.DirFiles.Qs(f.session).Insert(&dir2file, nil)
	if err != nil {
		app.Log.Error(app.Logf("Unable to update datadir2datafile for file %s, directory %s, error %s", fileID, dirID, err))
	}

	err = model.ProjectFiles.Qs(f.session).Insert(&proj2file, nil)
	if err != nil {
		app.Log.Error(app.Logf("Unable to update project2datafile for file %s, project %s, error %s", fileID, projectID, err))
	}

	return err
}

// Delete will delete the file from the directory and project. It will remove
// the file entry if the file isn't in any other directories or projects, or
// if it isn't referred to by other files (via usesid). If the file is no longer
// in a project and directory, but it is referenced by a usesid then it will remain
// in place as a disconnected file. In this case its current flag will be set to false.
// If you are deleting a file that has a parentid, then the parent will be set to
// current. This method will attempt to clean up as much as possible even in the face
// of errors. If there are any errors it will return the first error. It is the calling
// routines duty to figure out what steps could not be performed.
func (f rFiles) Delete(fileID, directoryID, projectID string) (*schema.File, error) {
	var firstError error // Keep the first error around

	// Get all projects this file is in
	projects, err := f.getProjects(fileID)
	if err != nil {
		firstError = err
	}

	// Get all directories this file is in
	dirs, err := f.getDirs(fileID)
	if err != nil && firstError == nil {
		firstError = err
	}

	// Delete file from directory
	err = f.deleteFromDir(fileID, directoryID)
	if err != nil && firstError == nil {
		firstError = err
	}

	file, err := f.ByID(fileID)
	if err != nil && firstError == nil {
		// Ugh, if we can't get the file then there
		// is nothing else we can do.
		return nil, err
	}

	filesUsedBy, err := f.getUsedBy(fileID)

	// We have deleted the file from one directory. At this
	// point if len(dirs) == 1 and len(filesUsedBy) == 0 and
	// len(projects) == 1 we can delete the entry.
	//
	// TODO: This logic is actually a lot more complicated when
	// files start being shared across projects and if we also
	// allow a file to be shared across directories in a project.
	// Right now across directories (in a project) isn't supported
	// but across projects is something we want to support.
	switch {
	case len(dirs) == 1 && len(projects) == 1 && len(filesUsedBy) == 0:
		model.Files.Qs(f.session).Delete(fileID)
	case len(projects) == 1 && len(dirs) == 1:
		// delete from project
		f.deleteFromProject(fileID, projectID)
	case file.Current:
		// File is referenced by somebody, so just mark it as
		// not current, since we cannot delete it.
		file.Current = false
		f.Update(file)
	}

	if file.Parent != "" {
		fields := map[string]interface{}{
			schema.FileFields.Current(): true,
		}
		f.UpdateFields(file.Parent, fields)
	}

	return file, firstError
}

// getProjects returns a list of all the projects containing this fileID.
func (f rFiles) getProjects(fileID string) ([]schema.Project2DataFile, error) {
	rql := model.ProjectFiles.T().GetAllByIndex("datafile_id", fileID)
	var projects []schema.Project2DataFile
	if err := model.ProjectFiles.Qs(f.session).Rows(rql, &projects); err != nil {
		return nil, err
	}
	return projects, nil
}

// getDirs returns a list of all the directories containing this fileID.
func (f rFiles) getDirs(fileID string) ([]schema.DataDir2DataFile, error) {
	rql := model.DirFiles.T().GetAllByIndex("datafile_id", fileID)
	var dirs []schema.DataDir2DataFile
	if err := model.DirFiles.Qs(f.session).Rows(rql, &dirs); err != nil {
		return nil, err
	}
	return dirs, nil
}

// getUsedBy returns all the files that point at this file.
func (f rFiles) getUsedBy(fileID string) ([]schema.File, error) {
	rql := model.Files.T().GetAllByIndex("usesid", fileID)
	var files []schema.File
	if err := model.Files.Qs(f.session).Rows(rql, &files); err != nil {
		return nil, err
	}
	return files, nil
}

// deleteFromDir will delete the given file from the directory.
func (f rFiles) deleteFromDir(fileID, directoryID string) error {
	rql := model.DirFiles.T().GetAllByIndex("datafile_id", fileID).
		Filter(r.Row.Field("datadir_id").Eq(directoryID)).Delete()
	rv, err := rql.RunWrite(f.session)
	switch {
	case err != nil:
		return err
	case rv.Errors != 0:
		return app.ErrNotFound
	case rv.Deleted == 0:
		return app.ErrNotFound
	default:
		return nil
	}
}

// deleteFromProject will delete the given file from the project.
func (f rFiles) deleteFromProject(fileID, projectID string) error {
	rql := model.ProjectFiles.T().GetAllByIndex("datafile_id", fileID).
		Filter(r.Row.Field("project_id").Eq(projectID)).Delete()
	rv, err := rql.RunWrite(f.session)
	switch {
	case err != nil:
		return err
	case rv.Errors != 0:
		return app.ErrNotFound
	case rv.Deleted == 0:
		return app.ErrNotFound
	default:
		return nil
	}
}

// Update updates an existing datafile.
func (f rFiles) Update(file *schema.File) error {
	if err := model.Files.Qs(f.session).Update(file.ID, file); err != nil {
		return err
	}
	return nil
}

// UpdateFields updates the fields for the given id
func (f rFiles) UpdateFields(fileID string, fields map[string]interface{}) error {
	if err := model.Files.Qs(f.session).Update(fileID, fields); err != nil {
		return err
	}
	return nil
}

func (f rFiles) GetProject(fileID string) (*schema.Project, error) {
	rql := r.Table("datadir2datafile").
		GetAllByIndex("datafile_id", fileID).
		EqJoin("datadir_id", r.Table("project2datadir"), r.EqJoinOpts{Index: "datadir_id"}).
		Zip().
		EqJoin("project_id", r.Table("projects")).
		Zip()
	var projects []schema.Project
	if err := model.Files.Qs(f.session).Rows(rql, &projects); err != nil {
		return nil, err
	}
	if len(projects) == 0 {
		return nil, app.ErrNotFound
	}
	// TODO: Fix this, only returns the first project
	return &projects[0], nil
}
<commit_msg>Use the correct schema.<commit_after>package dai

import (
	r "github.com/dancannon/gorethink"
	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/db/model"
	"github.com/materials-commons/mcstore/pkg/db/schema"
)

// rFiles implements the Files interface for RethinkDB
type rFiles struct {
	session *r.Session
}

// NewRFiles creates a new instance of rFiles
func NewRFiles(session *r.Session) rFiles {
	return rFiles{
		session: session,
	}
}

// ByID looks up a file by its primary key. In RethinkDB this is the id field.
func (f rFiles) ByID(id string) (*schema.File, error) {
	var file schema.File
	if err := model.Files.Qs(f.session).ByID(id, &file); err != nil {
		return nil, err
	}
	return &file, nil
}

// ByChecksum looks up a file by its checksum. This routine only returns the original
// root entry, it will not return entries that are duplicates and point at the root.
func (f rFiles) ByChecksum(checksum string) (*schema.File, error) {
	rql := model.Files.T().GetAllByIndex("checksum", checksum).Filter(r.Row.Field("usesid").Eq(""))
	var file schema.File
	if err := model.Files.Qs(f.session).Row(rql, &file); err != nil {
		return nil, err
	}
	return &file, nil
}

// ByPath looks up a file by its name in a specific directory. It only returns the
// current file, not hidden files.
func (f rFiles) ByPath(name, dirID string) (*schema.File, error) {
	rql := r.Table("datadir2datafile").GetAllByIndex("datadir_id", dirID).
		EqJoin("datafile_id", r.Table("datafiles")).
		Zip().
		Filter(r.Row.Field("current").Eq(true).And(r.Row.Field("name").Eq(name)))
	var file schema.File
	if err := model.Files.Qs(f.session).Row(rql, &file); err != nil {
		return nil, err
	}
	return &file, nil
}

// Insert adds a new file to the system.
func (f rFiles) Insert(file *schema.File, dirID string, projectID string) (*schema.File, error) {
	var newFile schema.File
	if err := model.Files.Qs(f.session).Insert(file, &newFile); err != nil {
		return nil, err
	}

	err := f.updateDependencies(newFile.ID, dirID, projectID)
	return &newFile, err
}

// updateDependencies updates the datadir2datafile and project2datafile tables.
func (f rFiles) updateDependencies(fileID, dirID, projectID string) error {
	proj2file := schema.Project2DataFile{
		ProjectID:  projectID,
		DataFileID: fileID,
	}

	dir2file := schema.DataDir2DataFile{
		DataDirID:  dirID,
		DataFileID: fileID,
	}

	// We will return one of the errors, even if both inserts failed. There really isn't anything the
	// system can do about a specific error, so we just want to communicate that a failure occured.
	err := model.DirFiles.Qs(f.session).Insert(&dir2file, nil)
	if err != nil {
		app.Log.Error(app.Logf("Unable to update datadir2datafile for file %s, directory %s, error %s", fileID, dirID, err))
	}

	err = model.ProjectFiles.Qs(f.session).Insert(&proj2file, nil)
	if err != nil {
		app.Log.Error(app.Logf("Unable to update project2datafile for file %s, project %s, error %s", fileID, projectID, err))
	}

	return err
}

// Delete will delete the file from the directory and project. It will remove
// the file entry if the file isn't in any other directories or projects, or
// if it isn't referred to by other files (via usesid). If the file is no longer
// in a project and directory, but it is referenced by a usesid then it will remain
// in place as a disconnected file. In this case its current flag will be set to false.
// If you are deleting a file that has a parentid, then the parent will be set to
// current. This method will attempt to clean up as much as possible even in the face
// of errors. If there are any errors it will return the first error. It is the calling
// routines duty to figure out what steps could not be performed.
func (f rFiles) Delete(fileID, directoryID, projectID string) (*schema.File, error) {
	var firstError error // Keep the first error around

	// Get all projects this file is in
	projects, err := f.getProjects(fileID)
	if err != nil {
		firstError = err
	}

	// Get all directories this file is in
	dirs, err := f.getDirs(fileID)
	if err != nil && firstError == nil {
		firstError = err
	}

	// Delete file from directory
	err = f.deleteFromDir(fileID, directoryID)
	if err != nil && firstError == nil {
		firstError = err
	}

	file, err := f.ByID(fileID)
	if err != nil && firstError == nil {
		// Ugh, if we can't get the file then there
		// is nothing else we can do.
		return nil, err
	}

	filesUsedBy, err := f.getUsedBy(fileID)

	// We have deleted the file from one directory. At this
	// point if len(dirs) == 1 and len(filesUsedBy) == 0 and
	// len(projects) == 1 we can delete the entry.
	//
	// TODO: This logic is actually a lot more complicated when
	// files start being shared across projects and if we also
	// allow a file to be shared across directories in a project.
	// Right now across directories (in a project) isn't supported
	// but across projects is something we want to support.
	switch {
	case len(dirs) == 1 && len(projects) == 1 && len(filesUsedBy) == 0:
		model.Files.Qs(f.session).Delete(fileID)
	case len(projects) == 1 && len(dirs) == 1:
		// delete from project
		f.deleteFromProject(fileID, projectID)
	case file.Current:
		// File is referenced by somebody, so just mark it as
		// not current, since we cannot delete it.
		file.Current = false
		f.Update(file)
	}

	if file.Parent != "" {
		fields := map[string]interface{}{
			schema.FileFields.Current(): true,
		}
		f.UpdateFields(file.Parent, fields)
	}

	return file, firstError
}

// getProjects returns a list of all the projects containing this fileID.
func (f rFiles) getProjects(fileID string) ([]schema.Project2DataFile, error) {
	rql := model.ProjectFiles.T().GetAllByIndex("datafile_id", fileID)
	var projects []schema.Project2DataFile
	if err := model.ProjectFiles.Qs(f.session).Rows(rql, &projects); err != nil {
		return nil, err
	}
	return projects, nil
}

// getDirs returns a list of all the directories containing this fileID.
func (f rFiles) getDirs(fileID string) ([]schema.DataDir2DataFile, error) {
	rql := model.DirFiles.T().GetAllByIndex("datafile_id", fileID)
	var dirs []schema.DataDir2DataFile
	if err := model.DirFiles.Qs(f.session).Rows(rql, &dirs); err != nil {
		return nil, err
	}
	return dirs, nil
}

// getUsedBy returns all the files that point at this file.
func (f rFiles) getUsedBy(fileID string) ([]schema.File, error) {
	rql := model.Files.T().GetAllByIndex("usesid", fileID)
	var files []schema.File
	if err := model.Files.Qs(f.session).Rows(rql, &files); err != nil {
		return nil, err
	}
	return files, nil
}

// deleteFromDir will delete the given file from the directory.
func (f rFiles) deleteFromDir(fileID, directoryID string) error {
	rql := model.DirFiles.T().GetAllByIndex("datafile_id", fileID).
		Filter(r.Row.Field("datadir_id").Eq(directoryID)).Delete()
	rv, err := rql.RunWrite(f.session)
	switch {
	case err != nil:
		return err
	case rv.Errors != 0:
		return app.ErrNotFound
	case rv.Deleted == 0:
		return app.ErrNotFound
	default:
		return nil
	}
}

// deleteFromProject will delete the given file from the project.
func (f rFiles) deleteFromProject(fileID, projectID string) error {
	rql := model.ProjectFiles.T().GetAllByIndex("datafile_id", fileID).
		Filter(r.Row.Field("project_id").Eq(projectID)).Delete()
	rv, err := rql.RunWrite(f.session)
	switch {
	case err != nil:
		return err
	case rv.Errors != 0:
		return app.ErrNotFound
	case rv.Deleted == 0:
		return app.ErrNotFound
	default:
		return nil
	}
}

// Update updates an existing datafile.
func (f rFiles) Update(file *schema.File) error {
	if err := model.Files.Qs(f.session).Update(file.ID, file); err != nil {
		return err
	}
	return nil
}

// UpdateFields updates the fields for the given id
func (f rFiles) UpdateFields(fileID string, fields map[string]interface{}) error {
	if err := model.Files.Qs(f.session).Update(fileID, fields); err != nil {
		return err
	}
	return nil
}

func (f rFiles) GetProject(fileID string) (*schema.Project, error) {
	rql := r.Table("datadir2datafile").
		GetAllByIndex("datafile_id", fileID).
		EqJoin("datadir_id", r.Table("project2datadir"), r.EqJoinOpts{Index: "datadir_id"}).
		Zip().
		EqJoin("project_id", r.Table("projects")).
		Zip()
	var projects []schema.Project
	if err := model.Projects.Qs(f.session).Rows(rql, &projects); err != nil {
		return nil, err
	}
	if len(projects) == 0 {
		return nil, app.ErrNotFound
	}
	// TODO: Fix this, only returns the first project
	return &projects[0], nil
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package loader has a data loading interface and various implementations.
package loader

import (
	"sigs.k8s.io/kustomize/pkg/fs"
	"sigs.k8s.io/kustomize/pkg/ifc"
)

const useHashiCorpCloner = true

func clonerToUse() gitCloner {
	if useHashiCorpCloner {
		return hashicorpGitCloner
	}
	return simpleGitCloner
}

// var clonerToUse = simpleGitCloner

// NewLoader returns a Loader.
func NewLoader(root string, fSys fs.FileSystem) (ifc.Loader, error) {
	if isRepoUrl(root) {
		return newGitLoader(
			root, fSys, []string{}, clonerToUse())
	}
	return newFileLoaderAt(
		root, fSys, []string{}, clonerToUse())
}
<commit_msg>Remove stray comment.<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package loader has a data loading interface and various implementations.
package loader

import (
	"sigs.k8s.io/kustomize/pkg/fs"
	"sigs.k8s.io/kustomize/pkg/ifc"
)

const useHashiCorpCloner = true

func clonerToUse() gitCloner {
	if useHashiCorpCloner {
		return hashicorpGitCloner
	}
	return simpleGitCloner
}

// NewLoader returns a Loader.
func NewLoader(root string, fSys fs.FileSystem) (ifc.Loader, error) {
	if isRepoUrl(root) {
		return newGitLoader(
			root, fSys, []string{}, clonerToUse())
	}
	return newFileLoaderAt(
		root, fSys, []string{}, clonerToUse())
}
<|endoftext|>