<commit_before>/*
Copyright IBM Corp 2016 All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
		 http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"errors"
	"fmt"
	"strconv"
	"time"
	"encoding/json"
	"github.com/hyperledger/fabric/core/chaincode/shim"
)

// BienChaincode is  a Chaincode for bien application implementation
type BienChaincode struct {
}
var orderIndexStr ="_orderindex"

type Bien struct{
		id int64 `json:"orderId"`
		name string `json:"name"`
		state string `json:"state"`
		price int `json:"price"`
		postage int `json:"postage"`
		owner string `json:"owner"`
}
var logger = shim.NewLogger("SimpleChaincode")
func main() {
    logger.SetLevel(shim.LogInfo)
	err := shim.Start(new(BienChaincode))
	if err != nil {
		fmt.Printf("Error starting BienChaincode chaincode: %s", err)
	}
}

// Init resets all the things
func (t *BienChaincode) Init(stub *shim.ChaincodeStub, function string, args []string) ([]byte, error) {
	fmt.Printf("hello init chaincode, it is for testing")
	var Aval int
	var err error
    logger.Warning("init logger should be 1 string")
	if len(args) != 1 {
		return nil, errors.New("Incorrect number of arguments. Expecting 1")
	}

	// Initialize the chaincode
	Aval, err = strconv.Atoi(args[0])
	if err != nil {
		return nil, errors.New("Expecting integer value for asset holding")
	}

	// Write the state to the ledger
	err = stub.PutState("abc", []byte(strconv.Itoa(Aval)))				//making a test var "abc", I find it handy to read/write to it right away to test the network
	if err != nil {
		return nil, err
	}
	logger.Infof("init logger arg0=%v", args[0])
	var empty []string
	jsonAsBytes, _ := json.Marshal(empty)								//marshal an emtpy array of strings to clear the index
	err = stub.PutState(orderIndexStr, jsonAsBytes)
	if err != nil {
		return nil, err
	}

	return nil, nil
}

// Invoke isur entry point to invoke a chaincode function
func (t *BienChaincode) Invoke(stub *shim.ChaincodeStub, function string, args []string) ([]byte, error) {
	fmt.Println("invoke is running " + function)

	// Handle different functions
	if function == "init" {
		return t.Init(stub, "init", args)
	} else if function == "write" {
		return t.write(stub, args)
	} else if function == "set_owner" {
		return t.set_owner(stub, args)
	} else if function == "change_state" {
		return t.change_state(stub, args)
	} else if function == "add_goods" {
		return t.add_goods(stub, args)
	}
	fmt.Println("invoke did not find func: " + function)

	return nil, errors.New("Received unknown function invocation")
}

// Query is our entry point for queries
func (t *BienChaincode) Query(stub *shim.ChaincodeStub, function string, args []string) ([]byte, error) {
	fmt.Println("query is running " + function)

	// Handle different functions
	if function == "read" { //read a variable
		return t.read(stub, args)
	}
	fmt.Println("query did not find func: " + function)

	return nil, errors.New("Received unknown function query")
}

// write - invoke function to write key/value pair
func (t *BienChaincode) write(stub *shim.ChaincodeStub, args []string) ([]byte, error) {
	var key, value string
	var err error
	fmt.Println("running write()")

	if len(args) != 2 {
		return nil, errors.New("Incorrect number of arguments. Expecting 2. name of the key and value to set")
	}

	key = args[0]
	value = args[1]
	err = stub.PutState(key, []byte(value)) //write the variable into the chaincode state
	if err != nil {
		return nil, err
	}
	return nil, nil
}

// read - query function to read key/value pair
func (t *BienChaincode) read(stub *shim.ChaincodeStub, args []string) ([]byte, error) {
	var key, jsonResp string
	var err error

	if len(args) != 1 {
		return nil, errors.New("Incorrect number of arguments. Expecting name of the key to query")
	}

	key = args[0]

	valAsbytes, err := stub.GetState(key)
	logger.Infof("query.read logger valAsbytes=%v", valAsbytes)
	if err != nil {
		jsonResp = "{\"Error\":\"Failed to get state for " + key + "\"}"
		return nil, errors.New(jsonResp)
	}

	return valAsbytes, nil
}

// read - query function to read key/value pair
func (t *BienChaincode) set_owner(stub *shim.ChaincodeStub, args []string) ([]byte, error) {
	var err error

	if len(args)<2 {
	 return nil,errors.New("Incorrect number of arguments. Expecting 2")
	}

	fmt.Println("- start set owner-")
	fmt.Println(args[0] + " - " + args[1])
	bienAsBytes, err := stub.GetState(args[0])
	if err != nil {
			return nil, errors.New("Failed to get item")
		}
		res := Bien{}
		json.Unmarshal(bienAsBytes, &res)										//un stringify it aka JSON.parse()
		res.owner = args[1]

		jsonAsBytes, _ := json.Marshal(res)
		err = stub.PutState(args[0], jsonAsBytes)								//rewrite the marble with id as key
		if err != nil {
			return nil, err
		}

		fmt.Println("- end set owner-")

		return nil, nil
}

// read - query function to read key/value pair, then change the data structure's state field
func (t *BienChaincode) change_state(stub *shim.ChaincodeStub, args []string) ([]byte, error) {
//   0       1       2          3       4     5
	//id  "name", "owner", "state", "price"  "postage"
	var err error

	if len(args)<2 {
	 return nil,errors.New("Incorrect number of arguments. Expecting 2")
	}

	bienAsBytes, err := stub.GetState(args[0])
	logger.Infof("change_state getState: logger bienAsBytes=%v", bienAsBytes)
	if err != nil {
			return nil, errors.New("Failed to get thing")
		}

    var res Bien
   //	res := Bien{}
		json.Unmarshal(bienAsBytes, &res)
		logger.Infof("change_state before set res: logger res=%v", res)									//un stringify it aka JSON.parse()
		res.state = args[1]

		logger.Infof("change_state res: logger res=%v", res)
		fmt.Println(res.id, ":",res.name, ":", res.owner, ":", res.state, ":", res.price, ":", res.postage)
		jsonAsBytes, _ := json.Marshal(res)
		err = stub.PutState(args[0], jsonAsBytes)								//rewrite the goods with name as key

		if err != nil {
			return nil, err
		}

		fmt.Println("- end change state-")

		//valAsbytes, err := stub.GetState(args[0])
	//logger.Infof("query.read logger valAsbytes=%v", valAsbytes)
	return nil, nil

}

func (t *BienChaincode) add_goods(stub *shim.ChaincodeStub, args []string) ([]byte, error) {
var err error
fmt.Println("hello add goods")
	//   0       1       2          3       4
	// "name", "owner", "state", "price"  "postage"
	if len(args) != 5 {
		return nil, errors.New("Incorrect number of arguments. Expecting 4")
	}

	fmt.Println("- start add goods")
	if len(args[0]) <= 0 {
		return nil, errors.New("1st argument must be a non-empty string")
	}
	if len(args[1]) <= 0 {
		return nil, errors.New("2nd argument must be a non-empty string")
	}
	if len(args[2]) <= 0 {
		return nil, errors.New("3rd argument must be a non-empty string")
	}
	if len(args[3]) <= 0 {
		return nil, errors.New("4th argument must be a non-empty string")
	}
	if len(args[4]) <= 0 {
		return nil, errors.New("5th argument must be a non-empty string")
	}

	timestamp := time.Now().Unix()
	//str := `{"id":"`+strconv.FormatInt(timestamp , 10)+`","name": "` + args[0] + `", "owner": "` + args[1] + `", "state": "` + args[2]+ `", "price": ` + args[3] + `, "postage": ` + args[4] +`}`
	//======

	res := Bien{}
	res.id      = "0000"
	res.name    = "mouse"
	res.owner   = "no"
	res.state   = "new"
	res.price = 100
	res.postage = 10
//	res.id      = timestamp
//	res.name    = args[0]
//	res.owner   = args[1]
//	res.state   = args[2]
//	res.price, err   = strconv.Atoi(args[3])
//	res.postage, err = strconv.Atoi(args[4])

	if err != nil {
		return nil, err
	}

	goodBytes, _ := json.Marshal(res)
	err = stub.PutState("test", goodBytes)
	err = stub.PutState(strconv.FormatInt(timestamp , 10), goodBytes)

	//=======
	//err = stub.PutState(strconv.FormatInt(timestamp , 10), []byte(str))								//store marble with id as key
	if err != nil {
		return nil, err
	}

	//get the  index
	bienAsBytes, err := stub.GetState(orderIndexStr)
	if err != nil {
		return nil, errors.New("Failed to get bien index")
	}
	var orderIndex []string
	json.Unmarshal(bienAsBytes, &orderIndex)							//un stringify it aka JSON.parse()
	fmt.Println("get order(bien) index: ", orderIndex)
	//append
	orderIndex = append(orderIndex,strconv.FormatInt(timestamp , 10))								//add bien id to index list
	fmt.Println("append:! order(bien) index: ", orderIndex)
	jsonAsBytes, _ := json.Marshal(orderIndex)
	err = stub.PutState(orderIndexStr, jsonAsBytes)						//store id of bien

	fmt.Println("- end add goods")
	return nil, nil
}<commit_msg>harcode to debug<commit_after>/*
Copyright IBM Corp 2016 All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
		 http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"errors"
	"fmt"
	"strconv"
	"time"
	"encoding/json"
	"github.com/hyperledger/fabric/core/chaincode/shim"
)

// BienChaincode is  a Chaincode for bien application implementation
type BienChaincode struct {
}
var orderIndexStr ="_orderindex"

type Bien struct{
		id int64 `json:"orderId"`
		name string `json:"name"`
		state string `json:"state"`
		price int `json:"price"`
		postage int `json:"postage"`
		owner string `json:"owner"`
}
var logger = shim.NewLogger("SimpleChaincode")
func main() {
    logger.SetLevel(shim.LogInfo)
	err := shim.Start(new(BienChaincode))
	if err != nil {
		fmt.Printf("Error starting BienChaincode chaincode: %s", err)
	}
}

// Init resets all the things
func (t *BienChaincode) Init(stub *shim.ChaincodeStub, function string, args []string) ([]byte, error) {
	fmt.Printf("hello init chaincode, it is for testing")
	var Aval int
	var err error
    logger.Warning("init logger should be 1 string")
	if len(args) != 1 {
		return nil, errors.New("Incorrect number of arguments. Expecting 1")
	}

	// Initialize the chaincode
	Aval, err = strconv.Atoi(args[0])
	if err != nil {
		return nil, errors.New("Expecting integer value for asset holding")
	}

	// Write the state to the ledger
	err = stub.PutState("abc", []byte(strconv.Itoa(Aval)))				//making a test var "abc", I find it handy to read/write to it right away to test the network
	if err != nil {
		return nil, err
	}
	logger.Infof("init logger arg0=%v", args[0])
	var empty []string
	jsonAsBytes, _ := json.Marshal(empty)								//marshal an emtpy array of strings to clear the index
	err = stub.PutState(orderIndexStr, jsonAsBytes)
	if err != nil {
		return nil, err
	}

	return nil, nil
}

// Invoke isur entry point to invoke a chaincode function
func (t *BienChaincode) Invoke(stub *shim.ChaincodeStub, function string, args []string) ([]byte, error) {
	fmt.Println("invoke is running " + function)

	// Handle different functions
	if function == "init" {
		return t.Init(stub, "init", args)
	} else if function == "write" {
		return t.write(stub, args)
	} else if function == "set_owner" {
		return t.set_owner(stub, args)
	} else if function == "change_state" {
		return t.change_state(stub, args)
	} else if function == "add_goods" {
		return t.add_goods(stub, args)
	}
	fmt.Println("invoke did not find func: " + function)

	return nil, errors.New("Received unknown function invocation")
}

// Query is our entry point for queries
func (t *BienChaincode) Query(stub *shim.ChaincodeStub, function string, args []string) ([]byte, error) {
	fmt.Println("query is running " + function)

	// Handle different functions
	if function == "read" { //read a variable
		return t.read(stub, args)
	}
	fmt.Println("query did not find func: " + function)

	return nil, errors.New("Received unknown function query")
}

// write - invoke function to write key/value pair
func (t *BienChaincode) write(stub *shim.ChaincodeStub, args []string) ([]byte, error) {
	var key, value string
	var err error
	fmt.Println("running write()")

	if len(args) != 2 {
		return nil, errors.New("Incorrect number of arguments. Expecting 2. name of the key and value to set")
	}

	key = args[0]
	value = args[1]
	err = stub.PutState(key, []byte(value)) //write the variable into the chaincode state
	if err != nil {
		return nil, err
	}
	return nil, nil
}

// read - query function to read key/value pair
func (t *BienChaincode) read(stub *shim.ChaincodeStub, args []string) ([]byte, error) {
	var key, jsonResp string
	var err error

	if len(args) != 1 {
		return nil, errors.New("Incorrect number of arguments. Expecting name of the key to query")
	}

	key = args[0]

	valAsbytes, err := stub.GetState(key)
	logger.Infof("query.read logger valAsbytes=%v", valAsbytes)
	if err != nil {
		jsonResp = "{\"Error\":\"Failed to get state for " + key + "\"}"
		return nil, errors.New(jsonResp)
	}

	return valAsbytes, nil
}

// read - query function to read key/value pair
func (t *BienChaincode) set_owner(stub *shim.ChaincodeStub, args []string) ([]byte, error) {
	var err error

	if len(args)<2 {
	 return nil,errors.New("Incorrect number of arguments. Expecting 2")
	}

	fmt.Println("- start set owner-")
	fmt.Println(args[0] + " - " + args[1])
	bienAsBytes, err := stub.GetState(args[0])
	if err != nil {
			return nil, errors.New("Failed to get item")
		}
		res := Bien{}
		json.Unmarshal(bienAsBytes, &res)										//un stringify it aka JSON.parse()
		res.owner = args[1]

		jsonAsBytes, _ := json.Marshal(res)
		err = stub.PutState(args[0], jsonAsBytes)								//rewrite the marble with id as key
		if err != nil {
			return nil, err
		}

		fmt.Println("- end set owner-")

		return nil, nil
}

// read - query function to read key/value pair, then change the data structure's state field
func (t *BienChaincode) change_state(stub *shim.ChaincodeStub, args []string) ([]byte, error) {
//   0       1       2          3       4     5
	//id  "name", "owner", "state", "price"  "postage"
	var err error

	if len(args)<2 {
	 return nil,errors.New("Incorrect number of arguments. Expecting 2")
	}

	bienAsBytes, err := stub.GetState(args[0])
	logger.Infof("change_state getState: logger bienAsBytes=%v", bienAsBytes)
	if err != nil {
			return nil, errors.New("Failed to get thing")
		}

    var res Bien
   //	res := Bien{}
		json.Unmarshal(bienAsBytes, &res)
		logger.Infof("change_state before set res: logger res=%v", res)									//un stringify it aka JSON.parse()
		res.state = args[1]

		logger.Infof("change_state res: logger res=%v", res)
		fmt.Println(res.id, ":",res.name, ":", res.owner, ":", res.state, ":", res.price, ":", res.postage)
		jsonAsBytes, _ := json.Marshal(res)
		err = stub.PutState(args[0], jsonAsBytes)								//rewrite the goods with name as key

		if err != nil {
			return nil, err
		}

		fmt.Println("- end change state-")

		//valAsbytes, err := stub.GetState(args[0])
	//logger.Infof("query.read logger valAsbytes=%v", valAsbytes)
	return nil, nil

}

func (t *BienChaincode) add_goods(stub *shim.ChaincodeStub, args []string) ([]byte, error) {
var err error
fmt.Println("hello add goods")
	//   0       1       2          3       4
	// "name", "owner", "state", "price"  "postage"
	if len(args) != 5 {
		return nil, errors.New("Incorrect number of arguments. Expecting 4")
	}

	fmt.Println("- start add goods")
	if len(args[0]) <= 0 {
		return nil, errors.New("1st argument must be a non-empty string")
	}
	if len(args[1]) <= 0 {
		return nil, errors.New("2nd argument must be a non-empty string")
	}
	if len(args[2]) <= 0 {
		return nil, errors.New("3rd argument must be a non-empty string")
	}
	if len(args[3]) <= 0 {
		return nil, errors.New("4th argument must be a non-empty string")
	}
	if len(args[4]) <= 0 {
		return nil, errors.New("5th argument must be a non-empty string")
	}

	timestamp := time.Now().Unix()
	//str := `{"id":"`+strconv.FormatInt(timestamp , 10)+`","name": "` + args[0] + `", "owner": "` + args[1] + `", "state": "` + args[2]+ `", "price": ` + args[3] + `, "postage": ` + args[4] +`}`
	//======

	res := Bien{}
	res.id      = 100000
	res.name    = "mouse"
	res.owner   = "no"
	res.state   = "new"
	res.price = 100
	res.postage = 10
//	res.id      = timestamp
//	res.name    = args[0]
//	res.owner   = args[1]
//	res.state   = args[2]
//	res.price, err   = strconv.Atoi(args[3])
//	res.postage, err = strconv.Atoi(args[4])

	if err != nil {
		return nil, err
	}

	goodBytes, _ := json.Marshal(res)
	err = stub.PutState("test", goodBytes)
	err = stub.PutState(strconv.FormatInt(timestamp , 10), goodBytes)

	//=======
	//err = stub.PutState(strconv.FormatInt(timestamp , 10), []byte(str))								//store marble with id as key
	if err != nil {
		return nil, err
	}

	//get the  index
	bienAsBytes, err := stub.GetState(orderIndexStr)
	if err != nil {
		return nil, errors.New("Failed to get bien index")
	}
	var orderIndex []string
	json.Unmarshal(bienAsBytes, &orderIndex)							//un stringify it aka JSON.parse()
	fmt.Println("get order(bien) index: ", orderIndex)
	//append
	orderIndex = append(orderIndex,strconv.FormatInt(timestamp , 10))								//add bien id to index list
	fmt.Println("append:! order(bien) index: ", orderIndex)
	jsonAsBytes, _ := json.Marshal(orderIndex)
	err = stub.PutState(orderIndexStr, jsonAsBytes)						//store id of bien

	fmt.Println("- end add goods")
	return nil, nil
}<|endoftext|>


<commit_before>package bgitter

import (
	"github.com/42wim/matterbridge/bridge/config"
	log "github.com/Sirupsen/logrus"
	"github.com/sromku/go-gitter"
	"strings"
)

type Bgitter struct {
	c        *gitter.Gitter
	Config   *config.Protocol
	Remote   chan config.Message
	protocol string
	origin   string
	Rooms    []gitter.Room
}

var flog *log.Entry
var protocol = "gitter"

func init() {
	flog = log.WithFields(log.Fields{"module": protocol})
}

func New(config config.Protocol, origin string, c chan config.Message) *Bgitter {
	b := &Bgitter{}
	b.Config = &config
	b.Remote = c
	b.protocol = protocol
	b.origin = origin
	return b
}

func (b *Bgitter) Connect() error {
	var err error
	flog.Info("Trying " + b.protocol + " connection")
	b.c = gitter.New(b.Config.Token)
	_, err = b.c.GetUser()
	if err != nil {
		flog.Debugf("%#v", err)
		return err
	}
	flog.Info("Connection succeeded")
	//b.setupChannels()
	b.Rooms, _ = b.c.GetRooms()
	//go b.handleGitter()
	return nil
}

func (b *Bgitter) FullOrigin() string {
	return b.protocol + "." + b.origin
}

func (b *Bgitter) JoinChannel(channel string) error {
	_, err := b.c.JoinRoom(channel)
	if err != nil {
		return err
	}
	room := channel
	roomID := b.getRoomID(room)
	if roomID == "" {
		return nil
	}
	stream := b.c.Stream(roomID)
	go b.c.Listen(stream)

	go func(stream *gitter.Stream, room string) {
		for {
			event := <-stream.Event
			switch ev := event.Data.(type) {
			case *gitter.MessageReceived:
				// check for ZWSP to see if it's not an echo
				if !strings.HasSuffix(ev.Message.Text, "") {
					b.Remote <- config.Message{Username: ev.Message.From.Username, Text: ev.Message.Text, Channel: room,
						Origin: b.origin, Protocol: b.protocol, FullOrigin: b.FullOrigin()}
				}
			case *gitter.GitterConnectionClosed:
				flog.Errorf("connection with gitter closed for room %s", room)
			}
		}
	}(stream, room)
	return nil
}

func (b *Bgitter) Name() string {
	return b.protocol + "." + b.origin
}

func (b *Bgitter) Protocol() string {
	return b.protocol
}

func (b *Bgitter) Origin() string {
	return b.origin
}

func (b *Bgitter) Send(msg config.Message) error {
	roomID := b.getRoomID(msg.Channel)
	if roomID == "" {
		flog.Errorf("Could not find roomID for %v", msg.Channel)
		return nil
	}
	// add ZWSP because gitter echoes our own messages
	return b.c.SendMessage(roomID, msg.Username+msg.Text+" ")
}

func (b *Bgitter) getRoomID(channel string) string {
	for _, v := range b.Rooms {
		if v.URI == channel {
			return v.ID
		}
	}
	return ""
}

/*
func (b *Bgitter) handleGitter() {
	for _, val := range b.Config.Channel {
		room := val.Gitter
		roomID := b.getRoomID(room)
		if roomID == "" {
			continue
		}
		stream := b.c.Stream(roomID)
		go b.c.Listen(stream)

		go func(stream *gitter.Stream, room string) {
			for {
				event := <-stream.Event
				switch ev := event.Data.(type) {
				case *gitter.MessageReceived:
					// check for ZWSP to see if it's not an echo
					if !strings.HasSuffix(ev.Message.Text, "") {
						b.Remote <- config.Message{Username: ev.Message.From.Username, Text: ev.Message.Text, Channel: room,
							Origin: b.origin, Protocol: b.protocol, FullOrigin: b.FullOrigin()}
					}
				case *gitter.GitterConnectionClosed:
					flog.Errorf("connection with gitter closed for room %s", room)
				}
			}
		}(stream, room)
	}
}
*/
<commit_msg>Cleanup gitter bridge debug/info messages<commit_after>package bgitter

import (
	"github.com/42wim/matterbridge/bridge/config"
	log "github.com/Sirupsen/logrus"
	"github.com/sromku/go-gitter"
	"strings"
)

type Bgitter struct {
	c        *gitter.Gitter
	Config   *config.Protocol
	Remote   chan config.Message
	protocol string
	origin   string
	Rooms    []gitter.Room
}

var flog *log.Entry
var protocol = "gitter"

func init() {
	flog = log.WithFields(log.Fields{"module": protocol})
}

func New(config config.Protocol, origin string, c chan config.Message) *Bgitter {
	b := &Bgitter{}
	b.Config = &config
	b.Remote = c
	b.protocol = protocol
	b.origin = origin
	return b
}

func (b *Bgitter) Connect() error {
	var err error
	flog.Info("Connecting")
	b.c = gitter.New(b.Config.Token)
	_, err = b.c.GetUser()
	if err != nil {
		flog.Debugf("%#v", err)
		return err
	}
	flog.Info("Connection succeeded")
	b.Rooms, _ = b.c.GetRooms()
	return nil
}

func (b *Bgitter) FullOrigin() string {
	return b.protocol + "." + b.origin
}

func (b *Bgitter) JoinChannel(channel string) error {
	_, err := b.c.JoinRoom(channel)
	if err != nil {
		return err
	}
	room := channel
	roomID := b.getRoomID(room)
	if roomID == "" {
		return nil
	}
	stream := b.c.Stream(roomID)
	go b.c.Listen(stream)

	go func(stream *gitter.Stream, room string) {
		for {
			event := <-stream.Event
			switch ev := event.Data.(type) {
			case *gitter.MessageReceived:
				// check for ZWSP to see if it's not an echo
				if !strings.HasSuffix(ev.Message.Text, "") {
					flog.Debugf("Sending message from %s on %s to gateway", ev.Message.From.Username, b.FullOrigin())
					b.Remote <- config.Message{Username: ev.Message.From.Username, Text: ev.Message.Text, Channel: room,
						Origin: b.origin, Protocol: b.protocol, FullOrigin: b.FullOrigin()}
				}
			case *gitter.GitterConnectionClosed:
				flog.Errorf("connection with gitter closed for room %s", room)
			}
		}
	}(stream, room)
	return nil
}

func (b *Bgitter) Name() string {
	return b.protocol + "." + b.origin
}

func (b *Bgitter) Protocol() string {
	return b.protocol
}

func (b *Bgitter) Origin() string {
	return b.origin
}

func (b *Bgitter) Send(msg config.Message) error {
	flog.Debugf("Receiving %#v", msg)
	roomID := b.getRoomID(msg.Channel)
	if roomID == "" {
		flog.Errorf("Could not find roomID for %v", msg.Channel)
		return nil
	}
	// add ZWSP because gitter echoes our own messages
	return b.c.SendMessage(roomID, msg.Username+msg.Text+" ")
}

func (b *Bgitter) getRoomID(channel string) string {
	for _, v := range b.Rooms {
		if v.URI == channel {
			return v.ID
		}
	}
	return ""
}
<|endoftext|>


<commit_before>package helper

import (
	"bytes"
	"fmt"
	"image/png"
	"io"
	"net/http"
	"regexp"
	"strings"
	"time"
	"unicode/utf8"

	"golang.org/x/image/webp"

	"github.com/42wim/matterbridge/bridge/config"
	"github.com/gomarkdown/markdown"
	"github.com/gomarkdown/markdown/html"
	"github.com/gomarkdown/markdown/parser"
	"github.com/sirupsen/logrus"
)

// DownloadFile downloads the given non-authenticated URL.
func DownloadFile(url string) (*[]byte, error) {
	return DownloadFileAuth(url, "")
}

// DownloadFileAuth downloads the given URL using the specified authentication token.
func DownloadFileAuth(url string, auth string) (*[]byte, error) {
	var buf bytes.Buffer
	client := &http.Client{
		Timeout: time.Second * 5,
	}
	req, err := http.NewRequest("GET", url, nil)
	if auth != "" {
		req.Header.Add("Authorization", auth)
	}
	if err != nil {
		return nil, err
	}
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	io.Copy(&buf, resp.Body)
	data := buf.Bytes()
	return &data, nil
}

// GetSubLines splits messages in newline-delimited lines. If maxLineLength is
// specified as non-zero GetSubLines will also clip long lines to the maximum
// length and insert a warning marker that the line was clipped.
//
// TODO: The current implementation has the inconvenient that it disregards
// word boundaries when splitting but this is hard to solve without potentially
// breaking formatting and other stylistic effects.
func GetSubLines(message string, maxLineLength int) []string {
	const clippingMessage = " <clipped message>"

	var lines []string
	for _, line := range strings.Split(strings.TrimSpace(message), "\n") {
		if maxLineLength == 0 || len([]byte(line)) <= maxLineLength {
			lines = append(lines, line)
			continue
		}

		// !!! WARNING !!!
		// Before touching the splitting logic below please ensure that you PROPERLY
		// understand how strings, runes and range loops over strings work in Go.
		// A good place to start is to read https://blog.golang.org/strings. :-)
		var splitStart int
		var startOfPreviousRune int
		for i := range line {
			if i-splitStart > maxLineLength-len([]byte(clippingMessage)) {
				lines = append(lines, line[splitStart:startOfPreviousRune]+clippingMessage)
				splitStart = startOfPreviousRune
			}
			startOfPreviousRune = i
		}
		// This last append is safe to do without looking at the remaining byte-length
		// as we assume that the byte-length of the last rune will never exceed that of
		// the byte-length of the clipping message.
		lines = append(lines, line[splitStart:])
	}
	return lines
}

// HandleExtra manages the supplementary details stored inside a message's 'Extra' field map.
func HandleExtra(msg *config.Message, general *config.Protocol) []config.Message {
	extra := msg.Extra
	rmsg := []config.Message{}
	for _, f := range extra[config.EventFileFailureSize] {
		fi := f.(config.FileInfo)
		text := fmt.Sprintf("file %s too big to download (%#v > allowed size: %#v)", fi.Name, fi.Size, general.MediaDownloadSize)
		rmsg = append(rmsg, config.Message{
			Text:     text,
			Username: "<system> ",
			Channel:  msg.Channel,
			Account:  msg.Account,
		})
	}
	return rmsg
}

// GetAvatar constructs a URL for a given user-avatar if it is available in the cache.
func GetAvatar(av map[string]string, userid string, general *config.Protocol) string {
	if sha, ok := av[userid]; ok {
		return general.MediaServerDownload + "/" + sha + "/" + userid + ".png"
	}
	return ""
}

// HandleDownloadSize checks a specified filename against the configured download blacklist
// and checks a specified file-size against the configure limit.
func HandleDownloadSize(logger *logrus.Entry, msg *config.Message, name string, size int64, general *config.Protocol) error {
	// check blacklist here
	for _, entry := range general.MediaDownloadBlackList {
		if entry != "" {
			re, err := regexp.Compile(entry)
			if err != nil {
				logger.Errorf("incorrect regexp %s for %s", entry, msg.Account)
				continue
			}
			if re.MatchString(name) {
				return fmt.Errorf("Matching blacklist %s. Not downloading %s", entry, name)
			}
		}
	}
	logger.Debugf("Trying to download %#v with size %#v", name, size)
	if int(size) > general.MediaDownloadSize {
		msg.Event = config.EventFileFailureSize
		msg.Extra[msg.Event] = append(msg.Extra[msg.Event], config.FileInfo{
			Name:    name,
			Comment: msg.Text,
			Size:    size,
		})
		return fmt.Errorf("File %#v to large to download (%#v). MediaDownloadSize is %#v", name, size, general.MediaDownloadSize)
	}
	return nil
}

// HandleDownloadData adds the data for a remote file into a Matterbridge gateway message.
func HandleDownloadData(logger *logrus.Entry, msg *config.Message, name, comment, url string, data *[]byte, general *config.Protocol) {
	var avatar bool
	logger.Debugf("Download OK %#v %#v", name, len(*data))
	if msg.Event == config.EventAvatarDownload {
		avatar = true
	}
	msg.Extra["file"] = append(msg.Extra["file"], config.FileInfo{
		Name:    name,
		Data:    data,
		URL:     url,
		Comment: comment,
		Avatar:  avatar,
	})
}

var emptyLineMatcher = regexp.MustCompile("\n+")

// RemoveEmptyNewLines collapses consecutive newline characters into a single one and
// trims any preceding or trailing newline characters as well.
func RemoveEmptyNewLines(msg string) string {
	return emptyLineMatcher.ReplaceAllString(strings.Trim(msg, "\n"), "\n")
}

// ClipMessage trims a message to the specified length if it exceeds it and adds a warning
// to the message in case it does so.
func ClipMessage(text string, length int) string {
	const clippingMessage = " <clipped message>"
	if len(text) > length {
		text = text[:length-len(clippingMessage)]
		if r, size := utf8.DecodeLastRuneInString(text); r == utf8.RuneError {
			text = text[:len(text)-size]
		}
		text += clippingMessage
	}
	return text
}

// ParseMarkdown takes in an input string as markdown and parses it to html
func ParseMarkdown(input string) string {
	extensions := parser.HardLineBreak
	markdownParser := parser.NewWithExtensions(extensions)
	renderer := html.NewRenderer(html.RendererOptions{
		Flags: 0,
	})
	parsedMarkdown := markdown.ToHTML([]byte(input), markdownParser, renderer)
	res := string(parsedMarkdown)
	res = strings.TrimPrefix(res, "
")
	res = strings.TrimSuffix(res, "
\n")
	return res
}

// ConvertWebPToPNG convert input data (which should be WebP format to PNG format)
func ConvertWebPToPNG(data *[]byte) error {
	r := bytes.NewReader(*data)
	m, err := webp.Decode(r)
	if err != nil {
		return err
	}
	var output []byte
	w := bytes.NewBuffer(output)
	if err := png.Encode(w, m); err != nil {
		return err
	}
	*data = w.Bytes()
	return nil
}
<commit_msg>Enable intra-word emphasis supression in markdown (#999)<commit_after>package helper

import (
	"bytes"
	"fmt"
	"image/png"
	"io"
	"net/http"
	"regexp"
	"strings"
	"time"
	"unicode/utf8"

	"golang.org/x/image/webp"

	"github.com/42wim/matterbridge/bridge/config"
	"github.com/gomarkdown/markdown"
	"github.com/gomarkdown/markdown/html"
	"github.com/gomarkdown/markdown/parser"
	"github.com/sirupsen/logrus"
)

// DownloadFile downloads the given non-authenticated URL.
func DownloadFile(url string) (*[]byte, error) {
	return DownloadFileAuth(url, "")
}

// DownloadFileAuth downloads the given URL using the specified authentication token.
func DownloadFileAuth(url string, auth string) (*[]byte, error) {
	var buf bytes.Buffer
	client := &http.Client{
		Timeout: time.Second * 5,
	}
	req, err := http.NewRequest("GET", url, nil)
	if auth != "" {
		req.Header.Add("Authorization", auth)
	}
	if err != nil {
		return nil, err
	}
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	io.Copy(&buf, resp.Body)
	data := buf.Bytes()
	return &data, nil
}

// GetSubLines splits messages in newline-delimited lines. If maxLineLength is
// specified as non-zero GetSubLines will also clip long lines to the maximum
// length and insert a warning marker that the line was clipped.
//
// TODO: The current implementation has the inconvenient that it disregards
// word boundaries when splitting but this is hard to solve without potentially
// breaking formatting and other stylistic effects.
func GetSubLines(message string, maxLineLength int) []string {
	const clippingMessage = " <clipped message>"

	var lines []string
	for _, line := range strings.Split(strings.TrimSpace(message), "\n") {
		if maxLineLength == 0 || len([]byte(line)) <= maxLineLength {
			lines = append(lines, line)
			continue
		}

		// !!! WARNING !!!
		// Before touching the splitting logic below please ensure that you PROPERLY
		// understand how strings, runes and range loops over strings work in Go.
		// A good place to start is to read https://blog.golang.org/strings. :-)
		var splitStart int
		var startOfPreviousRune int
		for i := range line {
			if i-splitStart > maxLineLength-len([]byte(clippingMessage)) {
				lines = append(lines, line[splitStart:startOfPreviousRune]+clippingMessage)
				splitStart = startOfPreviousRune
			}
			startOfPreviousRune = i
		}
		// This last append is safe to do without looking at the remaining byte-length
		// as we assume that the byte-length of the last rune will never exceed that of
		// the byte-length of the clipping message.
		lines = append(lines, line[splitStart:])
	}
	return lines
}

// HandleExtra manages the supplementary details stored inside a message's 'Extra' field map.
func HandleExtra(msg *config.Message, general *config.Protocol) []config.Message {
	extra := msg.Extra
	rmsg := []config.Message{}
	for _, f := range extra[config.EventFileFailureSize] {
		fi := f.(config.FileInfo)
		text := fmt.Sprintf("file %s too big to download (%#v > allowed size: %#v)", fi.Name, fi.Size, general.MediaDownloadSize)
		rmsg = append(rmsg, config.Message{
			Text:     text,
			Username: "<system> ",
			Channel:  msg.Channel,
			Account:  msg.Account,
		})
	}
	return rmsg
}

// GetAvatar constructs a URL for a given user-avatar if it is available in the cache.
func GetAvatar(av map[string]string, userid string, general *config.Protocol) string {
	if sha, ok := av[userid]; ok {
		return general.MediaServerDownload + "/" + sha + "/" + userid + ".png"
	}
	return ""
}

// HandleDownloadSize checks a specified filename against the configured download blacklist
// and checks a specified file-size against the configure limit.
func HandleDownloadSize(logger *logrus.Entry, msg *config.Message, name string, size int64, general *config.Protocol) error {
	// check blacklist here
	for _, entry := range general.MediaDownloadBlackList {
		if entry != "" {
			re, err := regexp.Compile(entry)
			if err != nil {
				logger.Errorf("incorrect regexp %s for %s", entry, msg.Account)
				continue
			}
			if re.MatchString(name) {
				return fmt.Errorf("Matching blacklist %s. Not downloading %s", entry, name)
			}
		}
	}
	logger.Debugf("Trying to download %#v with size %#v", name, size)
	if int(size) > general.MediaDownloadSize {
		msg.Event = config.EventFileFailureSize
		msg.Extra[msg.Event] = append(msg.Extra[msg.Event], config.FileInfo{
			Name:    name,
			Comment: msg.Text,
			Size:    size,
		})
		return fmt.Errorf("File %#v to large to download (%#v). MediaDownloadSize is %#v", name, size, general.MediaDownloadSize)
	}
	return nil
}

// HandleDownloadData adds the data for a remote file into a Matterbridge gateway message.
func HandleDownloadData(logger *logrus.Entry, msg *config.Message, name, comment, url string, data *[]byte, general *config.Protocol) {
	var avatar bool
	logger.Debugf("Download OK %#v %#v", name, len(*data))
	if msg.Event == config.EventAvatarDownload {
		avatar = true
	}
	msg.Extra["file"] = append(msg.Extra["file"], config.FileInfo{
		Name:    name,
		Data:    data,
		URL:     url,
		Comment: comment,
		Avatar:  avatar,
	})
}

var emptyLineMatcher = regexp.MustCompile("\n+")

// RemoveEmptyNewLines collapses consecutive newline characters into a single one and
// trims any preceding or trailing newline characters as well.
func RemoveEmptyNewLines(msg string) string {
	return emptyLineMatcher.ReplaceAllString(strings.Trim(msg, "\n"), "\n")
}

// ClipMessage trims a message to the specified length if it exceeds it and adds a warning
// to the message in case it does so.
func ClipMessage(text string, length int) string {
	const clippingMessage = " <clipped message>"
	if len(text) > length {
		text = text[:length-len(clippingMessage)]
		if r, size := utf8.DecodeLastRuneInString(text); r == utf8.RuneError {
			text = text[:len(text)-size]
		}
		text += clippingMessage
	}
	return text
}

// ParseMarkdown takes in an input string as markdown and parses it to html
func ParseMarkdown(input string) string {
	extensions := parser.HardLineBreak | parser.NoIntraEmphasis
	markdownParser := parser.NewWithExtensions(extensions)
	renderer := html.NewRenderer(html.RendererOptions{
		Flags: 0,
	})
	parsedMarkdown := markdown.ToHTML([]byte(input), markdownParser, renderer)
	res := string(parsedMarkdown)
	res = strings.TrimPrefix(res, "
")
	res = strings.TrimSuffix(res, "
\n")
	return res
}

// ConvertWebPToPNG convert input data (which should be WebP format to PNG format)
func ConvertWebPToPNG(data *[]byte) error {
	r := bytes.NewReader(*data)
	m, err := webp.Decode(r)
	if err != nil {
		return err
	}
	var output []byte
	w := bytes.NewBuffer(output)
	if err := png.Encode(w, m); err != nil {
		return err
	}
	*data = w.Bytes()
	return nil
}
<|endoftext|>


<commit_before>// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package dataflow contains the Dataflow runner for submitting pipelines
// to Google Cloud Dataflow.
package dataflow

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"path"
	"sync/atomic"
	"time"

	"cloud.google.com/go/storage"
	"github.com/apache/beam/sdks/go/pkg/beam"
	"github.com/apache/beam/sdks/go/pkg/beam/core/runtime/graphx"
	"github.com/apache/beam/sdks/go/pkg/beam/core/util/hooks"
	"github.com/apache/beam/sdks/go/pkg/beam/internal/errors"
	"github.com/apache/beam/sdks/go/pkg/beam/log"
	pb "github.com/apache/beam/sdks/go/pkg/beam/model/pipeline_v1"
	"github.com/apache/beam/sdks/go/pkg/beam/options/gcpopts"
	"github.com/apache/beam/sdks/go/pkg/beam/options/jobopts"
	"github.com/apache/beam/sdks/go/pkg/beam/runners/dataflow/dataflowlib"
	"github.com/apache/beam/sdks/go/pkg/beam/util/gcsx"
	"github.com/apache/beam/sdks/go/pkg/beam/x/hooks/perf"
	"github.com/golang/protobuf/proto"
)

// TODO(herohde) 5/16/2017: the Dataflow flags should match the other SDKs.

var (
	endpoint             = flag.String("dataflow_endpoint", "", "Dataflow endpoint (optional).")
	stagingLocation      = flag.String("staging_location", "", "GCS staging location (required).")
	image                = flag.String("worker_harness_container_image", "", "Worker harness container image (required).")
	labels               = flag.String("labels", "", "JSON-formatted map[string]string of job labels (optional).")
	serviceAccountEmail  = flag.String("service_account_email", "", "Service account email (optional).")
	numWorkers           = flag.Int64("num_workers", 0, "Number of workers (optional).")
	maxNumWorkers        = flag.Int64("max_num_workers", 0, "Maximum number of workers during scaling (optional).")
	autoscalingAlgorithm = flag.String("autoscaling_algorithm", "", "Autoscaling mode to use (optional).")
	zone                 = flag.String("zone", "", "GCP zone (optional)")
	region               = flag.String("region", "", "GCP Region (optional but encouraged)")
	network              = flag.String("network", "", "GCP network (optional)")
	subnetwork           = flag.String("subnetwork", "", "GCP subnetwork (optional)")
	tempLocation         = flag.String("temp_location", "", "Temp location (optional)")
	machineType          = flag.String("worker_machine_type", "", "GCE machine type (optional)")
	minCPUPlatform       = flag.String("min_cpu_platform", "", "GCE minimum cpu platform (optional)")
	workerJar            = flag.String("dataflow_worker_jar", "", "Dataflow worker jar (optional)")

	async          = flag.Bool("async", false, "Asynchronous execution. Submit the job and return immediately.")
	dryRun         = flag.Bool("dry_run", false, "Dry run. Just print the job, but don't submit it.")
	teardownPolicy = flag.String("teardown_policy", "", "Job teardown policy (internal only).")

	// SDK options
	cpuProfiling     = flag.String("cpu_profiling", "", "Job records CPU profiles to this GCS location (optional)")
	sessionRecording = flag.String("session_recording", "", "Job records session transcripts")
)

func init() {
	// Note that we also _ import harness/init to setup the remote execution hook.
	beam.RegisterRunner("dataflow", Execute)

	perf.RegisterProfCaptureHook("gcs_profile_writer", gcsRecorderHook)
}

var unique int32

// Execute runs the given pipeline on Google Cloud Dataflow. It uses the
// default application credentials to submit the job.
func Execute(ctx context.Context, p *beam.Pipeline) error {
	// (1) Gather job options

	project := *gcpopts.Project
	if project == "" {
		return errors.New("no Google Cloud project specified. Use --project=<project>")
	}
	if *stagingLocation == "" {
		return errors.New("no GCS staging location specified. Use --staging_location=gs://<bucket>/<path>")
	}
	if *region == "" {
		*region = "us-central1"
		log.Warn(ctx, "--region not set; will default to us-central1. Future releases of Beam will "+
			"require the user to set the region explicitly. "+
			"https://cloud.google.com/compute/docs/regions-zones/regions-zones")
	}
	if *image == "" {
		*image = getContainerImage(ctx)
	}
	var jobLabels map[string]string
	if *labels != "" {
		if err := json.Unmarshal([]byte(*labels), &jobLabels); err != nil {
			return errors.Wrapf(err, "error reading --label flag as JSON")
		}
	}

	if *cpuProfiling != "" {
		perf.EnableProfCaptureHook("gcs_profile_writer", *cpuProfiling)
	}

	if *sessionRecording != "" {
		// TODO(wcn): BEAM-4017
		// It's a bit inconvenient for GCS because the whole object is written in
		// one pass, whereas the session logs are constantly appended. We wouldn't
		// want to hold all the logs in memory to flush at the end of the pipeline
		// as we'd blow out memory on the worker. The implementation of the
		// CaptureHook should create an internal buffer and write chunks out to GCS
		// once they get to an appropriate size (50M or so?)
	}
	if *autoscalingAlgorithm != "" {
		if *autoscalingAlgorithm != "NONE" && *autoscalingAlgorithm != "THROUGHPUT_BASED" {
			return errors.New("invalid autoscaling algorithm. Use --autoscaling_algorithm=(NONE|THROUGHPUT_BASED)")
		}
	}

	hooks.SerializeHooksToOptions()

	experiments := jobopts.GetExperiments()
	if *minCPUPlatform != "" {
		experiments = append(experiments, fmt.Sprintf("min_cpu_platform=%v", *minCPUPlatform))
	}

	opts := &dataflowlib.JobOptions{
		Name:                jobopts.GetJobName(),
		Experiments:         experiments,
		Options:             beam.PipelineOptions.Export(),
		Project:             project,
		Region:              *region,
		Zone:                *zone,
		Network:             *network,
		Subnetwork:          *subnetwork,
		NumWorkers:          *numWorkers,
		MaxNumWorkers:       *maxNumWorkers,
		Algorithm:           *autoscalingAlgorithm,
		MachineType:         *machineType,
		Labels:              jobLabels,
		ServiceAccountEmail: *serviceAccountEmail,
		TempLocation:        *tempLocation,
		Worker:              *jobopts.WorkerBinary,
		WorkerJar:           *workerJar,
		TeardownPolicy:      *teardownPolicy,
	}
	if opts.TempLocation == "" {
		opts.TempLocation = gcsx.Join(*stagingLocation, "tmp")
	}

	// (1) Build and submit

	edges, _, err := p.Build()
	if err != nil {
		return err
	}
	model, err := graphx.Marshal(edges, &graphx.Options{Environment: createEnvironment(ctx)})
	if err != nil {
		return errors.WithContext(err, "generating model pipeline")
	}

	// NOTE(herohde) 10/8/2018: the last segment of the names must be "worker" and "dataflow-worker.jar".
	id := fmt.Sprintf("go-%v-%v", atomic.AddInt32(&unique, 1), time.Now().UnixNano())

	modelURL := gcsx.Join(*stagingLocation, id, "model")
	workerURL := gcsx.Join(*stagingLocation, id, "worker")
	jarURL := gcsx.Join(*stagingLocation, id, "dataflow-worker.jar")

	if *dryRun {
		log.Info(ctx, "Dry-run: not submitting job!")

		log.Info(ctx, proto.MarshalTextString(model))
		job, err := dataflowlib.Translate(model, opts, workerURL, jarURL, modelURL)
		if err != nil {
			return err
		}
		dataflowlib.PrintJob(ctx, job)
		return nil
	}

	_, err = dataflowlib.Execute(ctx, model, opts, workerURL, jarURL, modelURL, *endpoint, *async)
	return err
}
func gcsRecorderHook(opts []string) perf.CaptureHook {
	bucket, prefix, err := gcsx.ParseObject(opts[0])
	if err != nil {
		panic(fmt.Sprintf("Invalid hook configuration for gcsRecorderHook: %s", opts))
	}

	return func(ctx context.Context, spec string, r io.Reader) error {
		client, err := gcsx.NewClient(ctx, storage.ScopeReadWrite)
		if err != nil {
			return errors.WithContext(err, "establishing GCS client")
		}
		return gcsx.WriteObject(ctx, client, bucket, path.Join(prefix, spec), r)
	}
}

func getContainerImage(ctx context.Context) string {
	urn := jobopts.GetEnvironmentUrn(ctx)
	if urn == "" || urn == "beam:env:docker:v1" {
		return jobopts.GetEnvironmentConfig(ctx)
	}
	panic(fmt.Sprintf("Unsupported environment %v", urn))
}

func createEnvironment(ctx context.Context) pb.Environment {
	var environment pb.Environment
	switch urn := jobopts.GetEnvironmentUrn(ctx); urn {
	case "beam:env:process:v1":
		// TODO Support process based SDK Harness.
		panic(fmt.Sprintf("Unsupported environment %v", urn))
	case "beam:env:docker:v1":
		fallthrough
	default:
		config := *image
		payload := &pb.DockerPayload{ContainerImage: config}
		serializedPayload, err := proto.Marshal(payload)
		if err != nil {
			panic(errors.Wrapf(err,
				"Failed to serialize Environment payload %v for config %v", payload, config))
		}
		environment = pb.Environment{
			Urn:     urn,
			Payload: serializedPayload,
		}
	}
	return environment
}
<commit_msg>Renames flag to reduce ambiguity, as per review<commit_after>// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package dataflow contains the Dataflow runner for submitting pipelines
// to Google Cloud Dataflow.
package dataflow

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"path"
	"sync/atomic"
	"time"

	"cloud.google.com/go/storage"
	"github.com/apache/beam/sdks/go/pkg/beam"
	"github.com/apache/beam/sdks/go/pkg/beam/core/runtime/graphx"
	"github.com/apache/beam/sdks/go/pkg/beam/core/util/hooks"
	"github.com/apache/beam/sdks/go/pkg/beam/internal/errors"
	"github.com/apache/beam/sdks/go/pkg/beam/log"
	pb "github.com/apache/beam/sdks/go/pkg/beam/model/pipeline_v1"
	"github.com/apache/beam/sdks/go/pkg/beam/options/gcpopts"
	"github.com/apache/beam/sdks/go/pkg/beam/options/jobopts"
	"github.com/apache/beam/sdks/go/pkg/beam/runners/dataflow/dataflowlib"
	"github.com/apache/beam/sdks/go/pkg/beam/util/gcsx"
	"github.com/apache/beam/sdks/go/pkg/beam/x/hooks/perf"
	"github.com/golang/protobuf/proto"
)

// TODO(herohde) 5/16/2017: the Dataflow flags should match the other SDKs.

var (
	endpoint             = flag.String("dataflow_endpoint", "", "Dataflow endpoint (optional).")
	stagingLocation      = flag.String("staging_location", "", "GCS staging location (required).")
	image                = flag.String("worker_harness_container_image", "", "Worker harness container image (required).")
	labels               = flag.String("labels", "", "JSON-formatted map[string]string of job labels (optional).")
	serviceAccountEmail  = flag.String("service_account_email", "", "Service account email (optional).")
	numWorkers           = flag.Int64("num_workers", 0, "Number of workers (optional).")
	maxNumWorkers        = flag.Int64("max_num_workers", 0, "Maximum number of workers during scaling (optional).")
	autoscalingAlgorithm = flag.String("autoscaling_algorithm", "", "Autoscaling mode to use (optional).")
	zone                 = flag.String("zone", "", "GCP zone (optional)")
	region               = flag.String("region", "", "GCP Region (optional but encouraged)")
	network              = flag.String("network", "", "GCP network (optional)")
	subnetwork           = flag.String("subnetwork", "", "GCP subnetwork (optional)")
	tempLocation         = flag.String("temp_location", "", "Temp location (optional)")
	machineType          = flag.String("worker_machine_type", "", "GCE machine type (optional)")
	minCPUPlatform       = flag.String("min_cpu_platform", "", "GCE minimum cpu platform (optional)")
	workerJar            = flag.String("dataflow_worker_jar", "", "Dataflow worker jar (optional)")

	executeAsync   = flag.Bool("execute_async", false, "Asynchronous execution. Submit the job and return immediately.")
	dryRun         = flag.Bool("dry_run", false, "Dry run. Just print the job, but don't submit it.")
	teardownPolicy = flag.String("teardown_policy", "", "Job teardown policy (internal only).")

	// SDK options
	cpuProfiling     = flag.String("cpu_profiling", "", "Job records CPU profiles to this GCS location (optional)")
	sessionRecording = flag.String("session_recording", "", "Job records session transcripts")
)

func init() {
	// Note that we also _ import harness/init to setup the remote execution hook.
	beam.RegisterRunner("dataflow", Execute)

	perf.RegisterProfCaptureHook("gcs_profile_writer", gcsRecorderHook)
}

var unique int32

// Execute runs the given pipeline on Google Cloud Dataflow. It uses the
// default application credentials to submit the job.
func Execute(ctx context.Context, p *beam.Pipeline) error {
	// (1) Gather job options

	project := *gcpopts.Project
	if project == "" {
		return errors.New("no Google Cloud project specified. Use --project=<project>")
	}
	if *stagingLocation == "" {
		return errors.New("no GCS staging location specified. Use --staging_location=gs://<bucket>/<path>")
	}
	if *region == "" {
		*region = "us-central1"
		log.Warn(ctx, "--region not set; will default to us-central1. Future releases of Beam will "+
			"require the user to set the region explicitly. "+
			"https://cloud.google.com/compute/docs/regions-zones/regions-zones")
	}
	if *image == "" {
		*image = getContainerImage(ctx)
	}
	var jobLabels map[string]string
	if *labels != "" {
		if err := json.Unmarshal([]byte(*labels), &jobLabels); err != nil {
			return errors.Wrapf(err, "error reading --label flag as JSON")
		}
	}

	if *cpuProfiling != "" {
		perf.EnableProfCaptureHook("gcs_profile_writer", *cpuProfiling)
	}

	if *sessionRecording != "" {
		// TODO(wcn): BEAM-4017
		// It's a bit inconvenient for GCS because the whole object is written in
		// one pass, whereas the session logs are constantly appended. We wouldn't
		// want to hold all the logs in memory to flush at the end of the pipeline
		// as we'd blow out memory on the worker. The implementation of the
		// CaptureHook should create an internal buffer and write chunks out to GCS
		// once they get to an appropriate size (50M or so?)
	}
	if *autoscalingAlgorithm != "" {
		if *autoscalingAlgorithm != "NONE" && *autoscalingAlgorithm != "THROUGHPUT_BASED" {
			return errors.New("invalid autoscaling algorithm. Use --autoscaling_algorithm=(NONE|THROUGHPUT_BASED)")
		}
	}

	hooks.SerializeHooksToOptions()

	experiments := jobopts.GetExperiments()
	if *minCPUPlatform != "" {
		experiments = append(experiments, fmt.Sprintf("min_cpu_platform=%v", *minCPUPlatform))
	}

	opts := &dataflowlib.JobOptions{
		Name:                jobopts.GetJobName(),
		Experiments:         experiments,
		Options:             beam.PipelineOptions.Export(),
		Project:             project,
		Region:              *region,
		Zone:                *zone,
		Network:             *network,
		Subnetwork:          *subnetwork,
		NumWorkers:          *numWorkers,
		MaxNumWorkers:       *maxNumWorkers,
		Algorithm:           *autoscalingAlgorithm,
		MachineType:         *machineType,
		Labels:              jobLabels,
		ServiceAccountEmail: *serviceAccountEmail,
		TempLocation:        *tempLocation,
		Worker:              *jobopts.WorkerBinary,
		WorkerJar:           *workerJar,
		TeardownPolicy:      *teardownPolicy,
	}
	if opts.TempLocation == "" {
		opts.TempLocation = gcsx.Join(*stagingLocation, "tmp")
	}

	// (1) Build and submit

	edges, _, err := p.Build()
	if err != nil {
		return err
	}
	model, err := graphx.Marshal(edges, &graphx.Options{Environment: createEnvironment(ctx)})
	if err != nil {
		return errors.WithContext(err, "generating model pipeline")
	}

	// NOTE(herohde) 10/8/2018: the last segment of the names must be "worker" and "dataflow-worker.jar".
	id := fmt.Sprintf("go-%v-%v", atomic.AddInt32(&unique, 1), time.Now().UnixNano())

	modelURL := gcsx.Join(*stagingLocation, id, "model")
	workerURL := gcsx.Join(*stagingLocation, id, "worker")
	jarURL := gcsx.Join(*stagingLocation, id, "dataflow-worker.jar")

	if *dryRun {
		log.Info(ctx, "Dry-run: not submitting job!")

		log.Info(ctx, proto.MarshalTextString(model))
		job, err := dataflowlib.Translate(model, opts, workerURL, jarURL, modelURL)
		if err != nil {
			return err
		}
		dataflowlib.PrintJob(ctx, job)
		return nil
	}

	_, err = dataflowlib.Execute(ctx, model, opts, workerURL, jarURL, modelURL, *endpoint, *executeAsync)
	return err
}
func gcsRecorderHook(opts []string) perf.CaptureHook {
	bucket, prefix, err := gcsx.ParseObject(opts[0])
	if err != nil {
		panic(fmt.Sprintf("Invalid hook configuration for gcsRecorderHook: %s", opts))
	}

	return func(ctx context.Context, spec string, r io.Reader) error {
		client, err := gcsx.NewClient(ctx, storage.ScopeReadWrite)
		if err != nil {
			return errors.WithContext(err, "establishing GCS client")
		}
		return gcsx.WriteObject(ctx, client, bucket, path.Join(prefix, spec), r)
	}
}

func getContainerImage(ctx context.Context) string {
	urn := jobopts.GetEnvironmentUrn(ctx)
	if urn == "" || urn == "beam:env:docker:v1" {
		return jobopts.GetEnvironmentConfig(ctx)
	}
	panic(fmt.Sprintf("Unsupported environment %v", urn))
}

func createEnvironment(ctx context.Context) pb.Environment {
	var environment pb.Environment
	switch urn := jobopts.GetEnvironmentUrn(ctx); urn {
	case "beam:env:process:v1":
		// TODO Support process based SDK Harness.
		panic(fmt.Sprintf("Unsupported environment %v", urn))
	case "beam:env:docker:v1":
		fallthrough
	default:
		config := *image
		payload := &pb.DockerPayload{ContainerImage: config}
		serializedPayload, err := proto.Marshal(payload)
		if err != nil {
			panic(errors.Wrapf(err,
				"Failed to serialize Environment payload %v for config %v", payload, config))
		}
		environment = pb.Environment{
			Urn:     urn,
			Payload: serializedPayload,
		}
	}
	return environment
}
<|endoftext|>


<commit_before>package testhelper

import (
	"reflect"
	"testing"
)

// This function compares two arbitrary values and performs a comparison. If the
// comparison fails, a fatal error is raised that will fail the test
func Equals(t *testing.T, expected, actual interface{}) {
	if expected != actual {
		t.Fatalf("Expected [%#v] but got [%#v]", expected, actual)
	}
}

// This function, like Equals, performs a comparison - but on more complex
// structures that requires deeper inspection
func DeepEquals(t *testing.T, actual, expected interface{}) {
	if !reflect.DeepEqual(actual, expected) {
		t.Fatalf("Expected %#v but got %#v", expected, actual)
	}
}

// A convenience function for checking whether an error value is an actual error
func CheckErr(t *testing.T, e error) {
	if e != nil {
		t.Fatalf("Unexpected error: %#v", e)
	}
}
<commit_msg>Changes to names based on code review<commit_after>package testhelper

import (
	"reflect"
	"testing"
)

// AssertEquals compares two arbitrary values and performs a comparison. If the
// comparison fails, a fatal error is raised that will fail the test
func AssertEquals(t *testing.T, expected, actual interface{}) {
	if expected != actual {
		t.Fatalf("Expected [%#v] but got [%#v]", expected, actual)
	}
}

// CheckEquals is similar to AssertEquals, except with a non-fatal error
func CheckEquals(t *testing.T, expected, actual interface{}) {
	if expected != actual {
		t.Errorf("Expected [%#v] but got [%#v]", expected, actual)
	}
}

// AssertDeepEquals - like Equals - performs a comparison - but on more complex
// structures that requires deeper inspection
func AssertDeepEquals(t *testing.T, actual, expected interface{}) {
	if !reflect.DeepEqual(actual, expected) {
		t.Fatalf("Expected %#v but got %#v", expected, actual)
	}
}

// CheckDeepEquals is similar to AssertDeepEquals, except with a non-fatal error
func CheckDeepEquals(t *testing.T, actual, expected interface{}) {
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("Expected %#v but got %#v", expected, actual)
	}
}

// AssertNoErr is a convenience function for checking whether an error value is
// an actual error
func AssertNoErr(t *testing.T, e error) {
	if e != nil {
		t.Fatalf("Unexpected error: %#v", e)
	}
}

// CheckNoErr is similar to AssertNoErr, except with a non-fatal error
func CheckNoErr(t *testing.T, e error) {
	if e != nil {
		t.Errorf("Unexpected error: %#v", e)
	}
}
<|endoftext|>


<commit_before>package uploads

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("UploadService", func() {
	var (
		s UploadService
	)

	BeforeEach(func() {
		s = NewUploadService()
	})

	Describe("something", func() {
		Context("context", func() {
			It("Should do something", func() {
				str := ""
				Expect(str).To(Equal(""))
			})
		})
	})

})
<commit_msg>Add skeleton unit tests for upload service.<commit_after>package uploads

import (
	dmocks "github.com/materials-commons/mcstore/pkg/db/dai/mocks"

	"github.com/materials-commons/config"
	"github.com/materials-commons/gohandy/file"
	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/app/flow"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("UploadService", func() {
	var (
		s              *uploadService
		mdirs          *dmocks.Dirs
		muploads       *dmocks.Uploads
		mfiles         *dmocks.Files
		req            *UploadRequest
		f              *flow.Request
		savedMCDIRPath string
	)

	BeforeEach(func() {
		savedMCDIRPath = app.MCDir.Path()
		config.Set("MCDIR", "/tmp")
		mdirs = dmocks.NewMDirs()
		muploads = dmocks.NewMUploads()
		mfiles = dmocks.NewMFiles()
		s = &uploadService{
			files:       mfiles,
			dirs:        mdirs,
			uploads:     muploads,
			tracker:     requestBlockTracker,
			writer:      &blockRequestWriter{},
			requestPath: &mcdirRequestPath{},
			fops:        file.OS,
		}

		f = &flow.Request{
			FlowChunkSize:   1024,
			FlowTotalSize:   int64(len("hello")),
			FlowChunkNumber: 1,
			Chunk:           []byte("hello"),
			FlowIdentifier:  "req",
		}

		req = &UploadRequest{f}
	})

	AfterEach(func() {
		config.Set("MCDIR", savedMCDIRPath)
	})

	Describe("Upload method tests", func() {
		Context("Successful upload cases", func() {
			It("Should succeed for a single block upload", func() {
				str := ""
				Expect(str).To(Equal(""))
			})
		})

		Context("Failed upload cases", func() {
			It("Should succeed for a multiple block upload", func() {

			})
		})
	})

	Describe("assemble method tests", func() {

	})

})
<|endoftext|>


<commit_before>package clicommand

import (
	"fmt"
	"net/http"
	"os"
	"time"

	"github.com/buildkite/agent/v3/api"
	"github.com/buildkite/agent/v3/cliconfig"
	"github.com/buildkite/roko"
	"github.com/urfave/cli"
)

type OIDCTokenConfig struct {
	Audience string `cli:"audience"`
	Lifetime int    `cli:"lifetime"`
	Job      string `cli:"job"      validate:"required"`

	// Global flags
	Debug       bool     `cli:"debug"`
	LogLevel    string   `cli:"log-level"`
	NoColor     bool     `cli:"no-color"`
	Experiments []string `cli:"experiment" normalize:"list"`
	Profile     string   `cli:"profile"`

	// API config
	DebugHTTP        bool   `cli:"debug-http"`
	AgentAccessToken string `cli:"agent-access-token" validate:"required"`
	Endpoint         string `cli:"endpoint"           validate:"required"`
	NoHTTP2          bool   `cli:"no-http2"`
}

const (
	oidcTokenDescription = `Usage:

   buildkite-agent oidc request-token [options...]

Description:
   Requests and prints an OIDC token from Buildkite that claims the Job ID
   (amongst other things) and the specified audience. If no audience is
   specified, the endpoint's default audience will be claimed.

Example:
   $ buildkite-agent oidc request-token --audience sts.amazonaws.com

   Requests and prints an OIDC token from Buildkite that claims the Job ID
   (amongst other things) and the audience "sts.amazonaws.com".
`
	backoffSeconds = 2
	maxAttempts    = 5
)

var OIDCRequestTokenCommand = cli.Command{
	Name:        "request-token",
	Usage:       "Requests and prints an OIDC token from Buildkite with the specified audience,",
	Description: oidcTokenDescription,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "audience",
			Value: "",
			Usage: "The audience that will consume the OIDC token",
		},
		cli.IntFlag{
			Name:  "lifetime",
			Value: 0,
			Usage: "The time (in seconds) the OIDC token will be valid for before expiry",
		},
		cli.StringFlag{
			Name:   "job",
			Value:  "",
			Usage:  "Buildkite Job Id to claim in the OIDC token",
			EnvVar: "BUILDKITE_JOB_ID",
		},

		// API Flags
		AgentAccessTokenFlag,
		EndpointFlag,
		NoHTTP2Flag,
		DebugHTTPFlag,

		// Global flags
		NoColorFlag,
		DebugFlag,
		LogLevelFlag,
		ExperimentsFlag,
		ProfileFlag,
	},
	Action: func(c *cli.Context) error {
		// The configuration will be loaded into this struct
		cfg := OIDCTokenConfig{}

		loader := cliconfig.Loader{CLI: c, Config: &cfg}
		warnings, err := loader.Load()
		if err != nil {
			fmt.Fprintf(c.App.ErrWriter, "%s\n", err)
			os.Exit(1)
		}

		l := CreateLogger(&cfg)

		// Now that we have a logger, log out the warnings that loading config generated
		for _, warning := range warnings {
			l.Warn("%s", warning)
		}

		// Setup any global configuration options
		done := HandleGlobalFlags(l, cfg)
		defer done()

		// Create the API client
		client := api.NewClient(l, loadAPIClientConfig(cfg, "AgentAccessToken"))

		// Request the token
		var token *api.OIDCToken

		if err := roko.NewRetrier(
			roko.WithMaxAttempts(maxAttempts),
			roko.WithStrategy(roko.Exponential(backoffSeconds*time.Second, 0)),
		).Do(func(r *roko.Retrier) error {
			req := &api.OIDCTokenRequest{
				Job:      cfg.Job,
				Audience: cfg.Audience,
				Lifetime: cfg.Lifetime,
			}

			var resp *api.Response
			token, resp, err = client.OIDCToken(req)
			if resp != nil {
				switch resp.StatusCode {
				// Don't bother retrying if the response was one of these statuses
				case http.StatusBadRequest, http.StatusUnauthorized, http.StatusForbidden, http.StatusUnprocessableEntity:
					r.Break()
					return err
				}
			}

			if err != nil {
				l.Warn("%s (%s)", err, r)
			}

			return err
		}); err != nil {
			if len(cfg.Audience) > 0 {
				l.Error("Could not obtain OIDC token for audience %s", cfg.Audience)
			} else {
				l.Error("Could not obtain OIDC token for default audience")
			}
			return err
		}

		fmt.Println(token.Token)
		return nil
	},
}
<commit_msg>Clarify what happens when 0 is supplied to the lifetime arg<commit_after>package clicommand

import (
	"fmt"
	"net/http"
	"os"
	"time"

	"github.com/buildkite/agent/v3/api"
	"github.com/buildkite/agent/v3/cliconfig"
	"github.com/buildkite/roko"
	"github.com/urfave/cli"
)

type OIDCTokenConfig struct {
	Audience string `cli:"audience"`
	Lifetime int    `cli:"lifetime"`
	Job      string `cli:"job"      validate:"required"`

	// Global flags
	Debug       bool     `cli:"debug"`
	LogLevel    string   `cli:"log-level"`
	NoColor     bool     `cli:"no-color"`
	Experiments []string `cli:"experiment" normalize:"list"`
	Profile     string   `cli:"profile"`

	// API config
	DebugHTTP        bool   `cli:"debug-http"`
	AgentAccessToken string `cli:"agent-access-token" validate:"required"`
	Endpoint         string `cli:"endpoint"           validate:"required"`
	NoHTTP2          bool   `cli:"no-http2"`
}

const (
	oidcTokenDescription = `Usage:

   buildkite-agent oidc request-token [options...]

Description:
   Requests and prints an OIDC token from Buildkite that claims the Job ID
   (amongst other things) and the specified audience. If no audience is
   specified, the endpoint's default audience will be claimed.

Example:
   $ buildkite-agent oidc request-token --audience sts.amazonaws.com

   Requests and prints an OIDC token from Buildkite that claims the Job ID
   (amongst other things) and the audience "sts.amazonaws.com".
`
	backoffSeconds = 2
	maxAttempts    = 5
)

var OIDCRequestTokenCommand = cli.Command{
	Name:        "request-token",
	Usage:       "Requests and prints an OIDC token from Buildkite with the specified audience,",
	Description: oidcTokenDescription,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "audience",
			Value: "",
			Usage: "The audience that will consume the OIDC token",
		},
		cli.IntFlag{
			Name:  "lifetime",
			Value: 0,
			Usage: `The time (in seconds) the OIDC token will be valid for before expiry.
Specfying 0 will result the in the API substituting its default value for the lifetime instead of 0.`,
		},
		cli.StringFlag{
			Name:   "job",
			Value:  "",
			Usage:  "Buildkite Job Id to claim in the OIDC token",
			EnvVar: "BUILDKITE_JOB_ID",
		},

		// API Flags
		AgentAccessTokenFlag,
		EndpointFlag,
		NoHTTP2Flag,
		DebugHTTPFlag,

		// Global flags
		NoColorFlag,
		DebugFlag,
		LogLevelFlag,
		ExperimentsFlag,
		ProfileFlag,
	},
	Action: func(c *cli.Context) error {
		// The configuration will be loaded into this struct
		cfg := OIDCTokenConfig{}

		loader := cliconfig.Loader{CLI: c, Config: &cfg}
		warnings, err := loader.Load()
		if err != nil {
			fmt.Fprintf(c.App.ErrWriter, "%s\n", err)
			os.Exit(1)
		}

		l := CreateLogger(&cfg)

		// Now that we have a logger, log out the warnings that loading config generated
		for _, warning := range warnings {
			l.Warn("%s", warning)
		}

		// Setup any global configuration options
		done := HandleGlobalFlags(l, cfg)
		defer done()

		// Create the API client
		client := api.NewClient(l, loadAPIClientConfig(cfg, "AgentAccessToken"))

		// Request the token
		var token *api.OIDCToken

		if err := roko.NewRetrier(
			roko.WithMaxAttempts(maxAttempts),
			roko.WithStrategy(roko.Exponential(backoffSeconds*time.Second, 0)),
		).Do(func(r *roko.Retrier) error {
			req := &api.OIDCTokenRequest{
				Job:      cfg.Job,
				Audience: cfg.Audience,
				Lifetime: cfg.Lifetime,
			}

			var resp *api.Response
			token, resp, err = client.OIDCToken(req)
			if resp != nil {
				switch resp.StatusCode {
				// Don't bother retrying if the response was one of these statuses
				case http.StatusBadRequest, http.StatusUnauthorized, http.StatusForbidden, http.StatusUnprocessableEntity:
					r.Break()
					return err
				}
			}

			if err != nil {
				l.Warn("%s (%s)", err, r)
			}

			return err
		}); err != nil {
			if len(cfg.Audience) > 0 {
				l.Error("Could not obtain OIDC token for audience %s", cfg.Audience)
			} else {
				l.Error("Could not obtain OIDC token for default audience")
			}
			return err
		}

		fmt.Println(token.Token)
		return nil
	},
}
<|endoftext|>


<commit_before>package acmecli

import (
	"crypto"
	"encoding/base64"
	"encoding/csv"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"
	"syscall"

	"github.com/square/go-jose"
	"github.com/tommie/acme-go"
	"github.com/tommie/acme-go/protocol"
)

// A SolverMode is an identifier passed in the ACME_MODE environment
// variable. It corresponds with acme.Solver methods.
type SolverMode string

const (
	ModeCanSolve SolverMode = "cansolve"
	ModeSolve    SolverMode = "solve"
)

// A ProcessSolver is an acme.Solver that uses a child process to
// solve challenges. Note an individual solver may
// be instantiated multiple times. It is up to calling code and the
// solver child to block or handle concurrency.
//
// The parent communicates to the child via environment variables,
// stdin and stdout. ACME_MODE is one of the SolverMode
// constants. ACME_ACCOUNT_JWK is a base64-encoded jose.JsonWebKey.
//
// For stdin and stdout, CSV with new-line (record) and tab (field)
// separators are used. stdin receives challenges where the first
// field is the challenge type. stdout provides responses, also with
// the first field being the challenge type.
//
// A non-zero exit code will cause the solver to return failure.
type ProcessSolver struct {
	accKey *jose.JsonWebKey
	name   string
	argv   []string
	attr   os.ProcAttr
}

// NewProcessSolver creates a new process solver. name, argv and attr
// follow the os.StartProcess semantics.
func NewProcessSolver(accKey *jose.JsonWebKey, name string, argv []string, attr *os.ProcAttr) *ProcessSolver {
	if attr == nil {
		attr = &os.ProcAttr{}
	}
	return &ProcessSolver{accKey, name, argv, *attr}
}

// Cost computes the cost of solving the challenges.
//
// It runs the solver in ModeCanSolve, feeds the challenges as CSV
// records and expects a single float64 on stdout. If stdout is empty,
// it is assumed the challenges cannot be solved together.
func (s *ProcessSolver) Cost(cs []protocol.Challenge) (cost float64, errRet error) {
	p, r, stop, err := s.start(cs, ModeCanSolve)
	if err != nil {
		return 0, fmt.Errorf("starting solver %q: %v", s.name, err)
	}
	defer func() {
		if err := stop(); err != nil {
			errRet = err
		}
	}()

	cr := csv.NewReader(r)
	cr.Comma = '\t'
	cost, err = readCost(cr)
	if err == io.EOF {
		return 0, acme.ErrUnsolvable
	} else if err != nil {
		p.Kill()
		return 0, fmt.Errorf("reading cost from %q: %v", s.name, err)
	}

	return cost, nil
}

// readCost attempts to read a single field of a single CSV record and
// parsing it as a float64. Returns io.EOF if no record is found.
func readCost(cr *csv.Reader) (float64, error) {
	rec, err := cr.Read()
	if err == io.EOF {
		// Empty reply: cannot solve the challenges.
		return 0, io.EOF
	} else if err != nil {
		return 0, err
	}

	if len(rec) != 1 {
		return 0, fmt.Errorf("expected one field, got %v", rec)
	}
	cost, err := strconv.ParseFloat(rec[0], 64)
	if err != nil {
		return 0, fmt.Errorf("expected number, got %q", rec[0])
	}

	return cost, nil
}

// Solve instantiates the solver for the given challenges.
//
// It passes the challenges to the child, emits the blank trailing
// record and waits for responses. The child must output one response
// per challenge, in order.
//
// To stop the instance, call the returned stop function. This will
// close stdin, signaling the child to exit.
func (s *ProcessSolver) Solve(cs []protocol.Challenge) ([]protocol.Response, func() error, error) {
	p, r, stop, err := s.start(cs, ModeSolve)
	if err != nil {
		return nil, nil, fmt.Errorf("starting solver %q: %v", s.name, err)
	}

	cr := csv.NewReader(r)
	cr.Comma = '\t'
	resps, err := readResponses(cr, cs)
	if err != nil {
		p.Kill()
		stop()
		return nil, nil, err
	}

	return resps, stop, nil
}

// readResponses reads len(cs) records from r and parses them into
// protocol.Response objects.
func readResponses(cr *csv.Reader, cs []protocol.Challenge) ([]protocol.Response, error) {
	var ret []protocol.Response
	for i, c := range cs {
		resp, err := readResponse(cr)
		if err == io.EOF {
			return nil, fmt.Errorf("got %d responses, want %d", i, len(cs))
		} else if err != nil {
			return nil, err
		}
		if resp.GetType() != c.GetType() {
			return nil, fmt.Errorf("mismatching response type: got %q, want %q", resp.GetType(), c.GetType())
		}
		ret = append(ret, resp)
	}
	return ret, nil
}

// readResponse reads a single record and parses it.
func readResponse(r *csv.Reader) (protocol.Response, error) {
	rec, err := r.Read()
	if err != nil {
		return nil, err
	}

	t := protocol.ChallengeType(rec[0])
	switch t {
	case protocol.ChallengeDNS01:
		if len(rec) != 2 {
			return nil, fmt.Errorf("expected two fields for %s response, got %v", t, rec)
		}
		return &protocol.DNS01Response{Resource: protocol.ResourceChallenge, Type: t, KeyAuthorization: rec[1]}, nil

	case protocol.ChallengeHTTP01:
		if len(rec) != 2 {
			return nil, fmt.Errorf("expected two fields for %s response, got %v", t, rec)
		}
		return &protocol.HTTP01Response{Resource: protocol.ResourceChallenge, Type: t, KeyAuthorization: rec[1]}, nil

	case protocol.ChallengePossession01:
		if len(rec) != 2 {
			return nil, fmt.Errorf("expected two fields for %s response, got %v", t, rec)
		}
		jws, err := jose.ParseSigned(rec[1])
		if err != nil {
			return nil, err
		}
		return &protocol.Possession01Response{Resource: protocol.ResourceChallenge, Type: t, Authorization: protocol.JSONWebSignature(*jws)}, nil

	case protocol.ChallengeTLSSNI01:
		if len(rec) != 2 {
			return nil, fmt.Errorf("expected two fields for %s response, got %v", t, rec)
		}
		return &protocol.TLSSNI01Response{Resource: protocol.ResourceChallenge, Type: t, KeyAuthorization: rec[1]}, nil

	default:
		return nil, fmt.Errorf("unknown challenge response type: %v", rec)
	}
}

// start starts a new child process and feeds it the provided
// challenges. Returns the process, the stdout reader, and a function
// to stop the process.
func (s *ProcessSolver) start(cs []protocol.Challenge, mode SolverMode) (*os.Process, io.ReadCloser, func() error, error) {
	attr := *&s.attr
	jwk, err := json.Marshal(s.accKey)
	if err != nil {
		return nil, nil, nil, err
	}
	if attr.Env == nil {
		attr.Env = os.Environ()
	}
	attr.Env = append(attr.Env, "ACME_MODE="+string(mode), "ACME_ACCOUNT_JWK="+string(jwk))

	stdin, stdinw, err := os.Pipe()
	if err != nil {
		return nil, nil, nil, err
	}
	defer stdin.Close()
	go func() {
		cw := csv.NewWriter(stdinw)
		cw.Comma = '\t'
		for _, c := range cs {
			if err := writeChallenge(cw, c, s.accKey); err != nil {
				panic(fmt.Errorf("error: writing challenge %+v: %v\n", c, err))
			}
		}
		// Terminate challenges with an empty record.
		cw.Write(nil)
		cw.Flush()
	}()

	stdoutr, stdout, err := os.Pipe()
	if err != nil {
		return nil, nil, nil, err
	}
	defer stdout.Close()
	if len(attr.Files) <= syscall.Stderr {
		attr.Files = make([]*os.File, syscall.Stderr+1)
	}
	attr.Files[syscall.Stdin] = stdin
	attr.Files[syscall.Stdout] = stdout
	attr.Files[syscall.Stderr] = os.Stderr

	p, err := os.StartProcess(s.name, s.argv, &attr)
	stop := func() error {
		go func() {
			defer stdoutr.Close()
			bs := make([]byte, 1024)
			for {
				// Continue reading the stdout pipe and then close,
				// to avoid SIGPIPE in the child.
				_, err := stdoutr.Read(bs)
				if err == io.EOF {
					break
				} else if err != nil {
					return
				}
			}
		}()

		// Closing stdin signals to the child to stop the solver and terminate.
		err := stdinw.Close()
		if err != nil {
			return err
		}
		ps, err := p.Wait()
		if err != nil {
			return err
		}
		if !ps.Success() {
			return fmt.Errorf("solver %q failed: %s", s.name, ps)
		}
		return nil
	}
	return p, stdoutr, stop, err
}

// writeChallenge marshals the challenge and writes it as CSV.
func writeChallenge(w *csv.Writer, c protocol.Challenge, accKey *jose.JsonWebKey) error {
	switch cc := c.(type) {
	case *protocol.DNS01Challenge:
		ka, err := protocol.KeyAuthz(cc.Token, accKey)
		if err != nil {
			return err
		}
		return w.Write([]string{string(cc.GetType()), cc.Token, ka})

	case *protocol.HTTP01Challenge:
		ka, err := protocol.KeyAuthz(cc.Token, accKey)
		if err != nil {
			return err
		}
		return w.Write([]string{string(cc.GetType()), cc.Token, ka})

	case *protocol.Possession01Challenge:
		rec := []string{string(cc.GetType())}
		for _, bs := range cc.Certs {
			rec = append(rec, base64.URLEncoding.EncodeToString(bs))
		}
		return w.Write(rec)

	case *protocol.TLSSNI01Challenge:
		ka, err := protocol.KeyAuthz(cc.Token, accKey)
		if err != nil {
			return err
		}
		rec := []string{string(cc.GetType()), cc.Token, ka}
		z := ka
		h := crypto.SHA256.New()
		for i := 0; i < cc.N; i++ {
			h.Reset()
			h.Write([]byte(z))
			// EncodeToString casing is undefined. https://github.com/golang/go/issues/11254
			z = strings.ToLower(hex.EncodeToString(h.Sum(nil)))
			rec = append(rec, strings.Join([]string{z[:32], z[32:], protocol.TLSSNI01Suffix}, "."))
		}
		return w.Write(rec)

	default:
		return fmt.Errorf("unknown challenge type: %#v", c)
	}
}
<commit_msg>Fix return value of Cost.<commit_after>package acmecli

import (
	"crypto"
	"encoding/base64"
	"encoding/csv"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"
	"syscall"

	"github.com/square/go-jose"
	"github.com/tommie/acme-go"
	"github.com/tommie/acme-go/protocol"
)

// A SolverMode is an identifier passed in the ACME_MODE environment
// variable. It corresponds with acme.Solver methods.
type SolverMode string

const (
	ModeCanSolve SolverMode = "cansolve"
	ModeSolve    SolverMode = "solve"
)

// A ProcessSolver is an acme.Solver that uses a child process to
// solve challenges. Note an individual solver may
// be instantiated multiple times. It is up to calling code and the
// solver child to block or handle concurrency.
//
// The parent communicates to the child via environment variables,
// stdin and stdout. ACME_MODE is one of the SolverMode
// constants. ACME_ACCOUNT_JWK is a base64-encoded jose.JsonWebKey.
//
// For stdin and stdout, CSV with new-line (record) and tab (field)
// separators are used. stdin receives challenges where the first
// field is the challenge type. stdout provides responses, also with
// the first field being the challenge type.
//
// A non-zero exit code will cause the solver to return failure.
type ProcessSolver struct {
	accKey *jose.JsonWebKey
	name   string
	argv   []string
	attr   os.ProcAttr
}

// NewProcessSolver creates a new process solver. name, argv and attr
// follow the os.StartProcess semantics.
func NewProcessSolver(accKey *jose.JsonWebKey, name string, argv []string, attr *os.ProcAttr) *ProcessSolver {
	if attr == nil {
		attr = &os.ProcAttr{}
	}
	return &ProcessSolver{accKey, name, argv, *attr}
}

// Cost computes the cost of solving the challenges.
//
// It runs the solver in ModeCanSolve, feeds the challenges as CSV
// records and expects a single float64 on stdout. If stdout is empty,
// it is assumed the challenges cannot be solved together.
func (s *ProcessSolver) Cost(cs []protocol.Challenge) (cost float64, errRet error) {
	p, r, stop, err := s.start(cs, ModeCanSolve)
	if err != nil {
		return 0, fmt.Errorf("starting solver %q: %v", s.name, err)
	}
	defer func() {
		if err := stop(); err != nil && errRet == nil {
			errRet = err
		}
	}()

	cr := csv.NewReader(r)
	cr.Comma = '\t'
	cost, err = readCost(cr)
	if err == io.EOF {
		return 0, acme.ErrUnsolvable
	} else if err != nil {
		p.Kill()
		return 0, fmt.Errorf("reading cost from %q: %v", s.name, err)
	}

	return cost, nil
}

// readCost attempts to read a single field of a single CSV record and
// parsing it as a float64. Returns io.EOF if no record is found.
func readCost(cr *csv.Reader) (float64, error) {
	rec, err := cr.Read()
	if err == io.EOF {
		// Empty reply: cannot solve the challenges.
		return 0, io.EOF
	} else if err != nil {
		return 0, err
	}

	if len(rec) != 1 {
		return 0, fmt.Errorf("expected one field, got %v", rec)
	}
	cost, err := strconv.ParseFloat(rec[0], 64)
	if err != nil {
		return 0, fmt.Errorf("expected number, got %q", rec[0])
	}

	return cost, nil
}

// Solve instantiates the solver for the given challenges.
//
// It passes the challenges to the child, emits the blank trailing
// record and waits for responses. The child must output one response
// per challenge, in order.
//
// To stop the instance, call the returned stop function. This will
// close stdin, signaling the child to exit.
func (s *ProcessSolver) Solve(cs []protocol.Challenge) ([]protocol.Response, func() error, error) {
	p, r, stop, err := s.start(cs, ModeSolve)
	if err != nil {
		return nil, nil, fmt.Errorf("starting solver %q: %v", s.name, err)
	}

	cr := csv.NewReader(r)
	cr.Comma = '\t'
	resps, err := readResponses(cr, cs)
	if err != nil {
		p.Kill()
		stop()
		return nil, nil, err
	}

	return resps, stop, nil
}

// readResponses reads len(cs) records from r and parses them into
// protocol.Response objects.
func readResponses(cr *csv.Reader, cs []protocol.Challenge) ([]protocol.Response, error) {
	var ret []protocol.Response
	for i, c := range cs {
		resp, err := readResponse(cr)
		if err == io.EOF {
			return nil, fmt.Errorf("got %d responses, want %d", i, len(cs))
		} else if err != nil {
			return nil, err
		}
		if resp.GetType() != c.GetType() {
			return nil, fmt.Errorf("mismatching response type: got %q, want %q", resp.GetType(), c.GetType())
		}
		ret = append(ret, resp)
	}
	return ret, nil
}

// readResponse reads a single record and parses it.
func readResponse(r *csv.Reader) (protocol.Response, error) {
	rec, err := r.Read()
	if err != nil {
		return nil, err
	}

	t := protocol.ChallengeType(rec[0])
	switch t {
	case protocol.ChallengeDNS01:
		if len(rec) != 2 {
			return nil, fmt.Errorf("expected two fields for %s response, got %v", t, rec)
		}
		return &protocol.DNS01Response{Resource: protocol.ResourceChallenge, Type: t, KeyAuthorization: rec[1]}, nil

	case protocol.ChallengeHTTP01:
		if len(rec) != 2 {
			return nil, fmt.Errorf("expected two fields for %s response, got %v", t, rec)
		}
		return &protocol.HTTP01Response{Resource: protocol.ResourceChallenge, Type: t, KeyAuthorization: rec[1]}, nil

	case protocol.ChallengePossession01:
		if len(rec) != 2 {
			return nil, fmt.Errorf("expected two fields for %s response, got %v", t, rec)
		}
		jws, err := jose.ParseSigned(rec[1])
		if err != nil {
			return nil, err
		}
		return &protocol.Possession01Response{Resource: protocol.ResourceChallenge, Type: t, Authorization: protocol.JSONWebSignature(*jws)}, nil

	case protocol.ChallengeTLSSNI01:
		if len(rec) != 2 {
			return nil, fmt.Errorf("expected two fields for %s response, got %v", t, rec)
		}
		return &protocol.TLSSNI01Response{Resource: protocol.ResourceChallenge, Type: t, KeyAuthorization: rec[1]}, nil

	default:
		return nil, fmt.Errorf("unknown challenge response type: %v", rec)
	}
}

// start starts a new child process and feeds it the provided
// challenges. Returns the process, the stdout reader, and a function
// to stop the process.
func (s *ProcessSolver) start(cs []protocol.Challenge, mode SolverMode) (*os.Process, io.ReadCloser, func() error, error) {
	attr := *&s.attr
	jwk, err := json.Marshal(s.accKey)
	if err != nil {
		return nil, nil, nil, err
	}
	if attr.Env == nil {
		attr.Env = os.Environ()
	}
	attr.Env = append(attr.Env, "ACME_MODE="+string(mode), "ACME_ACCOUNT_JWK="+string(jwk))

	stdin, stdinw, err := os.Pipe()
	if err != nil {
		return nil, nil, nil, err
	}
	defer stdin.Close()
	go func() {
		cw := csv.NewWriter(stdinw)
		cw.Comma = '\t'
		for _, c := range cs {
			if err := writeChallenge(cw, c, s.accKey); err != nil {
				panic(fmt.Errorf("error: writing challenge %+v: %v\n", c, err))
			}
		}
		// Terminate challenges with an empty record.
		cw.Write(nil)
		cw.Flush()
	}()

	stdoutr, stdout, err := os.Pipe()
	if err != nil {
		return nil, nil, nil, err
	}
	defer stdout.Close()
	if len(attr.Files) <= syscall.Stderr {
		attr.Files = make([]*os.File, syscall.Stderr+1)
	}
	attr.Files[syscall.Stdin] = stdin
	attr.Files[syscall.Stdout] = stdout
	attr.Files[syscall.Stderr] = os.Stderr

	p, err := os.StartProcess(s.name, s.argv, &attr)
	stop := func() error {
		go func() {
			defer stdoutr.Close()
			bs := make([]byte, 1024)
			for {
				// Continue reading the stdout pipe and then close,
				// to avoid SIGPIPE in the child.
				_, err := stdoutr.Read(bs)
				if err == io.EOF {
					break
				} else if err != nil {
					return
				}
			}
		}()

		// Closing stdin signals to the child to stop the solver and terminate.
		err := stdinw.Close()
		if err != nil {
			return err
		}
		ps, err := p.Wait()
		if err != nil {
			return err
		}
		if !ps.Success() {
			return fmt.Errorf("solver %q failed: %s", s.name, ps)
		}
		return nil
	}
	return p, stdoutr, stop, err
}

// writeChallenge marshals the challenge and writes it as CSV.
func writeChallenge(w *csv.Writer, c protocol.Challenge, accKey *jose.JsonWebKey) error {
	switch cc := c.(type) {
	case *protocol.DNS01Challenge:
		ka, err := protocol.KeyAuthz(cc.Token, accKey)
		if err != nil {
			return err
		}
		return w.Write([]string{string(cc.GetType()), cc.Token, ka})

	case *protocol.HTTP01Challenge:
		ka, err := protocol.KeyAuthz(cc.Token, accKey)
		if err != nil {
			return err
		}
		return w.Write([]string{string(cc.GetType()), cc.Token, ka})

	case *protocol.Possession01Challenge:
		rec := []string{string(cc.GetType())}
		for _, bs := range cc.Certs {
			rec = append(rec, base64.URLEncoding.EncodeToString(bs))
		}
		return w.Write(rec)

	case *protocol.TLSSNI01Challenge:
		ka, err := protocol.KeyAuthz(cc.Token, accKey)
		if err != nil {
			return err
		}
		rec := []string{string(cc.GetType()), cc.Token, ka}
		z := ka
		h := crypto.SHA256.New()
		for i := 0; i < cc.N; i++ {
			h.Reset()
			h.Write([]byte(z))
			// EncodeToString casing is undefined. https://github.com/golang/go/issues/11254
			z = strings.ToLower(hex.EncodeToString(h.Sum(nil)))
			rec = append(rec, strings.Join([]string{z[:32], z[32:], protocol.TLSSNI01Suffix}, "."))
		}
		return w.Write(rec)

	default:
		return fmt.Errorf("unknown challenge type: %#v", c)
	}
}
<|endoftext|>


<commit_before>package indexer

import (
	"errors"
	"fmt"
	"net/url"
	"os"
	"regexp"
	"strconv"
	"strings"
	"text/scanner"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/cardigann/cardigann/torznab"
)

const (
	filterTimeFormat = time.RFC1123Z
)

var (
	filterLogger          logrus.FieldLogger
	filterCategoryMapping torznab.CategoryMapping
)

func init() {
	filterLogger = logrus.New()
	filterLogger.(*logrus.Logger).Level = logrus.DebugLevel
	filterLogger.(*logrus.Logger).Out = os.Stderr
}

func invokeFilter(name string, args interface{}, value string) (string, error) {
	switch name {
	case "querystring":
		param, ok := args.(string)
		if !ok {
			return "", fmt.Errorf("Filter %q requires a string argument", name)
		}
		return filterQueryString(param, value)

	case "dateparse":
		format, ok := args.(string)
		if !ok {
			return "", fmt.Errorf("Filter %q requires a string argument", name)
		}
		return filterDateParse(format, value)

	case "regexp":
		pattern, ok := args.(string)
		if !ok {
			return "", fmt.Errorf("Filter %q requires a string argument", name)
		}
		return filterRegexp(pattern, value)

	case "split":
		sep, ok := (args.([]interface{}))[0].(string)
		if !ok {
			return "", fmt.Errorf("Filter %q requires a string argument at idx 0", name)
		}
		pos, ok := (args.([]interface{}))[1].(int)
		if !ok {
			return "", fmt.Errorf("Filter %q requires an int argument at idx 1", name)
		}
		return filterSplit(sep, pos, value)

	case "timeago":
		return filterTimeAgo(value, time.Now())

	case "reltime":
		format, ok := args.(string)
		if !ok {
			return "", fmt.Errorf("Filter %q requires a string argument", name)
		}
		return filterRelTime(value, format, time.Now())

	case "mapcats":
		return filterMapCategory(value)
	}

	return "", errors.New("Unknown filter " + name)
}

func filterQueryString(param string, value string) (string, error) {
	u, err := url.Parse(value)
	if err != nil {
		return "", err
	}
	return u.Query().Get(param), nil
}

func filterDateParse(format string, value string) (string, error) {
	t, err := time.Parse(format, value)
	if err != nil {
		return "", err
	}
	return t.Format(filterTimeFormat), nil
}

func filterSplit(sep string, pos int, value string) (string, error) {
	frags := strings.Split(value, sep)
	if pos < 0 {
		pos = len(frags) + pos
	}
	return frags[pos], nil
}

func filterRegexp(pattern string, value string) (string, error) {
	re, err := regexp.Compile(pattern)
	if err != nil {
		return "", err
	}

	matches := re.FindStringSubmatch(value)

	if len(matches) == 0 {
		return "", errors.New("No matches found for pattern")
	}

	filterLogger.WithFields(logrus.Fields{"matches": matches}).Debug("Regex matched")

	if len(matches) > 1 {
		return matches[1], nil
	}

	return matches[0], nil
}

func splitDecimalStr(s string) (int, float64, error) {
	if parts := strings.SplitN(s, ".", 2); len(parts) == 2 {
		i, err := strconv.Atoi(parts[0])
		if err != nil {
			return 0, 0, err
		}
		f, err := strconv.ParseFloat("0."+parts[1], 64)
		if err != nil {
			return 0, 0, err
		}
		return i, f, nil
	}
	i, err := strconv.Atoi(s)
	if err != nil {
		return 0, 0, err
	}
	return i, 0, nil
}

func filterTimeAgo(src string, now time.Time) (string, error) {
	var s scanner.Scanner
	s.Init(strings.NewReader(src))
	var tok rune
	for tok != scanner.EOF {
		tok = s.Scan()
		switch s.TokenText() {
		case ",", "ago", "", "and":
			continue
		}

		v, fraction, err := splitDecimalStr(s.TokenText())
		if err != nil {
			return "", fmt.Errorf(
				"Failed to parse decimal time %q in time format at %s", s.TokenText(), s.Pos())
		}

		tok = s.Scan()
		if tok == scanner.EOF {
			return "", fmt.Errorf(
				"Expected a time unit at %s", s.TokenText(), s.Pos())
		}

		switch strings.TrimSuffix(s.TokenText(), "s") {
		case "year":
			now = now.AddDate(-v, 0, 0)
			if fraction > 0 {
				now = now.Add(time.Duration(float64(now.AddDate(-1, 0, 0).Sub(now)) * fraction))
			}
		case "month":
			now = now.AddDate(0, -v, 0)
			if fraction > 0 {
				now = now.Add(time.Duration(float64(now.AddDate(0, -1, 0).Sub(now)) * fraction))
			}
		case "week":
			now = now.AddDate(0, 0, -7)
			if fraction > 0 {
				now = now.Add(time.Duration(float64(now.AddDate(0, 0, -7).Sub(now)) * fraction))
			}
		case "day":
			now = now.AddDate(0, 0, -v)
			if fraction > 0 {
				now = now.Add(time.Minute * -time.Duration(fraction*1440))
			}
		case "hour":
			now = now.Add(time.Hour * -time.Duration(v))
			if fraction > 0 {
				now = now.Add(time.Second * -time.Duration(fraction*3600))
			}
		case "minute":
			now = now.Add(time.Minute * -time.Duration(v))
			if fraction > 0 {
				now = now.Add(time.Second * -time.Duration(fraction*60))
			}
		case "second":
			now = now.Add(time.Second * -time.Duration(v))
		default:
			return "", fmt.Errorf("Unsupporting unit of time %q", s.TokenText())
		}
	}

	return now.Format(filterTimeFormat), nil
}

func filterRelTime(src string, format string, now time.Time) (string, error) {
	out := strings.Replace(src, "Today", now.Format(format), -1)
	out = strings.Replace(src, "today", now.Format(format), -1)
	out = strings.Replace(out, "Yesterday", now.AddDate(0, 0, -1).Format(format), -1)
	out = strings.Replace(out, "yesterday", now.AddDate(0, 0, -1).Format(format), -1)
	return out, nil
}

func filterMapCategory(value string) (string, error) {
	catID, err := strconv.Atoi(value)
	if err != nil {
		return "", fmt.Errorf("Unable to parse category id %s", value)
	}
	mappedCat, ok := filterCategoryMapping[catID]
	if !ok {
		return "", fmt.Errorf("No category mapping found for id %d", catID)
	}
	return strconv.Itoa(mappedCat.ID), nil
}
<commit_msg>Support uppercase today/yesterday in filters<commit_after>package indexer

import (
	"errors"
	"fmt"
	"net/url"
	"os"
	"regexp"
	"strconv"
	"strings"
	"text/scanner"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/cardigann/cardigann/torznab"
)

const (
	filterTimeFormat = time.RFC1123Z
)

var (
	filterLogger          logrus.FieldLogger
	filterCategoryMapping torznab.CategoryMapping
)

func init() {
	filterLogger = logrus.New()
	filterLogger.(*logrus.Logger).Level = logrus.DebugLevel
	filterLogger.(*logrus.Logger).Out = os.Stderr
}

func invokeFilter(name string, args interface{}, value string) (string, error) {
	switch name {
	case "querystring":
		param, ok := args.(string)
		if !ok {
			return "", fmt.Errorf("Filter %q requires a string argument", name)
		}
		return filterQueryString(param, value)

	case "dateparse":
		format, ok := args.(string)
		if !ok {
			return "", fmt.Errorf("Filter %q requires a string argument", name)
		}
		return filterDateParse(format, value)

	case "regexp":
		pattern, ok := args.(string)
		if !ok {
			return "", fmt.Errorf("Filter %q requires a string argument", name)
		}
		return filterRegexp(pattern, value)

	case "split":
		sep, ok := (args.([]interface{}))[0].(string)
		if !ok {
			return "", fmt.Errorf("Filter %q requires a string argument at idx 0", name)
		}
		pos, ok := (args.([]interface{}))[1].(int)
		if !ok {
			return "", fmt.Errorf("Filter %q requires an int argument at idx 1", name)
		}
		return filterSplit(sep, pos, value)

	case "timeago":
		return filterTimeAgo(value, time.Now())

	case "reltime":
		format, ok := args.(string)
		if !ok {
			return "", fmt.Errorf("Filter %q requires a string argument", name)
		}
		return filterRelTime(value, format, time.Now())

	case "mapcats":
		return filterMapCategory(value)
	}

	return "", errors.New("Unknown filter " + name)
}

func filterQueryString(param string, value string) (string, error) {
	u, err := url.Parse(value)
	if err != nil {
		return "", err
	}
	return u.Query().Get(param), nil
}

func filterDateParse(format string, value string) (string, error) {
	t, err := time.Parse(format, value)
	if err != nil {
		return "", err
	}
	return t.Format(filterTimeFormat), nil
}

func filterSplit(sep string, pos int, value string) (string, error) {
	frags := strings.Split(value, sep)
	if pos < 0 {
		pos = len(frags) + pos
	}
	return frags[pos], nil
}

func filterRegexp(pattern string, value string) (string, error) {
	re, err := regexp.Compile(pattern)
	if err != nil {
		return "", err
	}

	matches := re.FindStringSubmatch(value)

	if len(matches) == 0 {
		return "", errors.New("No matches found for pattern")
	}

	filterLogger.WithFields(logrus.Fields{"matches": matches}).Debug("Regex matched")

	if len(matches) > 1 {
		return matches[1], nil
	}

	return matches[0], nil
}

func splitDecimalStr(s string) (int, float64, error) {
	if parts := strings.SplitN(s, ".", 2); len(parts) == 2 {
		i, err := strconv.Atoi(parts[0])
		if err != nil {
			return 0, 0, err
		}
		f, err := strconv.ParseFloat("0."+parts[1], 64)
		if err != nil {
			return 0, 0, err
		}
		return i, f, nil
	}
	i, err := strconv.Atoi(s)
	if err != nil {
		return 0, 0, err
	}
	return i, 0, nil
}

func filterTimeAgo(src string, now time.Time) (string, error) {
	var s scanner.Scanner
	s.Init(strings.NewReader(src))
	var tok rune
	for tok != scanner.EOF {
		tok = s.Scan()
		switch s.TokenText() {
		case ",", "ago", "", "and":
			continue
		}

		v, fraction, err := splitDecimalStr(s.TokenText())
		if err != nil {
			return "", fmt.Errorf(
				"Failed to parse decimal time %q in time format at %s", s.TokenText(), s.Pos())
		}

		tok = s.Scan()
		if tok == scanner.EOF {
			return "", fmt.Errorf(
				"Expected a time unit at %s", s.TokenText(), s.Pos())
		}

		switch strings.TrimSuffix(s.TokenText(), "s") {
		case "year":
			now = now.AddDate(-v, 0, 0)
			if fraction > 0 {
				now = now.Add(time.Duration(float64(now.AddDate(-1, 0, 0).Sub(now)) * fraction))
			}
		case "month":
			now = now.AddDate(0, -v, 0)
			if fraction > 0 {
				now = now.Add(time.Duration(float64(now.AddDate(0, -1, 0).Sub(now)) * fraction))
			}
		case "week":
			now = now.AddDate(0, 0, -7)
			if fraction > 0 {
				now = now.Add(time.Duration(float64(now.AddDate(0, 0, -7).Sub(now)) * fraction))
			}
		case "day":
			now = now.AddDate(0, 0, -v)
			if fraction > 0 {
				now = now.Add(time.Minute * -time.Duration(fraction*1440))
			}
		case "hour":
			now = now.Add(time.Hour * -time.Duration(v))
			if fraction > 0 {
				now = now.Add(time.Second * -time.Duration(fraction*3600))
			}
		case "minute":
			now = now.Add(time.Minute * -time.Duration(v))
			if fraction > 0 {
				now = now.Add(time.Second * -time.Duration(fraction*60))
			}
		case "second":
			now = now.Add(time.Second * -time.Duration(v))
		default:
			return "", fmt.Errorf("Unsupporting unit of time %q", s.TokenText())
		}
	}

	return now.Format(filterTimeFormat), nil
}

func filterRelTime(src string, format string, now time.Time) (string, error) {
	out := src

	for from, to := range map[string]string{
		"today":     now.Format(format),
		"Today":     now.Format(format),
		"yesterday": now.AddDate(0, 0, -1).Format(format),
		"Yesterday": now.AddDate(0, 0, -1).Format(format),
	} {
		out = strings.Replace(out, from, to, -1)
	}

	return out, nil
}

func filterMapCategory(value string) (string, error) {
	catID, err := strconv.Atoi(value)
	if err != nil {
		return "", fmt.Errorf("Unable to parse category id %s", value)
	}
	mappedCat, ok := filterCategoryMapping[catID]
	if !ok {
		return "", fmt.Errorf("No category mapping found for id %d", catID)
	}
	return strconv.Itoa(mappedCat.ID), nil
}
<|endoftext|>