<commit_before>/***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# The Initial Developer of the Original Code is the Mozilla Foundation.
# Portions created by the Initial Developer are Copyright (C) 2012
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Ian Neubert (ian@ianneubert.com)
#
# ***** END LICENSE BLOCK *****/

package sqs

import (
  "encoding/base64"
  "fmt"
  . "github.com/mozilla-services/heka/pipeline"
  "github.com/crowdmob/goamz/sqs"
  "sync"
)

type SqsInput struct {
  conf         *SqsInputConfig
  decoder      Decoder
  queue        *sqs.Queue
  running      bool
}

type SqsInputConfig struct {
  // AWS access key id
  AccessKey string `toml:"access_key"`
  // AWS secret access key
  SecretKey string `toml:"secret_key"`
  // Queue name
  Queue string
  // Encoder to use
  Decoder string
  // AWS Region to connect to
  Region string
}

func (s *SqsInput) ConfigStruct() interface{} {
  return &SqsInputConfig{
    Decoder: "ProtobufDecoder",
    Region: "us.east",
  }
}

func (s *SqsInput) Init(config interface{}) (err error) {
  s.running = true
  s.conf = config.(*SqsInputConfig)

  if s.conf.AccessKey == "" {
    return fmt.Errorf("AccessKey must be set")
  }

  if s.conf.SecretKey == "" {
    return fmt.Errorf("SecretKey must be set")
  }

  if s.conf.Queue == "" {
    return fmt.Errorf("Queue must be set")
  }

  // create sqs client
  client, err := sqs.NewFrom(s.conf.AccessKey, s.conf.SecretKey, s.conf.Region)
  if err != nil {
    return fmt.Errorf("Could not create SQS client: %v", err)
  }

  // get the SQS queue
  s.queue, err = client.GetQueue(s.conf.Queue)
  if err != nil {
    return fmt.Errorf("Could not get SQS queue: %v", err)
  }
  return
}

func (s *SqsInput) Run(ir InputRunner, h PluginHelper) (err error) {
  var (
    decoder Decoder
    pack    *PipelinePack
    ok      bool
    resp    *sqs.ReceiveMessageResponse
  )

  if s.conf.Decoder != "" {
    if decoder, ok = h.PipelineConfig().Decoder(s.conf.Decoder); !ok {
      return fmt.Errorf("Decoder not found: %s", s.conf.Decoder)
    }
  }

  packSupply := ir.InChan()

  // create a wait group
  var wg sync.WaitGroup

  for s.running {
    // ir.LogMessage("Getting messages from SQS...")
    resp, err = s.queue.ReceiveMessage(10)
    if err != nil {
      ir.LogError(fmt.Errorf("Could not receive messages: %v", err))
    }

    for _, message := range resp.Messages {
      pack = <-packSupply

      wg.Add(1)
      go s.processMessage(&message, ir, h, pack, decoder, &wg)
    }

    // wait for each goroutine to exit
    wg.Wait()
  }

  return
}

func (s *SqsInput) processMessage(message *sqs.Message, ir InputRunner, h PluginHelper, pack *PipelinePack, decoder Decoder, wg  *sync.WaitGroup) {
  var (
    packs   []*PipelinePack
    err     error
  )

  data, err := base64.StdEncoding.DecodeString(message.Body)
  if err != nil {
    ir.LogError(fmt.Errorf("Could not decode message: %v", err))
  }

  pack.MsgBytes = data

  packs, err = decoder.Decode(pack)
  if packs != nil {
    for _, p := range packs {
      // ir.LogMessage("Injecting pack...")
      ir.Inject(p)
    }
  } else {
    if err != nil {
      ir.LogError(fmt.Errorf("Couldn't parse SQS message: %s - %v", message.Body, err))
    }
  }

  // ir.LogMessage("Deleting message from SQS...")
  _, err = s.queue.DeleteMessage(message)
  if err != nil {
    ir.LogError(fmt.Errorf("Could not delete message: %v - %v", message, err))
  }

  // ir.LogMessage("Done.")
  wg.Done()
}

func (s *SqsInput) Stop() {
  s.running = false
}

func init() {
  RegisterPlugin("SqsInput", func() interface{} {
    return new(SqsInput)
  })
}
<commit_msg>SQS messages will now be deleted correctly<commit_after>/***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# The Initial Developer of the Original Code is the Mozilla Foundation.
# Portions created by the Initial Developer are Copyright (C) 2012
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Ian Neubert (ian@ianneubert.com)
#
# ***** END LICENSE BLOCK *****/

package sqs

import (
  "encoding/base64"
  "fmt"
  . "github.com/mozilla-services/heka/pipeline"
  "github.com/crowdmob/goamz/sqs"
  "sync"
)

type SqsInput struct {
  conf         *SqsInputConfig
  decoder      Decoder
  queue        *sqs.Queue
  running      bool
}

type SqsInputConfig struct {
  // AWS access key id
  AccessKey string `toml:"access_key"`
  // AWS secret access key
  SecretKey string `toml:"secret_key"`
  // Queue name
  Queue string
  // Encoder to use
  Decoder string
  // AWS Region to connect to
  Region string
}

func (s *SqsInput) ConfigStruct() interface{} {
  return &SqsInputConfig{
    Decoder: "ProtobufDecoder",
    Region: "us.east",
  }
}

func (s *SqsInput) Init(config interface{}) (err error) {
  s.running = true
  s.conf = config.(*SqsInputConfig)

  if s.conf.AccessKey == "" {
    return fmt.Errorf("AccessKey must be set")
  }

  if s.conf.SecretKey == "" {
    return fmt.Errorf("SecretKey must be set")
  }

  if s.conf.Queue == "" {
    return fmt.Errorf("Queue must be set")
  }

  // create sqs client
  client, err := sqs.NewFrom(s.conf.AccessKey, s.conf.SecretKey, s.conf.Region)
  if err != nil {
    return fmt.Errorf("Could not create SQS client: %v", err)
  }

  // get the SQS queue
  s.queue, err = client.GetQueue(s.conf.Queue)
  if err != nil {
    return fmt.Errorf("Could not get SQS queue: %v", err)
  }
  return
}

func (s *SqsInput) Run(ir InputRunner, h PluginHelper) (err error) {
  var (
    decoder Decoder
    pack    *PipelinePack
    ok      bool
    resp    *sqs.ReceiveMessageResponse
  )

  if s.conf.Decoder != "" {
    if decoder, ok = h.PipelineConfig().Decoder(s.conf.Decoder); !ok {
      return fmt.Errorf("Decoder not found: %s", s.conf.Decoder)
    }
  }

  packSupply := ir.InChan()

  // create a wait group
  var wg sync.WaitGroup

  for s.running {
    // ir.LogMessage("Getting messages from SQS...")
    resp, err = s.queue.ReceiveMessage(10)
    if err != nil {
      ir.LogError(fmt.Errorf("Could not receive messages: %v", err))
    }

    for _, message := range resp.Messages {
      pack = <-packSupply

      wg.Add(1)
      go s.processMessage(message, ir, h, pack, decoder, &wg)
    }

    // wait for each goroutine to exit
    // ir.LogMessage("Waiting...")
    wg.Wait()
  }

  return
}

func (s *SqsInput) processMessage(message sqs.Message, ir InputRunner, h PluginHelper, pack *PipelinePack, decoder Decoder, wg  *sync.WaitGroup) {
  var (
    packs   []*PipelinePack
    err     error
  )

  data, err := base64.StdEncoding.DecodeString(message.Body)
  if err != nil {
    ir.LogError(fmt.Errorf("Could not decode message: %v", err))
  }

  pack.MsgBytes = data

  packs, err = decoder.Decode(pack)
  if packs != nil {
    for _, p := range packs {
      // ir.LogMessage("Injecting pack...")
      ir.Inject(p)
    }
  } else {
    if err != nil {
      ir.LogError(fmt.Errorf("Couldn't parse SQS message: %s - %v", message.Body, err))
    }
  }

  // ir.LogMessage("Deleting message from SQS...")
  _, err = s.queue.DeleteMessage(&message)
  if err != nil {
    ir.LogError(fmt.Errorf("Could not delete message: %v - %v", message, err))
  } else {
    // ir.LogMessage(fmt.Sprintf("Deleted message: %v", message.MessageId))
  }

  // ir.LogMessage("Done.")
  wg.Done()
}

func (s *SqsInput) Stop() {
  s.running = false
}

func init() {
  RegisterPlugin("SqsInput", func() interface{} {
    return new(SqsInput)
  })
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	"code.google.com/p/gcfg"
	"github.com/mitchellh/go-homedir"
	"github.com/spf13/cobra"
)

// The priority order for the config file is:
//		Lowest: The defaults set by the flags.
//		Middle: The values loaded by the config file.
//		Highest: Flags manually set by the user at load.

var config Config

type Config struct {
	ApiPort     uint16
	RpcPort     uint16
	NoBootstrap bool

	ConfigFilename string
}

// confilgFilenameDefault checks multiple directories for a config file and
// loads the first one it finds. "" is returned if no config file is found.
func configFilenameDefault() string {
	// Try home folder.
	home, err := homedir.Dir()
	if err == nil {
		// Check home/.config/config
		filename := home + "/.config/config"
		if _, err := os.Stat(filename); err == nil {
			return filename
		}

		// Check home/.sia/config
		filename = home + "/.sia/config"
		if _, err := os.Stat(filename); err == nil {
			return filename
		}

		// Check home/.sia.conf
		filename = home + "/.sia.conf"
		if _, err := os.Stat(filename); err == nil {
			return filename
		}
	}

	// Try /etc/sia.conf
	filename := "etc/sia.conf"
	if _, err := os.Stat(filename); err == nil {
		return filename
	}

	return ""
}

// startEnvironment calls createEnvironment(), which will handle everything
// else.
func startEnvironment(cmd *cobra.Command, args []string) {
	_, err := CreateEnvironment(config.RpcPort, config.ApiPort, config.NoBootstrap)
	if err != nil {
		fmt.Println(err)
	}
}

// homeFolder displays a user's home directory to them, which is nice for
// windows users since they might not know which directory is their home
// directory.
func homeFolder(cmd *cobra.Command, args []string) {
	home, err := homedir.Dir()
	if err != nil {
		fmt.Println("Error:", err)
	} else {
		fmt.Println(home)
	}
}

// Prints version information about Sia Daemon.
func version(cmd *cobra.Command, args []string) {
	fmt.Println("Sia Daemon v0.1.0")
}

func main() {
	root := &cobra.Command{
		Use:   os.Args[0],
		Short: "Sia Daemon v0.1.0",
		Long:  "Sia Daemon v0.1.0",
		Run:   startEnvironment,
	}

	root.AddCommand(&cobra.Command{
		Use:   "home-folder",
		Short: "Print home folder",
		Long:  "Print the filepath of the home folder as seen by the binary.",
		Run:   homeFolder,
	})

	root.AddCommand(&cobra.Command{
		Use:   "version",
		Short: "Print version information",
		Long:  "Print version information about the Sia Daemon",
		Run:   version,
	})

	// Add flag defaults, which have the lowest priority. Every value will be
	// set.
	defaultConfigFile := configFilenameDefault()
	root.Flags().Uint16VarP(&config.ApiPort, "api-port", "a", 9980, "Which port is used to communicate with the user.")
	root.Flags().Uint16VarP(&config.RpcPort, "rpc-port", "r", 9988, "Which port is used when talking to other nodes on the network.")
	root.Flags().BoolVarP(&config.NoBootstrap, "no-bootstrap", "n", false, "Disable bootstrapping on this run.")
	root.Flags().StringVarP(&config.ConfigFilename, "config-file", "c", defaultConfigFile, "Tell siad where to load the config file.")

	// Load the config file, which has the middle priorty. Only values defined
	// in the config file will be set.
	if config.ConfigFilename != "" {
		err := gcfg.ReadFileInto(&config, config.ConfigFilename)
		if err != nil {
			fmt.Println(err)
			return
		}
	}

	// Start the party.
	root.Execute()
}
<commit_msg>proper priorites when loading default, config, and user set values<commit_after>package main

import (
	"fmt"
	"os"

	"code.google.com/p/gcfg"
	"github.com/mitchellh/go-homedir"
	"github.com/spf13/cobra"
)

var config Config

type Config struct {
	Siad struct {
		ApiPort        uint16
		RpcPort        uint16
		NoBootstrap    bool
		ConfigFilename string
	}
}

// confilgFilenameDefault checks multiple directories for a config file and
// loads the first one it finds. "" is returned if no config file is found.
func configFilenameDefault() string {
	// Try home folder.
	home, err := homedir.Dir()
	if err == nil {
		// Check home/.config/config
		filename := home + "/.config/sia/config"
		if _, err := os.Stat(filename); err == nil {
			return filename
		}

		// Check home/.sia/config
		filename = home + "/.sia/config"
		if _, err := os.Stat(filename); err == nil {
			return filename
		}

		// Check home/.sia.conf
		filename = home + "/.sia.conf"
		if _, err := os.Stat(filename); err == nil {
			return filename
		}
	}

	// Try /etc/sia.conf
	filename := "etc/sia.conf"
	if _, err := os.Stat(filename); err == nil {
		return filename
	}

	return ""
}

// startEnvironment calls createEnvironment(), which will handle everything
// else.
func startEnvironment(cmd *cobra.Command, args []string) {
	_, err := CreateEnvironment(config.Siad.RpcPort, config.Siad.ApiPort, config.Siad.NoBootstrap)
	if err != nil {
		fmt.Println(err)
	}
}

// homeFolder displays a user's home directory to them, which is nice for
// windows users since they might not know which directory is their home
// directory.
func homeFolder(cmd *cobra.Command, args []string) {
	home, err := homedir.Dir()
	if err != nil {
		fmt.Println("Error:", err)
	} else {
		fmt.Println(home)
	}
}

// Prints version information about Sia Daemon.
func version(cmd *cobra.Command, args []string) {
	fmt.Println("Sia Daemon v0.1.0")
}

func main() {
	root := &cobra.Command{
		Use:   os.Args[0],
		Short: "Sia Daemon v0.1.0",
		Long:  "Sia Daemon v0.1.0",
		Run:   startEnvironment,
	}

	root.AddCommand(&cobra.Command{
		Use:   "home-folder",
		Short: "Print home folder",
		Long:  "Print the filepath of the home folder as seen by the binary.",
		Run:   homeFolder,
	})

	root.AddCommand(&cobra.Command{
		Use:   "version",
		Short: "Print version information",
		Long:  "Print version information about the Sia Daemon",
		Run:   version,
	})

	// Add flag defaults, which have the lowest priority. Every value will be
	// set.
	defaultConfigFile := configFilenameDefault()
	root.PersistentFlags().Uint16VarP(&config.Siad.ApiPort, "api-port", "a", 9980, "Which port is used to communicate with the user.")
	root.PersistentFlags().Uint16VarP(&config.Siad.RpcPort, "rpc-port", "r", 9988, "Which port is used when talking to other nodes on the network.")
	root.PersistentFlags().BoolVarP(&config.Siad.NoBootstrap, "no-bootstrap", "n", false, "Disable bootstrapping on this run.")
	root.PersistentFlags().StringVarP(&config.Siad.ConfigFilename, "config-file", "c", defaultConfigFile, "Tell siad where to load the config file.")

	// Load the config file, which has the middle priorty. Only values defined
	// in the config file will be set.
	if config.Siad.ConfigFilename != "" {
		err := gcfg.ReadFileInto(&config, config.Siad.ConfigFilename)
		if err != nil {
			fmt.Println(err)
			return
		}
	}

	// Execute wil over-write any flags set by the config file, but only if the
	// user specified them manually.
	root.Execute()
}
<|endoftext|>


<commit_before>package boss

import (
	"strconv"
	"strings"

	"github.com/pajlada/pajbot2/common"
)

type parse struct {
	m *common.Msg
}

/*
Parse parses an IRC message into a more readable bot.Msg
*/
func Parse(line string) common.Msg {
	p := &parse{}
	p.m = &common.Msg{
		User: common.User{},
	}
	parseTags := true

	var splitline []string
	if strings.HasPrefix(line, ":") {
		parseTags = false
		splitline = strings.SplitN(line, ":", 2)
	} else {
		splitline = strings.SplitN(line, " :", 2)
	}
	tagsRaw := splitline[0]
	msg := splitline[1]
	tags := make(map[string]string)

	p.GetMessage(msg)
	if p.m.User.Name == "twitchnotify" {
		if !strings.Contains(p.m.Message, " to ") && !strings.Contains(p.m.Message, " while ") {
			p.m.Type = "sub"
			p.Sub()
		} else {
			p.m.Type = "hostSub" // useless xD
		}

	} else {
		if strings.Contains(msg, "PRIVMSG") {
			p.m.Type = "privmsg"
		} else {
			p.m.Type = "whisper"
		}

		// Should user properties stay at their zero value when there are no tags? Do we even care about this scenario?
		if parseTags {
			for _, tagValue := range strings.Split(tagsRaw, ";") {
				spl := strings.Split(tagValue, "=")
				k := spl[0]
				v := spl[1]
				tags[k] = v
			}
			p.GetTwitchEmotes(tags["emotes"])
			p.GetTags(tags)
		}
	}

	return *p.m
}

func (p *parse) GetTwitchEmotes(emotetag string) {
	// TODO: Parse more emote information (bttv (and ffz?), name, size, isGif)
	// will we done by a module in the bot itself
	p.m.Emotes = make([]common.Emote, 0)
	if emotetag == "" {
		return
	}
	emoteSlice := strings.Split(emotetag, "/")
	for i := range emoteSlice {
		id := strings.Split(emoteSlice[i], ":")[0]
		e := &common.Emote{}
		e.Type = "twitch"
		e.Name = ""
		e.ID = id
		// 28 px should be fine for twitch emotes
		e.SizeX = 28
		e.SizeY = 28
		e.Count = strings.Count(emoteSlice[i], "-")
		p.m.Emotes = append(p.m.Emotes, *e)
	}
}

func (p *parse) GetTags(tags map[string]string) {
	// TODO: Parse id and color
	// color and id is pretty useless imo
	if tags["display-name"] == "" {
		p.m.User.DisplayName = p.m.User.Name
	} else {
		p.m.User.DisplayName = tags["display-name"]
	}
	p.m.User.Type = tags["user-type"]
	if tags["turbo"] == "1" {
		p.m.User.Turbo = true
	}
	if tags["mod"] == "1" {
		p.m.User.Mod = true
	}
	if tags["subscriber"] == "1" {
		p.m.User.Mod = true
	}

}

func (p *parse) GetMessage(msg string) {
	if strings.HasPrefix(msg, ":") {
		msg = strings.Replace(msg, ":", "", 1)
	}
	p.m.Message = strings.SplitN(msg, " :", 2)[1]
	p.m.User.Name = strings.SplitN(msg, "!", 2)[0]
	c := strings.SplitN(msg, "#", 3)[1]
	p.m.Channel = strings.SplitN(c, " ", 2)[0]
}

func (p *parse) Sub() {
	m := p.m.Message
	if strings.Contains(m, "just ") {
		p.m.Length = 1
	} else {
		temp := strings.Split(m, " for ")[1]
		l := strings.Split(temp, " ")[0]
		length, err := strconv.Atoi(l)
		if err == nil {
			p.m.Length = length
		} else {
			panic(err)
		}
	}
	p.m.User.DisplayName = strings.Split(m, " ")[0]
	p.m.User.Name = strings.ToLower(p.m.User.DisplayName)
}
<commit_msg>now parsing /me correctly<commit_after>package boss

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/pajlada/pajbot2/common"
)

type parse struct {
	m *common.Msg
}

/*
Parse parses an IRC message into a more readable bot.Msg
*/
func Parse(line string) common.Msg {
	p := &parse{}
	p.m = &common.Msg{
		User: common.User{},
	}
	parseTags := true

	var splitline []string
	if strings.HasPrefix(line, ":") {
		parseTags = false
		splitline = strings.SplitN(line, ":", 2)
	} else {
		splitline = strings.SplitN(line, " :", 2)
	}
	tagsRaw := splitline[0]
	msg := splitline[1]
	tags := make(map[string]string)

	p.GetMessage(msg)
	if p.m.User.Name == "twitchnotify" {
		if !strings.Contains(p.m.Message, " to ") && !strings.Contains(p.m.Message, " while ") {
			p.m.Type = "sub"
			p.Sub()
		} else {
			p.m.Type = "hostSub" // useless xD
		}

	} else {
		if strings.Contains(msg, "PRIVMSG") {
			p.m.Type = "privmsg"
		} else {
			p.m.Type = "whisper"
		}

		// Should user properties stay at their zero value when there are no tags? Do we even care about this scenario?
		if parseTags {
			for _, tagValue := range strings.Split(tagsRaw, ";") {
				spl := strings.Split(tagValue, "=")
				k := spl[0]
				v := spl[1]
				tags[k] = v
			}
			p.GetTwitchEmotes(tags["emotes"])
			p.GetTags(tags)
		}
	}

	return *p.m
}

func (p *parse) GetTwitchEmotes(emotetag string) {
	// TODO: Parse more emote information (bttv (and ffz?), name, size, isGif)
	// will we done by a module in the bot itself
	p.m.Emotes = make([]common.Emote, 0)
	if emotetag == "" {
		return
	}
	emoteSlice := strings.Split(emotetag, "/")
	for i := range emoteSlice {
		id := strings.Split(emoteSlice[i], ":")[0]
		e := &common.Emote{}
		e.Type = "twitch"
		e.Name = ""
		e.ID = id
		// 28 px should be fine for twitch emotes
		e.SizeX = 28
		e.SizeY = 28
		e.Count = strings.Count(emoteSlice[i], "-")
		p.m.Emotes = append(p.m.Emotes, *e)
	}
}

func (p *parse) GetTags(tags map[string]string) {
	// TODO: Parse id and color
	// color and id is pretty useless imo
	if tags["display-name"] == "" {
		p.m.User.DisplayName = p.m.User.Name
	} else {
		p.m.User.DisplayName = tags["display-name"]
	}
	p.m.User.Type = tags["user-type"]
	if tags["turbo"] == "1" {
		p.m.User.Turbo = true
	}
	if tags["mod"] == "1" {
		p.m.User.Mod = true
	}
	if tags["subscriber"] == "1" {
		p.m.User.Mod = true
	}

}

func (p *parse) GetMessage(msg string) {
	if strings.HasPrefix(msg, ":") {
		msg = strings.Replace(msg, ":", "", 1)
	}
	p.m.Message = strings.SplitN(msg, " :", 2)[1]
	p.m.User.Name = strings.SplitN(msg, "!", 2)[0]
	c := strings.SplitN(msg, "#", 3)[1]
	p.m.Channel = strings.SplitN(c, " ", 2)[0]
	p.getAction()
}

// regex in 2016 LUL
func (p *parse) getAction() {
	fmt.Println(":" + p.m.Message + ":")
	if strings.HasPrefix(p.m.Message, "\u0001ACTION ") && strings.HasSuffix(p.m.Message, "\u0001") {
		p.m.Me = true
		m := p.m.Message
		m = strings.Replace(m, "\u0001ACTION ", "", 1)
		m = strings.Replace(m, "\u0001", "", 1)
		p.m.Message = m
	}
}

func (p *parse) Sub() {
	m := p.m.Message
	if strings.Contains(m, "just ") {
		p.m.Length = 1
	} else {
		temp := strings.Split(m, " for ")[1]
		l := strings.Split(temp, " ")[0]
		length, err := strconv.Atoi(l)
		if err == nil {
			p.m.Length = length
		} else {
			panic(err)
		}
	}
	p.m.User.DisplayName = strings.Split(m, " ")[0]
	p.m.User.Name = strings.ToLower(p.m.User.DisplayName)
}
<|endoftext|>


<commit_before>// Copyright 2014 Richard Lehane. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package siegfried describes the layout of the Siegfried signature file.
// This signature file contains the siegfried object that performs identification
package siegfried

import (
	"bytes"
	"encoding/gob"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strings"
	"time"

	"github.com/richardlehane/siegfried/config"
	"github.com/richardlehane/siegfried/pkg/core"
	"github.com/richardlehane/siegfried/pkg/core/bytematcher"
	"github.com/richardlehane/siegfried/pkg/core/containermatcher"
	"github.com/richardlehane/siegfried/pkg/core/extensionmatcher"
	"github.com/richardlehane/siegfried/pkg/core/siegreader"
	"github.com/richardlehane/siegfried/pkg/pronom"
)

type Siegfried struct {
	C  time.Time    // signature create time
	em core.Matcher // extensionmatcher
	cm core.Matcher // containermatcher
	bm core.Matcher // bytematcher
	// mutatable fields follow
	ids    []core.Identifier // at present only one identifier (the PRONOM identifier) is used, but can add other identifiers e.g. for FILE sigs
	buffer *siegreader.Buffer
}

func New() *Siegfried {
	s := &Siegfried{}
	s.C = time.Now()
	s.em = extensionmatcher.New()
	s.cm = containermatcher.New()
	s.bm = bytematcher.New()
	s.buffer = siegreader.New()
	return s
}

func (s *Siegfried) String() string {
	str := "IDENTIFIERS\n"
	for _, i := range s.ids {
		str += i.String()
	}
	str += "\nEXTENSION MATCHER\n"
	str += s.em.String()
	str += "\nCONTAINER MATCHER\n"
	str += s.cm.String()
	str += "\nBYTE MATCHER\n"
	str += s.bm.String()
	return str
}

func (s *Siegfried) InspectTTI(tti int) string {
	bm := s.bm.(*bytematcher.Matcher)
	idxs := bm.InspectTTI(tti)
	if idxs == nil {
		return "No test tree at this index"
	}
	res := make([]string, len(idxs))
	for i, v := range idxs {
		for _, id := range s.ids {
			ok, str := id.Recognise(core.ByteMatcher, v)
			if ok {
				res[i] = str
				break
			}
		}
	}
	return "Test tree indexes match:\n" + strings.Join(res, "\n")
}

func (s *Siegfried) Add(i core.Identifier) error {
	switch i := i.(type) {
	default:
		return fmt.Errorf("Siegfried: unknown identifier type %T", i)
	case *pronom.Identifier:
		if err := i.Add(s.em); err != nil {
			return err
		}
		if err := i.Add(s.cm); err != nil {
			return err
		}
		if err := i.Add(s.bm); err != nil {
			return err
		}
		s.ids = append(s.ids, i)
	}
	return nil
}

func (s *Siegfried) Yaml() string {
	version := config.Version()
	str := fmt.Sprintf(
		"---\nsiegfried   : %d.%d.%d\nscan date   : %v\nsignature   : %s\ncreated     : %v\nidentifiers : \n",
		version[0], version[1], version[2],
		time.Now().Format(time.RFC3339),
		config.SignatureBase(),
		s.C.Format(time.RFC3339))
	for _, id := range s.ids {
		str += id.Yaml()
	}
	return str
}

func (s *Siegfried) Update(t string) bool {
	tm, err := time.Parse(time.RFC3339, t)
	if err != nil {
		return false
	}
	return tm.After(s.C)
}

type Header struct {
	SSize int                // sigversion
	BSize int                // bytematcher
	CSize int                // container
	ESize int                // extension matcher
	Ids   []IdentifierHeader // size and types of identifiers
}

type IdentifierHeader struct {
	Typ identifierType
	Sz  int
}

type identifierType int

// Register additional identifier types here
const (
	Pronom identifierType = iota
)

func identifierSz(ids []IdentifierHeader) int {
	var sz int
	for _, v := range ids {
		sz += v.Sz
	}
	return sz
}

func (s *Siegfried) Save(path string) error {
	buf := new(bytes.Buffer)
	enc := gob.NewEncoder(buf)
	err := enc.Encode(s)
	if err != nil {
		return err
	}
	ssz := buf.Len()
	bsz, err := s.bm.Save(buf)
	if err != nil {
		return err
	}
	csz, err := s.cm.Save(buf)
	if err != nil {
		return err
	}
	esz, err := s.em.Save(buf)
	if err != nil {
		return err
	}
	ids := make([]IdentifierHeader, len(s.ids))
	for i, v := range s.ids {
		sz, err := v.Save(buf)
		if err != nil {
			return err
		}
		// add any additional identifiers to this type switch
		switch t := v.(type) {
		default:
			return fmt.Errorf("Siegfried: unexpected type for an identifier %T", t)
		case *pronom.Identifier:
			ids[i].Typ = Pronom
		}
		ids[i].Sz = sz
	}
	hbuf := new(bytes.Buffer)
	henc := gob.NewEncoder(hbuf)
	err = henc.Encode(Header{ssz, bsz, csz, esz, ids})
	f, err := os.Create(path)
	defer f.Close()
	if err != nil {
		return err
	}
	_, err = f.Write(hbuf.Bytes())
	if err != nil {
		return err
	}
	_, err = f.Write(buf.Bytes())
	if err != nil {
		return err
	}
	return nil
}

func Load(path string) (*Siegfried, error) {
	c, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf("Siegfried: error opening signature file; got %s\nTry running `sf -update`", err)
	}
	buf := bytes.NewBuffer(c)
	dec := gob.NewDecoder(buf)
	var h Header
	err = dec.Decode(&h)
	if err != nil {
		return nil, fmt.Errorf("Siegfried: error reading signature file; got %s\nTry running `sf -update`", err)
	}
	iSize := identifierSz(h.Ids)
	sstart := len(c) - h.SSize - h.BSize - h.CSize - h.ESize - iSize
	bstart := len(c) - h.ESize - h.CSize - h.BSize - iSize
	cstart := len(c) - h.ESize - h.CSize - iSize
	estart := len(c) - h.ESize - iSize
	istart := len(c) - iSize
	sbuf := bytes.NewBuffer(c[sstart : sstart+h.SSize])
	bbuf := bytes.NewBuffer(c[bstart : bstart+h.BSize])
	cbuf := bytes.NewBuffer(c[cstart : cstart+h.CSize])
	ebuf := bytes.NewBuffer(c[estart : estart+h.ESize])
	sdec := gob.NewDecoder(sbuf)
	var s Siegfried
	err = sdec.Decode(&s)
	if err != nil {
		return nil, err
	}
	bm, err := bytematcher.Load(bbuf)
	if err != nil {
		return nil, fmt.Errorf("Siegfried: error loading bytematcher; got %s", err)
	}
	cm, err := containermatcher.Load(cbuf)
	if err != nil {
		return nil, fmt.Errorf("Siegfried: error loading containermatcher; got %s", err)
	}
	em, err := extensionmatcher.Load(ebuf)
	if err != nil {
		return nil, fmt.Errorf("Siegfried: error loading extensionmatcher; got %s", err)
	}
	s.bm = bm
	s.cm = cm
	s.em = em
	s.ids = make([]core.Identifier, len(h.Ids))
	for i, v := range h.Ids {
		ibuf := bytes.NewBuffer(c[istart : istart+v.Sz])
		var id core.Identifier
		var err error
		switch v.Typ {
		default:
			return nil, fmt.Errorf("Siegfried: loading, unknown identifier type %d", v.Typ)
		case Pronom:
			id, err = pronom.Load(ibuf)
			if err != nil {
				return nil, fmt.Errorf("Siegfried: loading PRONOM identifier; got %s", err)
			}
		}
		s.ids[i] = id
		istart += v.Sz
	}
	s.buffer = siegreader.New()
	return &s, nil
}

func (s *Siegfried) Identify(n string, r io.Reader) (chan core.Identification, error) {
	err := s.buffer.SetSource(r)
	if err != nil && err != io.EOF {
		return nil, fmt.Errorf("Siegfried: error reading input, got %v", err)
	}
	res := make(chan core.Identification)
	recs := make([]core.Recorder, len(s.ids))
	for i, v := range s.ids {
		recs[i] = v.Recorder()
	}
	// Extension Matcher
	if len(n) > 0 {
		ems := s.em.Identify(n, nil)
		for v := range ems {
			for _, rec := range recs {
				if rec.Record(core.ExtensionMatcher, v) {
					break
				}
			}
		}
	}

	// Container Matcher
	if s.cm != nil {
		cms := s.cm.Identify(n, s.buffer)
		for v := range cms {
			for _, rec := range recs {
				if rec.Record(core.ContainerMatcher, v) {
					break
				}
			}
		}
	}
	satisfied := true
	for _, rec := range recs {
		if !rec.Satisfied() {
			satisfied = false
		}
	}
	// Byte Matcher
	if !satisfied {
		ids := s.bm.Identify("", s.buffer)
		for v := range ids {
			for _, rec := range recs {
				if rec.Record(core.ByteMatcher, v) {
					break
				}
			}
		}
	}
	go func() {
		for _, rec := range recs {
			rec.Report(res)
		}
		close(res)
	}()
	return res, nil
}
<commit_msg>scandate<commit_after>// Copyright 2014 Richard Lehane. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package siegfried describes the layout of the Siegfried signature file.
// This signature file contains the siegfried object that performs identification
package siegfried

import (
	"bytes"
	"encoding/gob"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strings"
	"time"

	"github.com/richardlehane/siegfried/config"
	"github.com/richardlehane/siegfried/pkg/core"
	"github.com/richardlehane/siegfried/pkg/core/bytematcher"
	"github.com/richardlehane/siegfried/pkg/core/containermatcher"
	"github.com/richardlehane/siegfried/pkg/core/extensionmatcher"
	"github.com/richardlehane/siegfried/pkg/core/siegreader"
	"github.com/richardlehane/siegfried/pkg/pronom"
)

type Siegfried struct {
	C  time.Time    // signature create time
	em core.Matcher // extensionmatcher
	cm core.Matcher // containermatcher
	bm core.Matcher // bytematcher
	// mutatable fields follow
	ids    []core.Identifier // at present only one identifier (the PRONOM identifier) is used, but can add other identifiers e.g. for FILE sigs
	buffer *siegreader.Buffer
}

func New() *Siegfried {
	s := &Siegfried{}
	s.C = time.Now()
	s.em = extensionmatcher.New()
	s.cm = containermatcher.New()
	s.bm = bytematcher.New()
	s.buffer = siegreader.New()
	return s
}

func (s *Siegfried) String() string {
	str := "IDENTIFIERS\n"
	for _, i := range s.ids {
		str += i.String()
	}
	str += "\nEXTENSION MATCHER\n"
	str += s.em.String()
	str += "\nCONTAINER MATCHER\n"
	str += s.cm.String()
	str += "\nBYTE MATCHER\n"
	str += s.bm.String()
	return str
}

func (s *Siegfried) InspectTTI(tti int) string {
	bm := s.bm.(*bytematcher.Matcher)
	idxs := bm.InspectTTI(tti)
	if idxs == nil {
		return "No test tree at this index"
	}
	res := make([]string, len(idxs))
	for i, v := range idxs {
		for _, id := range s.ids {
			ok, str := id.Recognise(core.ByteMatcher, v)
			if ok {
				res[i] = str
				break
			}
		}
	}
	return "Test tree indexes match:\n" + strings.Join(res, "\n")
}

func (s *Siegfried) Add(i core.Identifier) error {
	switch i := i.(type) {
	default:
		return fmt.Errorf("Siegfried: unknown identifier type %T", i)
	case *pronom.Identifier:
		if err := i.Add(s.em); err != nil {
			return err
		}
		if err := i.Add(s.cm); err != nil {
			return err
		}
		if err := i.Add(s.bm); err != nil {
			return err
		}
		s.ids = append(s.ids, i)
	}
	return nil
}

func (s *Siegfried) Yaml() string {
	version := config.Version()
	str := fmt.Sprintf(
		"---\nsiegfried   : %d.%d.%d\nscandate    : %v\nsignature   : %s\ncreated     : %v\nidentifiers : \n",
		version[0], version[1], version[2],
		time.Now().Format(time.RFC3339),
		config.SignatureBase(),
		s.C.Format(time.RFC3339))
	for _, id := range s.ids {
		str += id.Yaml()
	}
	return str
}

func (s *Siegfried) Update(t string) bool {
	tm, err := time.Parse(time.RFC3339, t)
	if err != nil {
		return false
	}
	return tm.After(s.C)
}

type Header struct {
	SSize int                // sigversion
	BSize int                // bytematcher
	CSize int                // container
	ESize int                // extension matcher
	Ids   []IdentifierHeader // size and types of identifiers
}

type IdentifierHeader struct {
	Typ identifierType
	Sz  int
}

type identifierType int

// Register additional identifier types here
const (
	Pronom identifierType = iota
)

func identifierSz(ids []IdentifierHeader) int {
	var sz int
	for _, v := range ids {
		sz += v.Sz
	}
	return sz
}

func (s *Siegfried) Save(path string) error {
	buf := new(bytes.Buffer)
	enc := gob.NewEncoder(buf)
	err := enc.Encode(s)
	if err != nil {
		return err
	}
	ssz := buf.Len()
	bsz, err := s.bm.Save(buf)
	if err != nil {
		return err
	}
	csz, err := s.cm.Save(buf)
	if err != nil {
		return err
	}
	esz, err := s.em.Save(buf)
	if err != nil {
		return err
	}
	ids := make([]IdentifierHeader, len(s.ids))
	for i, v := range s.ids {
		sz, err := v.Save(buf)
		if err != nil {
			return err
		}
		// add any additional identifiers to this type switch
		switch t := v.(type) {
		default:
			return fmt.Errorf("Siegfried: unexpected type for an identifier %T", t)
		case *pronom.Identifier:
			ids[i].Typ = Pronom
		}
		ids[i].Sz = sz
	}
	hbuf := new(bytes.Buffer)
	henc := gob.NewEncoder(hbuf)
	err = henc.Encode(Header{ssz, bsz, csz, esz, ids})
	f, err := os.Create(path)
	defer f.Close()
	if err != nil {
		return err
	}
	_, err = f.Write(hbuf.Bytes())
	if err != nil {
		return err
	}
	_, err = f.Write(buf.Bytes())
	if err != nil {
		return err
	}
	return nil
}

func Load(path string) (*Siegfried, error) {
	c, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf("Siegfried: error opening signature file; got %s\nTry running `sf -update`", err)
	}
	buf := bytes.NewBuffer(c)
	dec := gob.NewDecoder(buf)
	var h Header
	err = dec.Decode(&h)
	if err != nil {
		return nil, fmt.Errorf("Siegfried: error reading signature file; got %s\nTry running `sf -update`", err)
	}
	iSize := identifierSz(h.Ids)
	sstart := len(c) - h.SSize - h.BSize - h.CSize - h.ESize - iSize
	bstart := len(c) - h.ESize - h.CSize - h.BSize - iSize
	cstart := len(c) - h.ESize - h.CSize - iSize
	estart := len(c) - h.ESize - iSize
	istart := len(c) - iSize
	sbuf := bytes.NewBuffer(c[sstart : sstart+h.SSize])
	bbuf := bytes.NewBuffer(c[bstart : bstart+h.BSize])
	cbuf := bytes.NewBuffer(c[cstart : cstart+h.CSize])
	ebuf := bytes.NewBuffer(c[estart : estart+h.ESize])
	sdec := gob.NewDecoder(sbuf)
	var s Siegfried
	err = sdec.Decode(&s)
	if err != nil {
		return nil, err
	}
	bm, err := bytematcher.Load(bbuf)
	if err != nil {
		return nil, fmt.Errorf("Siegfried: error loading bytematcher; got %s", err)
	}
	cm, err := containermatcher.Load(cbuf)
	if err != nil {
		return nil, fmt.Errorf("Siegfried: error loading containermatcher; got %s", err)
	}
	em, err := extensionmatcher.Load(ebuf)
	if err != nil {
		return nil, fmt.Errorf("Siegfried: error loading extensionmatcher; got %s", err)
	}
	s.bm = bm
	s.cm = cm
	s.em = em
	s.ids = make([]core.Identifier, len(h.Ids))
	for i, v := range h.Ids {
		ibuf := bytes.NewBuffer(c[istart : istart+v.Sz])
		var id core.Identifier
		var err error
		switch v.Typ {
		default:
			return nil, fmt.Errorf("Siegfried: loading, unknown identifier type %d", v.Typ)
		case Pronom:
			id, err = pronom.Load(ibuf)
			if err != nil {
				return nil, fmt.Errorf("Siegfried: loading PRONOM identifier; got %s", err)
			}
		}
		s.ids[i] = id
		istart += v.Sz
	}
	s.buffer = siegreader.New()
	return &s, nil
}

func (s *Siegfried) Identify(n string, r io.Reader) (chan core.Identification, error) {
	err := s.buffer.SetSource(r)
	if err != nil && err != io.EOF {
		return nil, fmt.Errorf("Siegfried: error reading input, got %v", err)
	}
	res := make(chan core.Identification)
	recs := make([]core.Recorder, len(s.ids))
	for i, v := range s.ids {
		recs[i] = v.Recorder()
	}
	// Extension Matcher
	if len(n) > 0 {
		ems := s.em.Identify(n, nil)
		for v := range ems {
			for _, rec := range recs {
				if rec.Record(core.ExtensionMatcher, v) {
					break
				}
			}
		}
	}

	// Container Matcher
	if s.cm != nil {
		cms := s.cm.Identify(n, s.buffer)
		for v := range cms {
			for _, rec := range recs {
				if rec.Record(core.ContainerMatcher, v) {
					break
				}
			}
		}
	}
	satisfied := true
	for _, rec := range recs {
		if !rec.Satisfied() {
			satisfied = false
		}
	}
	// Byte Matcher
	if !satisfied {
		ids := s.bm.Identify("", s.buffer)
		for v := range ids {
			for _, rec := range recs {
				if rec.Record(core.ByteMatcher, v) {
					break
				}
			}
		}
	}
	go func() {
		for _, rec := range recs {
			rec.Report(res)
		}
		close(res)
	}()
	return res, nil
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/sha1"
	"fmt"
)

// check that message came from someone with
// who signed it with the same clusterid. The
// signature itself has to be blanked first.
func GetJobSignature(j *Job, cfg *Config) string {
	jobcopy := *j
	SignJob(&jobcopy, cfg)
	return jobcopy.Signature
}

func JobSignatureOkay(j *Job, cfg *Config) bool {
	sig := j.Signature

	computed := GetJobSignature(j, cfg)
	if computed == sig {
		return true
	}
	return false
}

// sign by filling in the j.Signature field
//
// have to zero out both Signature and DesinationSocket before signing.
//
// SignJob() cannot modify Job, since it is used on the reciever for verification too.
//
func SignJob(j *Job, cfg *Config) {
	j.Signature = ""
	saveSock := j.destinationSock
	j.destinationSock = nil

	str := fmt.Sprintf("%+v\nclusterid:%s", *j, cfg.ClusterId)
	//fmt.Printf("\n SignJob() signing this: '%s'\n", str)
	j.Signature = Sha1sum(str)
	j.destinationSock = saveSock
}

func Sha1sum(s string) string {
	sum := sha1.Sum([]byte(s))
	slsum := sum[:]
	return fmt.Sprintf("%x", slsum)
}
<commit_msg>sha1 hmac inside<commit_after>package main

import (
	"crypto/hmac"
	"crypto/sha1"
	"fmt"
)

// check that message came from someone with
// who signed it with the same clusterid. The
// signature itself has to be blanked first.
func GetJobSignature(j *Job, cfg *Config) string {
	jobcopy := *j
	SignJob(&jobcopy, cfg)
	return jobcopy.Signature
}

func JobSignatureOkay(j *Job, cfg *Config) bool {
	sig := j.Signature

	computed := GetJobSignature(j, cfg)
	if computed == sig {
		return true
	}
	return false
}

// sign by filling in the j.Signature field
//
// have to zero out both Signature and DesinationSocket before signing.
//
// SignJob() cannot modify Job, since it is used on the reciever for verification too.
//
func SignJob(j *Job, cfg *Config) {
	j.Signature = ""
	saveSock := j.destinationSock
	j.destinationSock = nil

	str := fmt.Sprintf("%+v\nclusterid:%s", *j, cfg.ClusterId)
	//fmt.Printf("\n SignJob() signing this: '%s'\n", str)
	secretForHMAC := cfg.Cypher.Encrypt([]byte(str))
	j.Signature = string(Sha1HMAC([]byte(str), secretForHMAC))
	j.destinationSock = saveSock
}

func Sha1sum(s string) string {
	sum := sha1.Sum([]byte(s))
	slsum := sum[:]
	return fmt.Sprintf("%x", slsum)
}

func Sha1HMAC(message, key []byte) []byte {
	mac := hmac.New(sha1.New, key)
	mac.Write(message)
	return mac.Sum(nil)
}
<|endoftext|>


<commit_before>package scraper

import (
	"crypto/tls"
	"crypto/x509"
	"errors"
	"net"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"time"
	"unicode/utf8"
)

// CustomClient is the state for our custom http wrapper, which houses
// the needed data to be able to rewrite the outgoing request during
// redirects.
type CustomClient struct {
	URL       string
	IP        string
	Host      string
	ResultURL *url.URL // represents the url for the resulting request, without modifications
	OriginURL *url.URL // represents the url from the original request, without modifications
	ipmap     map[string]string
}

// CustomResponse is the wrapped response from http.Client.Do() which also
// includes a timer of how long the request took, and a few other minor
// extras.
type CustomResponse struct {
	*http.Response
	Time *TimerResult
	URL  string
}

var reIP = regexp.MustCompile(`^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$`)

func (c *CustomClient) redirectHandler(req *http.Request, via []*http.Request) error {
	c.requestWrap(req)

	// rewrite Referer (Referrer) if it exists, to have the proper hostname
	uri := via[len(via)-1].URL
	uri.Host = via[len(via)-1].Host
	req.Header.Set("Referer", uri.String())

	if len(via) > 3 {
		// assume too many redirects
		return errors.New("too many redirects (3)")
	}

	if reIP.MatchString(req.Host) && req.Host != c.IP {
		return errors.New("Redirected to IP that doesn't match proxy")
	}

	if _, ok := c.ipmap[req.Host]; !ok {
		if c.OriginURL.Path == "" {
			return errors.New("Redirection does not match origin host")
		}
	}

	return nil
}

func (c *CustomClient) requestWrap(req *http.Request) *http.Request {
	// spoof useragent, as there are going to be sites/servers that are
	// setup to deny by a specific useragent string (or lack there of)
	req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36")

	// if an IP address is provided, rewrite the Host headers
	// of note: if we plan to support custom ports, these should be rewritten
	// within the header. E.g. "hostname.com:8080" -- though, common ports like
	// 80 and 443 are left out.

	// assign the origin host to the host header value, ONLY if it matches the domains
	// hostname
	if _, ok := c.ipmap[req.URL.Host]; ok {
		req.Host = req.URL.Host

		// and overwrite the host used to make the connection
		req.URL.Host = c.ipmap[req.URL.Host]
	}

	// update our cached resulting uri
	c.ResultURL = req.URL
	if len(req.Host) > 0 {
		c.ResultURL.Host = req.Host
	}

	return req
}

type HostnameError struct {
	Certificate *x509.Certificate
	Host        string
}

func (h HostnameError) Error() string {
	c := h.Certificate

	var valid string
	if ip := net.ParseIP(h.Host); ip != nil {
		// Trying to validate an IP
		if len(c.IPAddresses) == 0 {
			return "x509: cannot validate certificate for " + h.Host + " because it doesn't contain any IP SANs"
		}
		for _, san := range c.IPAddresses {
			if len(valid) > 0 {
				valid += ", "
			}
			valid += san.String()
		}
	} else {
		if len(c.DNSNames) > 0 {
			valid = strings.Join(c.DNSNames, ", ")
		} else {
			valid = c.Subject.CommonName
		}
	}
	return "x509: certificate is valid for " + valid + ", not " + h.Host
}

// toLowerCaseASCII returns a lower-case version of in. See RFC 6125 6.4.1. We use
// an explicitly ASCII function to avoid any sharp corners resulting from
// performing Unicode operations on DNS labels.
func toLowerCaseASCII(in string) string {
	// If the string is already lower-case then there's nothing to do.
	isAlreadyLowerCase := true
	for _, c := range in {
		if c == utf8.RuneError {
			// If we get a UTF-8 error then there might be
			// upper-case ASCII bytes in the invalid sequence.
			isAlreadyLowerCase = false
			break
		}
		if 'A' <= c && c <= 'Z' {
			isAlreadyLowerCase = false
			break
		}
	}

	if isAlreadyLowerCase {
		return in
	}

	out := []byte(in)
	for i, c := range out {
		if 'A' <= c && c <= 'Z' {
			out[i] += 'a' - 'A'
		}
	}
	return string(out)
}

func matchHostnames(pattern, host string) bool {
	host = strings.TrimSuffix(host, ".")
	pattern = strings.TrimSuffix(pattern, ".")

	if len(pattern) == 0 || len(host) == 0 {
		return false
	}

	patternParts := strings.Split(pattern, ".")
	hostParts := strings.Split(host, ".")

	if len(patternParts) != len(hostParts) {
		return false
	}

	for i, patternPart := range patternParts {
		if i == 0 && patternPart == "*" {
			continue
		}
		if patternPart != hostParts[i] {
			return false
		}
	}

	return true
}

// verifyx509 returns nil if c is a valid certificate for the named host.
// Otherwise it returns an error describing the mismatch.
func verifyx509(c *x509.Certificate, h string) error {
	// IP addresses may be written in [ ].
	candidateIP := h
	if len(h) >= 3 && h[0] == '[' && h[len(h)-1] == ']' {
		candidateIP = h[1 : len(h)-1]
	}
	if ip := net.ParseIP(candidateIP); ip != nil {
		// We only match IP addresses against IP SANs.
		// https://tools.ietf.org/html/rfc6125#appendix-B.2
		for _, candidate := range c.IPAddresses {
			if ip.Equal(candidate) {
				return nil
			}
		}
		return HostnameError{c, candidateIP}
	}

	lowered := toLowerCaseASCII(h)

	if len(c.DNSNames) > 0 {
		for _, match := range c.DNSNames {
			if matchHostnames(toLowerCaseASCII(match), lowered) {
				return nil
			}
		}
		// If Subject Alt Name is given, we ignore the common name.
	} else if matchHostnames(toLowerCaseASCII(c.Subject.CommonName), lowered) {
		return nil
	}

	return HostnameError{c, h}
}

func VerifyHostname(c *tls.ConnectionState, host string) error {
	if c == nil {
		return nil
	}

	// if len(c.VerifiedChains) == 0 {
	// 	return errors.New("tls: handshake did not verify certificate chain")
	// }

	return verifyx509(c.PeerCertificates[0], host)
}

// getHandler wraps the standard net/http library, allowing us to spoof hostnames and IP addresses
func (c *CustomClient) getHandler() (*CustomResponse, error) {
	transport := &http.Transport{
		TLSClientConfig: &tls.Config{
			// unfortunately, ServerName will not persist over a redirect. so... we have to ignore
			// ssl invalidations and do them somewhat manually.
			InsecureSkipVerify: true,
			ServerName:         c.Host,
		},
	}
	client := &http.Client{
		CheckRedirect: c.redirectHandler,
		Timeout:       time.Duration(10) * time.Second,
		Transport:     transport,
	}

	req, err := http.NewRequest("GET", c.URL, nil)

	if err != nil {
		return nil, err
	}

	c.OriginURL = req.URL // set origin url for use in redirect wrapper
	c.requestWrap(req)

	// start tracking how long the request is going to take
	timer := NewTimer()

	// actually make the request here
	resp, err := client.Do(req)

	// stop tracking the request
	timer.End()

	var url string

	if err == nil {
		url = c.ResultURL.String()
		if err := VerifyHostname(resp.TLS, c.ResultURL.Host); err != nil {
			return nil, err
		}
	} else {
		url = req.URL.String()
	}

	wrappedResp := &CustomResponse{resp, timer.Result, url}

	return wrappedResp, err
}

// Get wraps GetHandler -- easy interface for making get requests
func Get(ipmap map[string]string, url string) (*CustomResponse, error) {
	host, err := getHost(url)
	if err != nil {
		return nil, err
	}

	ip, ok := ipmap[host]

	if ok {
		if len(ip) > 0 && !reIP.MatchString(ip) {
			return nil, errors.New("IP address provided is invalid")
		}
	}

	c := &CustomClient{URL: url, IP: ip, Host: host, ipmap: ipmap}

	return c.getHandler()
}
<commit_msg>only invoke proxy if an IP is supplied<commit_after>package scraper

import (
	"crypto/tls"
	"crypto/x509"
	"errors"
	"net"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"time"
	"unicode/utf8"
)

// CustomClient is the state for our custom http wrapper, which houses
// the needed data to be able to rewrite the outgoing request during
// redirects.
type CustomClient struct {
	URL       string
	IP        string
	Host      string
	ResultURL *url.URL // represents the url for the resulting request, without modifications
	OriginURL *url.URL // represents the url from the original request, without modifications
	ipmap     map[string]string
}

// CustomResponse is the wrapped response from http.Client.Do() which also
// includes a timer of how long the request took, and a few other minor
// extras.
type CustomResponse struct {
	*http.Response
	Time *TimerResult
	URL  string
}

var reIP = regexp.MustCompile(`^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$`)

func (c *CustomClient) redirectHandler(req *http.Request, via []*http.Request) error {
	c.requestWrap(req)

	// rewrite Referer (Referrer) if it exists, to have the proper hostname
	uri := via[len(via)-1].URL
	uri.Host = via[len(via)-1].Host
	req.Header.Set("Referer", uri.String())

	if len(via) > 3 {
		// assume too many redirects
		return errors.New("too many redirects (3)")
	}

	if reIP.MatchString(req.Host) && req.Host != c.IP {
		return errors.New("Redirected to IP that doesn't match proxy")
	}

	if _, ok := c.ipmap[req.Host]; !ok {
		if c.OriginURL.Path == "" {
			return errors.New("Redirection does not match origin host")
		}
	}

	return nil
}

func (c *CustomClient) requestWrap(req *http.Request) *http.Request {
	// spoof useragent, as there are going to be sites/servers that are
	// setup to deny by a specific useragent string (or lack there of)
	req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36")

	// if an IP address is provided, rewrite the Host headers
	// of note: if we plan to support custom ports, these should be rewritten
	// within the header. E.g. "hostname.com:8080" -- though, common ports like
	// 80 and 443 are left out.

	// assign the origin host to the host header value, ONLY if it matches the domains
	// hostname
	if _, ok := c.ipmap[req.URL.Host]; ok {
		req.Host = req.URL.Host

		// and overwrite the host used to make the connection
		if len(c.ipmap[req.URL.Host]) > 0 {
			req.URL.Host = c.ipmap[req.URL.Host]
		}
	}

	// update our cached resulting uri
	c.ResultURL = req.URL
	if len(req.Host) > 0 {
		c.ResultURL.Host = req.Host
	}

	return req
}

type HostnameError struct {
	Certificate *x509.Certificate
	Host        string
}

func (h HostnameError) Error() string {
	c := h.Certificate

	var valid string
	if ip := net.ParseIP(h.Host); ip != nil {
		// Trying to validate an IP
		if len(c.IPAddresses) == 0 {
			return "x509: cannot validate certificate for " + h.Host + " because it doesn't contain any IP SANs"
		}
		for _, san := range c.IPAddresses {
			if len(valid) > 0 {
				valid += ", "
			}
			valid += san.String()
		}
	} else {
		if len(c.DNSNames) > 0 {
			valid = strings.Join(c.DNSNames, ", ")
		} else {
			valid = c.Subject.CommonName
		}
	}
	return "x509: certificate is valid for " + valid + ", not " + h.Host
}

// toLowerCaseASCII returns a lower-case version of in. See RFC 6125 6.4.1. We use
// an explicitly ASCII function to avoid any sharp corners resulting from
// performing Unicode operations on DNS labels.
func toLowerCaseASCII(in string) string {
	// If the string is already lower-case then there's nothing to do.
	isAlreadyLowerCase := true
	for _, c := range in {
		if c == utf8.RuneError {
			// If we get a UTF-8 error then there might be
			// upper-case ASCII bytes in the invalid sequence.
			isAlreadyLowerCase = false
			break
		}
		if 'A' <= c && c <= 'Z' {
			isAlreadyLowerCase = false
			break
		}
	}

	if isAlreadyLowerCase {
		return in
	}

	out := []byte(in)
	for i, c := range out {
		if 'A' <= c && c <= 'Z' {
			out[i] += 'a' - 'A'
		}
	}
	return string(out)
}

func matchHostnames(pattern, host string) bool {
	host = strings.TrimSuffix(host, ".")
	pattern = strings.TrimSuffix(pattern, ".")

	if len(pattern) == 0 || len(host) == 0 {
		return false
	}

	patternParts := strings.Split(pattern, ".")
	hostParts := strings.Split(host, ".")

	if len(patternParts) != len(hostParts) {
		return false
	}

	for i, patternPart := range patternParts {
		if i == 0 && patternPart == "*" {
			continue
		}
		if patternPart != hostParts[i] {
			return false
		}
	}

	return true
}

// verifyx509 returns nil if c is a valid certificate for the named host.
// Otherwise it returns an error describing the mismatch.
func verifyx509(c *x509.Certificate, h string) error {
	// IP addresses may be written in [ ].
	candidateIP := h
	if len(h) >= 3 && h[0] == '[' && h[len(h)-1] == ']' {
		candidateIP = h[1 : len(h)-1]
	}
	if ip := net.ParseIP(candidateIP); ip != nil {
		// We only match IP addresses against IP SANs.
		// https://tools.ietf.org/html/rfc6125#appendix-B.2
		for _, candidate := range c.IPAddresses {
			if ip.Equal(candidate) {
				return nil
			}
		}
		return HostnameError{c, candidateIP}
	}

	lowered := toLowerCaseASCII(h)

	if len(c.DNSNames) > 0 {
		for _, match := range c.DNSNames {
			if matchHostnames(toLowerCaseASCII(match), lowered) {
				return nil
			}
		}
		// If Subject Alt Name is given, we ignore the common name.
	} else if matchHostnames(toLowerCaseASCII(c.Subject.CommonName), lowered) {
		return nil
	}

	return HostnameError{c, h}
}

func VerifyHostname(c *tls.ConnectionState, host string) error {
	if c == nil {
		return nil
	}

	// if len(c.VerifiedChains) == 0 {
	// 	return errors.New("tls: handshake did not verify certificate chain")
	// }

	return verifyx509(c.PeerCertificates[0], host)
}

// getHandler wraps the standard net/http library, allowing us to spoof hostnames and IP addresses
func (c *CustomClient) getHandler() (*CustomResponse, error) {
	transport := &http.Transport{
		TLSClientConfig: &tls.Config{
			// unfortunately, ServerName will not persist over a redirect. so... we have to ignore
			// ssl invalidations and do them somewhat manually.
			InsecureSkipVerify: true,
			ServerName:         c.Host,
		},
	}
	client := &http.Client{
		CheckRedirect: c.redirectHandler,
		Timeout:       time.Duration(10) * time.Second,
		Transport:     transport,
	}

	req, err := http.NewRequest("GET", c.URL, nil)

	if err != nil {
		return nil, err
	}

	c.OriginURL = req.URL // set origin url for use in redirect wrapper
	c.requestWrap(req)

	// start tracking how long the request is going to take
	timer := NewTimer()

	// actually make the request here
	resp, err := client.Do(req)

	// stop tracking the request
	timer.End()

	var url string

	if err == nil {
		url = c.ResultURL.String()
		if err := VerifyHostname(resp.TLS, c.ResultURL.Host); err != nil {
			return nil, err
		}
	} else {
		url = req.URL.String()
	}

	wrappedResp := &CustomResponse{resp, timer.Result, url}

	return wrappedResp, err
}

// Get wraps GetHandler -- easy interface for making get requests
func Get(ipmap map[string]string, url string) (*CustomResponse, error) {
	host, err := getHost(url)
	if err != nil {
		return nil, err
	}

	ip, ok := ipmap[host]

	if ok {
		if len(ip) > 0 && !reIP.MatchString(ip) {
			return nil, errors.New("IP address provided is invalid")
		}
	}

	c := &CustomClient{URL: url, IP: ip, Host: host, ipmap: ipmap}

	return c.getHandler()
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/github/git-lfs/lfs"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

var (
	BuildOS    = flag.String("os", "", "OS to target: darwin, freebsd, linux, windows")
	BuildArch  = flag.String("arch", "", "Arch to target: 386, amd64")
	BuildAll   = flag.Bool("all", false, "Builds all architectures")
	ShowHelp   = flag.Bool("help", false, "Shows help")
	matrixKeys = map[string]string{
		"darwin":  "Mac",
		"freebsd": "FreeBSD",
		"linux":   "Linux",
		"windows": "Windows",
		"amd64":   "AMD64",
	}
)

func mainBuild() {
	cmd, err := exec.Command("script/fmt").Output()
	if err != nil {
		panic(err)
	}

	if len(cmd) > 0 {
		fmt.Println(string(cmd))
	}

	if *ShowHelp {
		fmt.Println("usage: script/bootstrap [-os] [-arch] [-all]")
		flag.PrintDefaults()
		return
	}

	buildMatrix := make(map[string]Release)

	errored := false

	if *BuildAll {
		for _, buildos := range []string{"darwin", "freebsd", "linux", "windows"} {
			for _, buildarch := range []string{"386", "amd64"} {
				if err := build(buildos, buildarch, buildMatrix); err != nil {
					errored = true
				}
			}
		}
	} else {
		build(*BuildOS, *BuildArch, buildMatrix)
		errored = true // skip build matrix stuff
	}

	if !errored {
		by, err := json.Marshal(buildMatrix)
		if err != nil {
			fmt.Println("Error encoding build matrix to json:", err)
			os.Exit(1)
		}

		file, err := os.Create("bin/releases/build_matrix.json")
		if err != nil {
			fmt.Println("Error creating build_matrix.json:", err)
			os.Exit(1)
		}

		written, err := file.Write(by)
		file.Close()

		if err != nil {
			fmt.Println("Error writing build_matrix.json", err)
			os.Exit(1)
		}

		if jsonSize := len(by); written != jsonSize {
			fmt.Printf("Expected to write %d bytes, actually wrote %d.\n", jsonSize, written)
			os.Exit(1)
		}
	}
}

func build(buildos, buildarch string, buildMatrix map[string]Release) error {
	addenv := len(buildos) > 0 && len(buildarch) > 0
	name := "git-lfs-" + lfs.Version
	dir := "bin"

	if addenv {
		fmt.Printf("Building for %s/%s\n", buildos, buildarch)
		dir = filepath.Join(dir, "releases", buildos+"-"+buildarch, name)
	}

	filepath.Walk("cmd", func(path string, info os.FileInfo, err error) error {
		if !strings.HasSuffix(path, ".go") {
			return nil
		}

		cmd := filepath.Base(path)
		cmd = cmd[0 : len(cmd)-3]
		return buildCommand(path, dir, buildos, buildarch)
	})

	if addenv {
		err := os.MkdirAll(dir, 0755)
		if err != nil {
			fmt.Println("Error setting up installer:\n", err.Error())
			return err
		}

		err = setupInstaller(buildos, buildarch, dir, buildMatrix)
		if err != nil {
			fmt.Println("Error setting up installer:\n", err.Error())
			return err
		}
	}

	return nil
}

func buildCommand(path, dir, buildos, buildarch string) error {
	addenv := len(buildos) > 0 && len(buildarch) > 0
	name := filepath.Base(path)
	name = name[0 : len(name)-3]

	bin := filepath.Join(dir, name)

	if buildos == "windows" {
		bin = bin + ".exe"
	}

	cmd := exec.Command("go", "build", "-o", bin, path)
	if addenv {
		cmd.Env = []string{
			"GOOS=" + buildos,
			"GOARCH=" + buildarch,
			"GOPATH=" + os.Getenv("GOPATH"),
			"GOROOT=" + os.Getenv("GOROOT"),
		}
	}

	output, err := cmd.CombinedOutput()
	if len(output) > 0 {
		fmt.Println(string(output))
	}
	return err
}

func setupInstaller(buildos, buildarch, dir string, buildMatrix map[string]Release) error {
	if buildos == "windows" {
		return winInstaller(buildos, buildarch, dir, buildMatrix)
	} else {
		return unixInstaller(buildos, buildarch, dir, buildMatrix)
	}
}

func unixInstaller(buildos, buildarch, dir string, buildMatrix map[string]Release) error {
	fullInstallPath := filepath.Join(dir, "install.sh")
	cmd := exec.Command("cp", "script/install.sh.example", fullInstallPath)
	if err := logAndRun(cmd); err != nil {
		return err
	}

	if err := os.Chmod(fullInstallPath, 0755); err != nil {
		return err
	}

	name := zipName(buildos, buildarch) + ".tar.gz"

	addToMatrix(buildMatrix, buildos, buildarch, name)

	cmd = exec.Command("tar", "czf", "../"+name, filepath.Base(dir))
	cmd.Dir = filepath.Dir(dir)
	return logAndRun(cmd)
}

func addToMatrix(buildMatrix map[string]Release, buildos, buildarch, name string) {
	buildMatrix[fmt.Sprintf("%s-%s", buildos, buildarch)] = Release{
		Label:    releaseLabel(buildos, buildarch),
		Filename: name,
	}
}

func winInstaller(buildos, buildarch, dir string, buildMatrix map[string]Release) error {
	cmd := exec.Command("cp", "script/install.bat.example", filepath.Join(dir, "install.bat"))
	if err := logAndRun(cmd); err != nil {
		return err
	}

	installerPath := filepath.Dir(filepath.Dir(dir))

	name := zipName(buildos, buildarch) + ".zip"
	full := filepath.Join(installerPath, name)
	matches, err := filepath.Glob(dir + "/*")
	if err != nil {
		return err
	}

	addToMatrix(buildMatrix, buildos, buildarch, name)

	args := make([]string, len(matches)+2)
	args[0] = "-j" // junk the zip paths
	args[1] = full
	copy(args[2:], matches)

	cmd = exec.Command("zip", args...)
	return logAndRun(cmd)
}

func logAndRun(cmd *exec.Cmd) error {
	fmt.Printf(" - %s\n", strings.Join(cmd.Args, " "))
	if len(cmd.Dir) > 0 {
		fmt.Printf("   - in %s\n", cmd.Dir)
	}

	output, err := cmd.CombinedOutput()
	fmt.Println(string(output))
	return err
}

func zipName(os, arch string) string {
	return fmt.Sprintf("git-lfs-%s-%s-%s", os, arch, lfs.Version)
}

func releaseLabel(buildos, buildarch string) string {
	return fmt.Sprintf("%s %s", key(buildos), key(buildarch))
}

func key(k string) string {
	if s, ok := matrixKeys[k]; ok {
		return s
	}
	return k
}
<commit_msg>fix build script<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/github/git-lfs/lfs"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

var (
	BuildOS    = flag.String("os", "", "OS to target: darwin, freebsd, linux, windows")
	BuildArch  = flag.String("arch", "", "Arch to target: 386, amd64")
	BuildAll   = flag.Bool("all", false, "Builds all architectures")
	ShowHelp   = flag.Bool("help", false, "Shows help")
	matrixKeys = map[string]string{
		"darwin":  "Mac",
		"freebsd": "FreeBSD",
		"linux":   "Linux",
		"windows": "Windows",
		"amd64":   "AMD64",
	}
)

func mainBuild() {
	cmd, err := exec.Command("script/fmt").Output()
	if err != nil {
		panic(err)
	}

	if len(cmd) > 0 {
		fmt.Println(string(cmd))
	}

	if *ShowHelp {
		fmt.Println("usage: script/bootstrap [-os] [-arch] [-all]")
		flag.PrintDefaults()
		return
	}

	buildMatrix := make(map[string]Release)

	errored := false

	if *BuildAll {
		for _, buildos := range []string{"darwin", "freebsd", "linux", "windows"} {
			for _, buildarch := range []string{"386", "amd64"} {
				if err := build(buildos, buildarch, buildMatrix); err != nil {
					errored = true
				}
			}
		}
	} else {
		build(*BuildOS, *BuildArch, buildMatrix)
		errored = true // skip build matrix stuff
	}

	if !errored {
		by, err := json.Marshal(buildMatrix)
		if err != nil {
			fmt.Println("Error encoding build matrix to json:", err)
			os.Exit(1)
		}

		file, err := os.Create("bin/releases/build_matrix.json")
		if err != nil {
			fmt.Println("Error creating build_matrix.json:", err)
			os.Exit(1)
		}

		written, err := file.Write(by)
		file.Close()

		if err != nil {
			fmt.Println("Error writing build_matrix.json", err)
			os.Exit(1)
		}

		if jsonSize := len(by); written != jsonSize {
			fmt.Printf("Expected to write %d bytes, actually wrote %d.\n", jsonSize, written)
			os.Exit(1)
		}
	}
}

func build(buildos, buildarch string, buildMatrix map[string]Release) error {
	addenv := len(buildos) > 0 && len(buildarch) > 0
	name := "git-lfs-" + lfs.Version
	dir := "bin"

	if addenv {
		fmt.Printf("Building for %s/%s\n", buildos, buildarch)
		dir = filepath.Join(dir, "releases", buildos+"-"+buildarch, name)
	}

	filepath.Walk("cmd/git-lfs", func(path string, info os.FileInfo, err error) error {
		if !strings.HasSuffix(path, ".go") {
			return nil
		}

		cmd := filepath.Base(path)
		cmd = cmd[0 : len(cmd)-3]
		return buildCommand(path, dir, buildos, buildarch)
	})

	if addenv {
		err := os.MkdirAll(dir, 0755)
		if err != nil {
			fmt.Println("Error setting up installer:\n", err.Error())
			return err
		}

		err = setupInstaller(buildos, buildarch, dir, buildMatrix)
		if err != nil {
			fmt.Println("Error setting up installer:\n", err.Error())
			return err
		}
	}

	return nil
}

func buildCommand(path, dir, buildos, buildarch string) error {
	addenv := len(buildos) > 0 && len(buildarch) > 0
	name := filepath.Base(path)
	name = name[0 : len(name)-3]

	bin := filepath.Join(dir, name)

	if buildos == "windows" {
		bin = bin + ".exe"
	}

	cmd := exec.Command("go", "build", "-o", bin, path)
	if addenv {
		cmd.Env = []string{
			"GOOS=" + buildos,
			"GOARCH=" + buildarch,
			"GOPATH=" + os.Getenv("GOPATH"),
			"GOROOT=" + os.Getenv("GOROOT"),
		}
	}

	output, err := cmd.CombinedOutput()
	if len(output) > 0 {
		fmt.Println(string(output))
	}
	return err
}

func setupInstaller(buildos, buildarch, dir string, buildMatrix map[string]Release) error {
	if buildos == "windows" {
		return winInstaller(buildos, buildarch, dir, buildMatrix)
	} else {
		return unixInstaller(buildos, buildarch, dir, buildMatrix)
	}
}

func unixInstaller(buildos, buildarch, dir string, buildMatrix map[string]Release) error {
	fullInstallPath := filepath.Join(dir, "install.sh")
	cmd := exec.Command("cp", "script/install.sh.example", fullInstallPath)
	if err := logAndRun(cmd); err != nil {
		return err
	}

	if err := os.Chmod(fullInstallPath, 0755); err != nil {
		return err
	}

	name := zipName(buildos, buildarch) + ".tar.gz"

	addToMatrix(buildMatrix, buildos, buildarch, name)

	cmd = exec.Command("tar", "czf", "../"+name, filepath.Base(dir))
	cmd.Dir = filepath.Dir(dir)
	return logAndRun(cmd)
}

func addToMatrix(buildMatrix map[string]Release, buildos, buildarch, name string) {
	buildMatrix[fmt.Sprintf("%s-%s", buildos, buildarch)] = Release{
		Label:    releaseLabel(buildos, buildarch),
		Filename: name,
	}
}

func winInstaller(buildos, buildarch, dir string, buildMatrix map[string]Release) error {
	cmd := exec.Command("cp", "script/install.bat.example", filepath.Join(dir, "install.bat"))
	if err := logAndRun(cmd); err != nil {
		return err
	}

	installerPath := filepath.Dir(filepath.Dir(dir))

	name := zipName(buildos, buildarch) + ".zip"
	full := filepath.Join(installerPath, name)
	matches, err := filepath.Glob(dir + "/*")
	if err != nil {
		return err
	}

	addToMatrix(buildMatrix, buildos, buildarch, name)

	args := make([]string, len(matches)+2)
	args[0] = "-j" // junk the zip paths
	args[1] = full
	copy(args[2:], matches)

	cmd = exec.Command("zip", args...)
	return logAndRun(cmd)
}

func logAndRun(cmd *exec.Cmd) error {
	fmt.Printf(" - %s\n", strings.Join(cmd.Args, " "))
	if len(cmd.Dir) > 0 {
		fmt.Printf("   - in %s\n", cmd.Dir)
	}

	output, err := cmd.CombinedOutput()
	fmt.Println(string(output))
	return err
}

func zipName(os, arch string) string {
	return fmt.Sprintf("git-lfs-%s-%s-%s", os, arch, lfs.Version)
}

func releaseLabel(buildos, buildarch string) string {
	return fmt.Sprintf("%s %s", key(buildos), key(buildarch))
}

func key(k string) string {
	if s, ok := matrixKeys[k]; ok {
		return s
	}
	return k
}
<|endoftext|>


<commit_before>package simplessh

import (
	"code.google.com/p/go.crypto/ssh"
	"github.com/pkg/sftp"

	"io"
	"io/ioutil"
	"net"
	"os"
	"os/user"
	"path/filepath"
)

type Client struct {
	SSHClient *ssh.Client
}

func ConnectWithPassword(host, username, pass string) (*Client, error) {
	authMethod := ssh.Password(pass)

	return connect(username, host, authMethod)
}

// Connect with a private key. If privKeyPath is an empty string it will attempt
// to use $HOME/.ssh/id_rsa.
func ConnectWithKeyFile(host, username, privKeyPath string) (*Client, error) {
	if privKeyPath == "" {
		currentUser, err := user.Current()
		if err == nil {
			privKeyPath = filepath.Join(currentUser.HomeDir, ".ssh", "id_rsa")
		}
	}

	privKey, err := ioutil.ReadFile(privKeyPath)
	if err != nil {
		return nil, err
	}

	return ConnectWithKey(host, username, string(privKey))
}

// Connect with a private key.
func ConnectWithKey(host, username, privKey string) (*Client, error) {
	signer, err := ssh.ParsePrivateKey([]byte(privKey))
	if err != nil {
		return nil, err
	}

	authMethod := ssh.PublicKeys(signer)

	return connect(username, host, authMethod)
}

func connect(username, host string, authMethod ssh.AuthMethod) (*Client, error) {
	config := &ssh.ClientConfig{
		User: username,
		Auth: []ssh.AuthMethod{authMethod},
	}

	host = addPortToHost(host)

	client, err := ssh.Dial("tcp", host, config)
	if err != nil {
		return nil, err
	}

	c := &Client{SSHClient: client}
	return c, nil
}

func (c *Client) Exec(cmd string) ([]byte, error) {
	session, err := c.SSHClient.NewSession()
	if err != nil {
		return nil, err
	}
	defer session.Close()

	return session.CombinedOutput(cmd)
}

func (c *Client) Download(remote, local string) error {
	client, err := sftp.NewClient(c.SSHClient)
	if err != nil {
		return err
	}
	defer client.Close()

	remoteFile, err := client.Open(remote)
	if err != nil {
		return err
	}
	defer remoteFile.Close()

	localFile, err := os.Create(local)
	if err != nil {
		return err
	}
	defer localFile.Close()

	_, err = io.Copy(localFile, remoteFile)
	return err
}

func (c *Client) Upload(local, remote string) error {
	client, err := sftp.NewClient(c.SSHClient)
	if err != nil {
		return err
	}
	defer client.Close()

	localFile, err := os.Open(local)
	if err != nil {
		return err
	}
	defer localFile.Close()

	remoteFile, err := client.Create(remote)
	if err != nil {
		return err
	}

	_, err = io.Copy(remoteFile, localFile)
	return err
}

// Read a remote file and return the contents.
func (c *Client) ReadAll(filepath string) ([]byte, error) {
	sftp, err := c.SFTPClient()
	if err != nil {
		panic(err)
	}
	defer sftp.Close()

	file, err := sftp.Open(filepath)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	return ioutil.ReadAll(file)
}

// Close the underlying SSH connection
func (c *Client) Close() error {
	return c.SSHClient.Close()
}

// Return an sftp client. The client needs to be closed when it's no
// longer needed.
func (c *Client) SFTPClient() (*sftp.Client, error) {
	return sftp.NewClient(c.SSHClient)
}

func addPortToHost(host string) string {
	_, _, err := net.SplitHostPort(host)

	// We got an error so blindly try to add a port number
	if err != nil {
		return net.JoinHostPort(host, "22")
	}

	return host
}
<commit_msg>Allow custom timeouts when connecting to remote servers.<commit_after>package simplessh

import (
	"code.google.com/p/go.crypto/ssh"
	"github.com/pkg/sftp"

	"io"
	"io/ioutil"
	"net"
	"os"
	"os/user"
	"path/filepath"
	"time"
)

var DefaultTimeout = 30 * time.Second

type Client struct {
	SSHClient *ssh.Client
}

// Connect with a password.
func ConnectWithPassword(host, username, pass string) (*Client, error) {
	return ConnectWithPasswordTimeout(host, username, pass, DefaultTimeout)
}

// Same as ConnectWithPassword but allows a custom timeout.
func ConnectWithPasswordTimeout(host, username, pass string, timeout time.Duration) (*Client, error) {
	authMethod := ssh.Password(pass)

	return connect(username, host, authMethod, timeout)
}

// Connect with a private key. If privKeyPath is an empty string it will attempt
// to use $HOME/.ssh/id_rsa.
func ConnectWithKeyFileTimeout(host, username, privKeyPath string, timeout time.Duration) (*Client, error) {
	if privKeyPath == "" {
		currentUser, err := user.Current()
		if err == nil {
			privKeyPath = filepath.Join(currentUser.HomeDir, ".ssh", "id_rsa")
		}
	}

	privKey, err := ioutil.ReadFile(privKeyPath)
	if err != nil {
		return nil, err
	}

	return ConnectWithKey(host, username, string(privKey), timeout)
}

// Same as ConnectWithKeyFile but allows a custom timeout.
func ConnectWithKeyFile(host, username, privKeyPath string) (*Client, error) {
	return ConnectWithKeyFileTimeout(host, username, privKeyPath, DefaultTimeout)
}

// Connect with a private key.
func ConnectWithKeyTimeout(host, username, privKey string, timeout time.Duration) (*Client, error) {
	signer, err := ssh.ParsePrivateKey([]byte(privKey))
	if err != nil {
		return nil, err
	}

	authMethod := ssh.PublicKeys(signer)

	return connect(username, host, authMethod, timeout)
}

// Same as ConnectWithKey but allows a custom timeout.
func ConnectWithKey(host, username, privKey string, timeout time.Duration) (*Client, error) {
	return ConnectWithKeyTimeout(host, username, privKey, DefaultTimeout)
}

func connect(username, host string, authMethod ssh.AuthMethod, timeout time.Duration) (*Client, error) {
	config := &ssh.ClientConfig{
		User: username,
		Auth: []ssh.AuthMethod{authMethod},
	}

	host = addPortToHost(host)

	conn, err := net.DialTimeout("tcp", host, timeout)
	if err != nil {
		return nil, err
	}
	sshConn, chans, reqs, err := ssh.NewClientConn(conn, host, config)
	if err != nil {
		return nil, err
	}
	client := ssh.NewClient(sshConn, chans, reqs)

	c := &Client{SSHClient: client}
	return c, nil
}

func (c *Client) Exec(cmd string) ([]byte, error) {
	session, err := c.SSHClient.NewSession()
	if err != nil {
		return nil, err
	}
	defer session.Close()

	return session.CombinedOutput(cmd)
}

func (c *Client) Download(remote, local string) error {
	client, err := sftp.NewClient(c.SSHClient)
	if err != nil {
		return err
	}
	defer client.Close()

	remoteFile, err := client.Open(remote)
	if err != nil {
		return err
	}
	defer remoteFile.Close()

	localFile, err := os.Create(local)
	if err != nil {
		return err
	}
	defer localFile.Close()

	_, err = io.Copy(localFile, remoteFile)
	return err
}

func (c *Client) Upload(local, remote string) error {
	client, err := sftp.NewClient(c.SSHClient)
	if err != nil {
		return err
	}
	defer client.Close()

	localFile, err := os.Open(local)
	if err != nil {
		return err
	}
	defer localFile.Close()

	remoteFile, err := client.Create(remote)
	if err != nil {
		return err
	}

	_, err = io.Copy(remoteFile, localFile)
	return err
}

// Read a remote file and return the contents.
func (c *Client) ReadAll(filepath string) ([]byte, error) {
	sftp, err := c.SFTPClient()
	if err != nil {
		panic(err)
	}
	defer sftp.Close()

	file, err := sftp.Open(filepath)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	return ioutil.ReadAll(file)
}

// Close the underlying SSH connection
func (c *Client) Close() error {
	return c.SSHClient.Close()
}

// Return an sftp client. The client needs to be closed when it's no
// longer needed.
func (c *Client) SFTPClient() (*sftp.Client, error) {
	return sftp.NewClient(c.SSHClient)
}

func addPortToHost(host string) string {
	_, _, err := net.SplitHostPort(host)

	// We got an error so blindly try to add a port number
	if err != nil {
		return net.JoinHostPort(host, "22")
	}

	return host
}
<|endoftext|>


<commit_before>//  project main.go
package main

import (
	"fmt"
	"log"
	"os"
	"net/http"
	"net/url"
	"time"
	"strings"
	"encoding/json"
	"bitbucket.org/kardianos/service"
	"bitbucket.org/kardianos/osext"
	"github.com/BurntSushi/toml"
	"path"
)

type TeamID string
type TeamName string
type Channel string

type Team struct {
	id TeamID
	url string
}

type Destination struct {
	url string
	channel Channel
}

type MappingID struct {
	id TeamID
	channel Channel
}

// Special Structures for the config parsing

type configTeam struct {
	ID TeamID
	URL string
}

type configMapping struct {
	FromTeam TeamName
	FromChannel Channel
	ToTeam TeamName
	ToChannel Channel
}

type tomlConfig struct {
	BindAddr string
	Port uint16
	SSLCrt string
	SSLKey string
	Teams map[TeamName]configTeam `toml:"team"`
	Mappings []configMapping `toml:"mapping"`
}

var teams=make(map[TeamName]Team)
var team_namen=make(map[TeamID]TeamName)
var mapping=make(map[MappingID][]Destination)

func AddMapping(from_team TeamName, from_channel Channel, to_team TeamName, to_channel Channel) {
	id:=MappingID{
		id:teams[from_team].id,
		channel:from_channel,
	}
	mapping[id]=append(mapping[id],Destination{
			url: teams[to_team].url,
			channel: to_channel })
}

func AddTeam(name TeamName, id TeamID, url string) {
	teams[name]=Team{
		id: id,
		url: url }
	team_namen[id]=name
}

func NoCache(w http.ResponseWriter) {
	h:=w.Header()
	h.Add("Expires","Mon, 1 Jan 2000 00:00:00 UTC")
	h.Add("Last-Modified",time.Now().UTC().Format(time.RFC1123))
	h.Add("Cache-Control", "no-store, no-cache, must-revalidate")
	//h.Add("Cache-Control", "post-check=0, pre-check=0");
	h.Add("Pragma","no-cache")
}

func BotAnswers(w http.ResponseWriter, text string) {
	w.Header().Set("Content-Type", "application/json")
	tmp := map[string]interface{}{"text": text }
	res, _ := json.Marshal(tmp)
	fmt.Fprintf(w, string(res));
	return
}

func OnRequest(w http.ResponseWriter, r *http.Request) {
	NoCache(w)	// well we never want it cached!

	team_id:=TeamID(r.PostFormValue("team_id"))

	if(team_id=="") {
		w.WriteHeader(http.StatusPreconditionFailed)
		fmt.Fprintf(w, "Illegal access to Slackgate... (c) METATEXX GmbH 2014")
		return
	}

	from_channel:=Channel("#"+r.PostFormValue("channel_name"))
	mapping_id:=MappingID{
		id: team_id,
		channel: from_channel,
	}

	user_name:=r.PostFormValue("user_name");
	if(user_name=="slackbot") {
		w.WriteHeader(http.StatusNoContent);
		return
	}

	dsts, ok:=mapping[mapping_id]
	if(! ok) {
		BotAnswers(w, fmt.Sprintf("Unknown Team: %q",mapping_id))
		return
	}

	msg_body:=r.PostFormValue("text");

	fmt.Printf("%s %s / %s: %q\n",team_id,from_channel,user_name,msg_body)

	if(msg_body=="+info") {
		BotAnswers(w, "This is SlackToGo Server (c) METATEXX GmbH 2014 - Written by Hans Raaf")
		return
	}

	if(msg_body=="+date") {
		BotAnswers(w, time.Now().UTC().Format(time.RFC1123))
		return
	}

	if(msg_body=="+routing") {
		out:=make([]string,0)
		for mid, maps := range mapping {
			if(len(out)>0) {
				out=append(out,"");
			}
			out=append(out,fmt.Sprintf("Routing messages from %q (%s) %s to:", team_namen[mid.id], mid.id, mid.channel));
			for idx, dst := range maps {
				out=append(out,fmt.Sprintf("> %d:  %q @ %q",idx,dst.url, dst.channel));
			}
		}
		BotAnswers(w, strings.Join(out,"\n"))
		return
	}

	for _, dst := range dsts {
		tmp := map[string]interface{}{
			"channel": dst.channel,
			"username": fmt.Sprintf("*%s",user_name),
			"text": msg_body,
			"icon_emoji": ":twisted_rightwards_arrows:" }
		res, _ := json.Marshal(tmp)

		resp, err := http.PostForm(dst.url,
			url.Values{"payload": { string(res)}})
		if(err!=nil) {
			w.Header().Set("Content-Type", "application/json")
			tmp := map[string]interface{}{"text": fmt.Sprintf("Error: %q!",err.Error()) }
			res, _ := json.Marshal(tmp)
			fmt.Fprintf(w, string(res));
			return
		}

		if(resp.StatusCode!=200) {
			fmt.Println(resp)
		}

	}

	return
}

func checkConfig() {
	fmt.Printf("BindAddr: %q\n",config.BindAddr)
	fmt.Printf("Port: %d\n",config.Port)
	fmt.Printf("SSL Certificate: %q\n",config.SSLCrt)
	fmt.Printf("SSL Key: %q\n",config.SSLKey)

	fmt.Println("\nTeams:")
	for name, team := range config.Teams {
		fmt.Printf("%s (%s) %s\n",
			name,
			team.ID,
			team.URL,
		)
	}

	fmt.Println("\nMappings:")
	for idx, mapping := range config.Mappings {
		fmt.Printf("%d: %s %s -> %s %s\n",
			idx,
			mapping.FromTeam,
			mapping.FromChannel,
			mapping.ToTeam,
			mapping.ToChannel,
		)
	}
}

var logit service.Logger

var config tomlConfig

func main() {

	// Config Defaults
	config.BindAddr="[::]"
	config.Port=8080
	config.SSLCrt=""
	config.SSLKey=""

	// The "go run" command creates the executable in the tmp dir of the system
	// the "service" wrapper runs a strange current dir

	curdir, err := os.Getwd()
	if err != nil {
		log.Fatal(err)
	}

	//dir, err := filepath.Abs(filepath.Dir(os.Args[0]))
	prgdir, err := osext.ExecutableFolder()
	if err != nil {
		log.Fatal(err)
	}

	// trying to find the config file
	if _, err := toml.DecodeFile(path.Join(prgdir,"config.toml"), &config); err != nil {
		if _, err := toml.DecodeFile(path.Join(curdir,"config.toml"), &config); err != nil {
			fmt.Println(err)
			return
		}
	}

	// Service Setup
	var name = "SlackToGo"
	var displayName = "Slack Team Gateway"
	var desc = "This Gateway connects two channels on different teams ins slack with each other."

	s, err := service.NewService(name, displayName, desc)
	logit = s

	if err != nil {
		fmt.Printf("%s unable to start: %s", displayName, err)
		return
	}

	if len(os.Args) > 1 {
		var err error
		verb := os.Args[1]
		switch verb {
		case "install":
			err = s.Install()
			if err != nil {
				fmt.Printf("Failed to install: %s\n", err)
				return
			}
			fmt.Printf("Service %q installed.\n", displayName)
		case "remove":
			err = s.Remove()
			if err != nil {
				fmt.Printf("Failed to remove: %s\n", err)
				return
			}
			fmt.Printf("Service %q removed.\n", displayName)
		case "start":
			err = s.Start()
			if err != nil {
				fmt.Printf("Failed to start: %s\n", err)
				return
			}
			fmt.Printf("Service %q started.\n", displayName)
		case "stop":
			err = s.Stop()
			if err != nil {
				fmt.Printf("Failed to stop: %s\n", err)
				return
			}
			fmt.Printf("Service %q stopped.\n", displayName)
		case "check":
			checkConfig()
		case "run":
			doWork()
		}
		return
	}
	err = s.Run(func() error {
			// start
			go doWork()
			return nil
		}, func() error {
			// stop
			stopWork()
			return nil
		})
	if err != nil {
		logit.Error(err.Error())
	}
}

func msgCreator() {
	logit.Info("Notifying all target teams and channels!")

	for _, all := range mapping {
		for _, dst := range all {
			tmp := map[string]interface{}{
				"channel": dst.channel,
				"username": "*SlackToGo",
				"text": "Up and ready!",
				"icon_emoji": ":twisted_rightwards_arrows:" }
			res, _ := json.Marshal(tmp)

			resp, err := http.PostForm(dst.url,
				url.Values{"payload": { string(res)}})
			if(err!=nil) {
				fmt.Printf("Error: %q!\n",err.Error())
				return
			}

			if (resp.StatusCode != 200) {
				fmt.Println(resp)
			}
		}
	}
}

func doWork() {
	logit.Info("SlackToGo Server started!")

	// SetupRouting
	for name, team := range config.Teams {
		AddTeam(name,team.ID,team.URL)
	}

	for _, mp := range config.Mappings {
		AddMapping(mp.FromTeam,mp.FromChannel,mp.ToTeam,mp.ToChannel)
	}

	msgCreator()

	http.HandleFunc("/", OnRequest)

	if(config.SSLCrt=="" || config.SSLKey=="") {
		log.Fatal(http.ListenAndServe(fmt.Sprintf("%s:%d",config.BindAddr,config.Port),
			nil))
	} else {
		log.Fatal(http.ListenAndServeTLS(fmt.Sprintf("%s:%d",config.BindAddr,config.Port),
			config.SSLCrt,
			config.SSLKey,
			nil))
	}

	select {}
}

func stopWork() {
	logit.Info("SlackToGo Server stopping!")
}
<commit_msg>Most certainly fixed the problem with failed hosts lookups after some "thousand" messages. Added simple version information in the +info command.<commit_after>//  project main.go
package main

import (
	"fmt"
	"log"
	"os"
	"net/http"
	"net/url"
	"time"
	"strings"
	"encoding/json"
	"bitbucket.org/kardianos/service"
	"bitbucket.org/kardianos/osext"
	"github.com/BurntSushi/toml"
	"path"
)

type TeamID string
type TeamName string
type Channel string

type Team struct {
	id TeamID
	url string
}

type Destination struct {
	url string
	channel Channel
}

type MappingID struct {
	id TeamID
	channel Channel
}

// Special Structures for the config parsing

type configTeam struct {
	ID TeamID
	URL string
}

type configMapping struct {
	FromTeam TeamName
	FromChannel Channel
	ToTeam TeamName
	ToChannel Channel
}

type tomlConfig struct {
	BindAddr string
	Port uint16
	SSLCrt string
	SSLKey string
	Teams map[TeamName]configTeam `toml:"team"`
	Mappings []configMapping `toml:"mapping"`
}

var teams=make(map[TeamName]Team)
var team_namen=make(map[TeamID]TeamName)
var mapping=make(map[MappingID][]Destination)

func AddMapping(from_team TeamName, from_channel Channel, to_team TeamName, to_channel Channel) {
	id:=MappingID{
		id:teams[from_team].id,
		channel:from_channel,
	}
	mapping[id]=append(mapping[id],Destination{
			url: teams[to_team].url,
			channel: to_channel })
}

func AddTeam(name TeamName, id TeamID, url string) {
	teams[name]=Team{
		id: id,
		url: url }
	team_namen[id]=name
}

func NoCache(w http.ResponseWriter) {
	h:=w.Header()
	h.Add("Expires","Mon, 1 Jan 2000 00:00:00 UTC")
	h.Add("Last-Modified",time.Now().UTC().Format(time.RFC1123))
	h.Add("Cache-Control", "no-store, no-cache, must-revalidate")
	//h.Add("Cache-Control", "post-check=0, pre-check=0");
	h.Add("Pragma","no-cache")
}

func BotAnswers(w http.ResponseWriter, text string) {
	w.Header().Set("Content-Type", "application/json")
	tmp := map[string]interface{}{"text": text }
	res, _ := json.Marshal(tmp)
	fmt.Fprintf(w, string(res));
	return
}

func OnRequest(w http.ResponseWriter, r *http.Request) {
	defer r.Body.Close()

	NoCache(w)	// well we never want it cached!

	team_id:=TeamID(r.PostFormValue("team_id"))

	if(team_id=="") {
		w.WriteHeader(http.StatusPreconditionFailed)
		fmt.Fprintf(w, "Illegal access to Slackgate... (c) METATEXX GmbH 2014")
		return
	}

	from_channel:=Channel("#"+r.PostFormValue("channel_name"))
	mapping_id:=MappingID{
		id: team_id,
		channel: from_channel,
	}

	user_name:=r.PostFormValue("user_name");
	if(user_name=="slackbot") {
		w.WriteHeader(http.StatusNoContent);
		return
	}

	dsts, ok:=mapping[mapping_id]
	if(! ok) {
		BotAnswers(w, fmt.Sprintf("Unknown Team: %q",mapping_id))
		return
	}

	msg_body:=r.PostFormValue("text");

	fmt.Printf("%s %s / %s: %q\n",team_id,from_channel,user_name,msg_body)

	if(msg_body=="+info") {
		BotAnswers(w, "This is SlackToGo Server V0.1.1 (c) METATEXX GmbH 2014 - Written by Hans Raaf")
		return
	}

	if(msg_body=="+date") {
		BotAnswers(w, time.Now().UTC().Format(time.RFC1123))
		return
	}

	if(msg_body=="+routing") {
		out:=make([]string,0)
		for mid, maps := range mapping {
			if(len(out)>0) {
				out=append(out,"");
			}
			out=append(out,fmt.Sprintf("Routing messages from %q (%s) %s to:", team_namen[mid.id], mid.id, mid.channel));
			for idx, dst := range maps {
				out=append(out,fmt.Sprintf("> %d:  %q @ %q",idx,dst.url, dst.channel));
			}
		}
		BotAnswers(w, strings.Join(out,"\n"))
		return
	}

	for _, dst := range dsts {
		tmp := map[string]interface{}{
			"channel": dst.channel,
			"username": fmt.Sprintf("*%s",user_name),
			"text": msg_body,
			"icon_emoji": ":twisted_rightwards_arrows:" }
		res, _ := json.Marshal(tmp)

		resp, err := http.PostForm(dst.url,
			url.Values{"payload": { string(res)}})
		if(err!=nil) {
			w.Header().Set("Content-Type", "application/json")
			tmp := map[string]interface{}{"text": fmt.Sprintf("Error: %q!",err.Error()) }
			res, _ := json.Marshal(tmp)
			fmt.Fprintf(w, string(res));
			return
		}

		if(resp.StatusCode!=200) {
			fmt.Println(resp)
		}

	}

	return
}

func checkConfig() {
	fmt.Printf("BindAddr: %q\n",config.BindAddr)
	fmt.Printf("Port: %d\n",config.Port)
	fmt.Printf("SSL Certificate: %q\n",config.SSLCrt)
	fmt.Printf("SSL Key: %q\n",config.SSLKey)

	fmt.Println("\nTeams:")
	for name, team := range config.Teams {
		fmt.Printf("%s (%s) %s\n",
			name,
			team.ID,
			team.URL,
		)
	}

	fmt.Println("\nMappings:")
	for idx, mapping := range config.Mappings {
		fmt.Printf("%d: %s %s -> %s %s\n",
			idx,
			mapping.FromTeam,
			mapping.FromChannel,
			mapping.ToTeam,
			mapping.ToChannel,
		)
	}
}

var logit service.Logger

var config tomlConfig

func main() {

	// Config Defaults
	config.BindAddr="[::]"
	config.Port=8080
	config.SSLCrt=""
	config.SSLKey=""

	// The "go run" command creates the executable in the tmp dir of the system
	// the "service" wrapper runs a strange current dir

	curdir, err := os.Getwd()
	if err != nil {
		log.Fatal(err)
	}

	//dir, err := filepath.Abs(filepath.Dir(os.Args[0]))
	prgdir, err := osext.ExecutableFolder()
	if err != nil {
		log.Fatal(err)
	}

	// trying to find the config file
	if _, err := toml.DecodeFile(path.Join(prgdir,"config.toml"), &config); err != nil {
		if _, err := toml.DecodeFile(path.Join(curdir,"config.toml"), &config); err != nil {
			fmt.Println(err)
			return
		}
	}

	// Service Setup
	var name = "SlackToGo"
	var displayName = "Slack Team Gateway"
	var desc = "This Gateway connects two channels on different teams ins slack with each other."

	s, err := service.NewService(name, displayName, desc)
	logit = s

	if err != nil {
		fmt.Printf("%s unable to start: %s", displayName, err)
		return
	}

	if len(os.Args) > 1 {
		var err error
		verb := os.Args[1]
		switch verb {
		case "install":
			err = s.Install()
			if err != nil {
				fmt.Printf("Failed to install: %s\n", err)
				return
			}
			fmt.Printf("Service %q installed.\n", displayName)
		case "remove":
			err = s.Remove()
			if err != nil {
				fmt.Printf("Failed to remove: %s\n", err)
				return
			}
			fmt.Printf("Service %q removed.\n", displayName)
		case "start":
			err = s.Start()
			if err != nil {
				fmt.Printf("Failed to start: %s\n", err)
				return
			}
			fmt.Printf("Service %q started.\n", displayName)
		case "stop":
			err = s.Stop()
			if err != nil {
				fmt.Printf("Failed to stop: %s\n", err)
				return
			}
			fmt.Printf("Service %q stopped.\n", displayName)
		case "check":
			checkConfig()
		case "run":
			doWork()
		}
		return
	}
	err = s.Run(func() error {
			// start
			go doWork()
			return nil
		}, func() error {
			// stop
			stopWork()
			return nil
		})
	if err != nil {
		logit.Error(err.Error())
	}
}

func msgCreator() {
	logit.Info("Notifying all target teams and channels!")

	for _, all := range mapping {
		for _, dst := range all {
			tmp := map[string]interface{}{
				"channel": dst.channel,
				"username": "*SlackToGo",
				"text": "Up and ready!",
				"icon_emoji": ":twisted_rightwards_arrows:" }
			res, _ := json.Marshal(tmp)

			resp, err := http.PostForm(dst.url,
				url.Values{"payload": { string(res)}})
			if(err!=nil) {
				fmt.Printf("Error: %q!\n",err.Error())
				return
			}

			if (resp.StatusCode != 200) {
				fmt.Println(resp)
			}
		}
	}
}

func doWork() {
	logit.Info("SlackToGo Server started!")

	// SetupRouting
	for name, team := range config.Teams {
		AddTeam(name,team.ID,team.URL)
	}

	for _, mp := range config.Mappings {
		AddMapping(mp.FromTeam,mp.FromChannel,mp.ToTeam,mp.ToChannel)
	}

	msgCreator()

	http.HandleFunc("/", OnRequest)

	if(config.SSLCrt=="" || config.SSLKey=="") {
		log.Fatal(http.ListenAndServe(fmt.Sprintf("%s:%d",config.BindAddr,config.Port),
			nil))
	} else {
		log.Fatal(http.ListenAndServeTLS(fmt.Sprintf("%s:%d",config.BindAddr,config.Port),
			config.SSLCrt,
			config.SSLKey,
			nil))
	}

	select {}
}

func stopWork() {
	logit.Info("SlackToGo Server stopping!")
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"os/user"
	"path"
	"syscall"
)

type SlangoutsApp struct {
	configFile     string
	config         SlangoutsConfig
	hangoutsClient *HangoutsClient
	slackClient    *SlackClient
}

type SlangoutsConfig struct {
	HangoutsRefreshToken string `json:"hangoutsRefreshToken"`
	SlackApiToken        string `json:"slackApiToken"`
}

func runSlangouts(hangoutsPollFreq int, configPath string) {
	if configPath == "" {
		configPath = getConfigPath()
	}
	app := &SlangoutsApp{configFile: configPath, hangoutsClient: &HangoutsClient{PollFrequency: hangoutsPollFreq}, slackClient: &SlackClient{}}
	app.Run()
}

func (app *SlangoutsApp) Run() {
	err := app.loadConfig()
	if err != nil {
		log.Println(err, ". Generating blank config...")
		app.saveConfig()
	}

	// check Slack token
	err = app.slackClient.Init(app.config.SlackApiToken)
	if err != nil {
		log.Fatal(err)
	}

	// update Slack token if changed
	if app.slackClient.Token != app.config.SlackApiToken {
		app.config.SlackApiToken = app.slackClient.Token
		app.saveConfig()
		log.Println("Slack token changed. Saving it.")
	}

	// check Hangouts token
	err = app.hangoutsClient.Init(app.config.HangoutsRefreshToken)
	if err != nil {
		//re-try without a refresh token
		app.config.HangoutsRefreshToken = ""
		err = app.hangoutsClient.Init(app.config.HangoutsRefreshToken)
		if err != nil {
			log.Fatal(err)
		}
	}

	// update Hangouts token if changed
	if app.hangoutsClient.Session.RefreshToken != app.config.HangoutsRefreshToken {
		app.config.HangoutsRefreshToken = app.hangoutsClient.Session.RefreshToken
		app.saveConfig()
		log.Println("Hangouts refresh token changed. Saving it.")
	}

	// start Slack Polling
	app.slackClient.StartPolling()
	//defer app.slackClient.StopPolling()

	// start Hangouts Polling
	app.hangoutsClient.StartPolling()
	defer app.hangoutsClient.StopPolling()

	// catch Ctrl+C
	signals := make(chan os.Signal, 1)
	signal.Notify(signals, os.Interrupt, os.Kill, syscall.SIGABRT)

	// event loop
	for {
		select {
		case <-signals:
			return
		case msg := <-app.hangoutsClient.Messages:
			log.Println("Hangouts: ", msg)
			err = app.slackClient.SendMessage(msg)
			if err != nil {
				log.Println(err)
			}
		case msg := <-app.slackClient.Messages:
			log.Println("Slack: ", msg)
			err = app.hangoutsClient.SendMessage(msg)
			if err != nil {
				log.Println(err)
			}
		}
	}
}

func (app *SlangoutsApp) loadConfig() error {
	b, err := ioutil.ReadFile(app.configFile)
	if err == nil {
		return json.Unmarshal(b, &app.config)
	}
	return err
}

func (app *SlangoutsApp) saveConfig() error {
	j, err := json.MarshalIndent(&app.config, "", "\t")
	if err == nil {
		os.MkdirAll(path.Dir(app.configFile), os.ModePerm)
		return ioutil.WriteFile(app.configFile, j, 0600)
	}
	return err
}

func getConfigPath() string {
	user, err := user.Current()
	OrDie(err)
	return path.Join(user.HomeDir, ".slangouts", "config.json")
}

func OrDie(e error) {
	if e != nil {
		log.Fatal(e)
	}
}
<commit_msg>Dont rely on os/user to get homedir<commit_after>package main

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"path"
	"syscall"
)

type SlangoutsApp struct {
	configFile     string
	config         SlangoutsConfig
	hangoutsClient *HangoutsClient
	slackClient    *SlackClient
}

type SlangoutsConfig struct {
	HangoutsRefreshToken string `json:"hangoutsRefreshToken"`
	SlackApiToken        string `json:"slackApiToken"`
}

func runSlangouts(hangoutsPollFreq int, configPath string) {
	if configPath == "" {
		configPath = getConfigPath()
	}
	app := &SlangoutsApp{configFile: configPath, hangoutsClient: &HangoutsClient{PollFrequency: hangoutsPollFreq}, slackClient: &SlackClient{}}
	app.Run()
}

func (app *SlangoutsApp) Run() {
	err := app.loadConfig()
	if err != nil {
		log.Println(err, ". Generating blank config...")
		app.saveConfig()
	}

	// check Slack token
	err = app.slackClient.Init(app.config.SlackApiToken)
	if err != nil {
		log.Fatal(err)
	}

	// update Slack token if changed
	if app.slackClient.Token != app.config.SlackApiToken {
		app.config.SlackApiToken = app.slackClient.Token
		app.saveConfig()
		log.Println("Slack token changed. Saving it.")
	}

	// check Hangouts token
	err = app.hangoutsClient.Init(app.config.HangoutsRefreshToken)
	if err != nil {
		//re-try without a refresh token
		app.config.HangoutsRefreshToken = ""
		err = app.hangoutsClient.Init(app.config.HangoutsRefreshToken)
		if err != nil {
			log.Fatal(err)
		}
	}

	// update Hangouts token if changed
	if app.hangoutsClient.Session.RefreshToken != app.config.HangoutsRefreshToken {
		app.config.HangoutsRefreshToken = app.hangoutsClient.Session.RefreshToken
		app.saveConfig()
		log.Println("Hangouts refresh token changed. Saving it.")
	}

	// start Slack Polling
	app.slackClient.StartPolling()
	//defer app.slackClient.StopPolling()

	// start Hangouts Polling
	app.hangoutsClient.StartPolling()
	defer app.hangoutsClient.StopPolling()

	// catch Ctrl+C
	signals := make(chan os.Signal, 1)
	signal.Notify(signals, os.Interrupt, os.Kill, syscall.SIGABRT)

	// event loop
	for {
		select {
		case <-signals:
			return
		case msg := <-app.hangoutsClient.Messages:
			log.Println("Hangouts: ", msg)
			err = app.slackClient.SendMessage(msg)
			if err != nil {
				log.Println(err)
			}
		case msg := <-app.slackClient.Messages:
			log.Println("Slack: ", msg)
			err = app.hangoutsClient.SendMessage(msg)
			if err != nil {
				log.Println(err)
			}
		}
	}
}

func (app *SlangoutsApp) loadConfig() error {
	b, err := ioutil.ReadFile(app.configFile)
	if err == nil {
		return json.Unmarshal(b, &app.config)
	}
	return err
}

func (app *SlangoutsApp) saveConfig() error {
	j, err := json.MarshalIndent(&app.config, "", "\t")
	if err == nil {
		os.MkdirAll(path.Dir(app.configFile), os.ModePerm)
		return ioutil.WriteFile(app.configFile, j, 0600)
	}
	return err
}

func getConfigPath() string {
	workingDir := "."
	for _, name := range []string{"HOME", "USERPROFILE"} { // *nix, windows
		if dir := os.Getenv(name); dir != "" {
			workingDir = dir
		}
	}

	return path.Join(workingDir, ".slangouts", "config.json")
}

func OrDie(e error) {
	if e != nil {
		log.Fatal(e)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"os"

	"github.com/codegangsta/cli"
	"github.com/markbates/pop/soda/commands"
)

func main() {
	app := cli.NewApp()
	app.Name = "soda"
	app.Usage = "A tasty treat for all your database needs"
	app.Version = "2.7.2"

	app.Commands = []cli.Command{
		commands.Migrate(),
		commands.Create(),
		commands.Drop(),
	}

	app.Run(os.Args)
}
<commit_msg>tick version number up<commit_after>package main

import (
	"os"

	"github.com/codegangsta/cli"
	"github.com/markbates/pop/soda/commands"
)

func main() {
	app := cli.NewApp()
	app.Name = "soda"
	app.Usage = "A tasty treat for all your database needs"
	app.Version = "2.7.3"

	app.Commands = []cli.Command{
		commands.Migrate(),
		commands.Create(),
		commands.Drop(),
	}

	app.Run(os.Args)
}
<|endoftext|>