<commit_before>package router

import (
	"github.com/smancke/guble/server/metrics"
)

var (
	mTotalSubscriptionAttempts                 = metrics.NewInt("guble.router.total_subscription_attempts")
	mTotalDuplicateSubscriptionsAttempts       = metrics.NewInt("guble.router.total_subscription_attempts_duplicate")
	mTotalSubscriptions                        = metrics.NewInt("guble.router.total_subscriptions")
	mTotalUnsubscriptionAttempts               = metrics.NewInt("guble.router.total_unsubscription_attempts")
	mTotalInvalidTopicOnUnsubscriptionAttempts = metrics.NewInt("guble.router.total_unsubscription_attempts_invalid_topic")
	mTotalInvalidUnsubscriptionAttempts        = metrics.NewInt("guble.router.total_unsubscription_attempts_invalid")
	mTotalUnsubscriptions                      = metrics.NewInt("guble.router.total_unsubscriptions")
	mCurrentSubscriptions                      = metrics.NewInt("guble.router.current_subscriptions")
	mCurrentRoutes                             = metrics.NewInt("guble.router.current_routes")
	mTotalMessagesIncoming                     = metrics.NewInt("guble.router.total_messages_incoming")
	mTotalMessagesIncomingBytes                = metrics.NewInt("guble.router.total_messages_bytes_incoming")
	mTotalMessagesStoredBytes                  = metrics.NewInt("guble.router.total_messages_bytes_stored")
	mTotalMessagesRouted                       = metrics.NewInt("guble.router.total_messages_routed")
	mTotalOverloadedHandleChannel              = metrics.NewInt("guble.router.total_overloaded_handle_channel")
	mTotalMessagesNotMatchingTopic             = metrics.NewInt("guble.router.total_messages_not_matching_topic")
	mTotalMessageStoreErrors                   = metrics.NewInt("guble.router.total_errors_message_store")
	mTotalDeliverMessageErrors                 = metrics.NewInt("guble.router.total_errors_deliver_message")
	mTotalNotMatchedByFilters                  = metrics.NewInt("guble.router.total_not_matched_by_filters")
)

func resetRouterMetrics() {
	mTotalSubscriptionAttempts.Set(0)
	mTotalDuplicateSubscriptionsAttempts.Set(0)
	mTotalSubscriptions.Set(0)
	mTotalUnsubscriptionAttempts.Set(0)
	mTotalInvalidTopicOnUnsubscriptionAttempts.Set(0)
	mTotalUnsubscriptions.Set(0)
	mTotalInvalidUnsubscriptionAttempts.Set(0)
	mCurrentSubscriptions.Set(0)
	mCurrentRoutes.Set(0)
	mTotalMessagesIncoming.Set(0)
	mTotalMessagesRouted.Set(0)
	mTotalOverloadedHandleChannel.Set(0)
	mTotalMessagesNotMatchingTopic.Set(0)
	mTotalDeliverMessageErrors.Set(0)
	mTotalMessageStoreErrors.Set(0)
	mTotalMessagesIncomingBytes.Set(0)
	mTotalMessagesStoredBytes.Set(0)
	mTotalNotMatchedByFilters.Set(0)
}
<commit_msg>shorter metrics keys - router<commit_after>package router

import (
	"github.com/smancke/guble/server/metrics"
)

var (
	mTotalSubscriptionAttempts                 = metrics.NewInt("router.total_subscription_attempts")
	mTotalDuplicateSubscriptionsAttempts       = metrics.NewInt("router.total_subscription_attempts_duplicate")
	mTotalSubscriptions                        = metrics.NewInt("router.total_subscriptions")
	mTotalUnsubscriptionAttempts               = metrics.NewInt("router.total_unsubscription_attempts")
	mTotalInvalidTopicOnUnsubscriptionAttempts = metrics.NewInt("router.total_unsubscription_attempts_invalid_topic")
	mTotalInvalidUnsubscriptionAttempts        = metrics.NewInt("router.total_unsubscription_attempts_invalid")
	mTotalUnsubscriptions                      = metrics.NewInt("router.total_unsubscriptions")
	mCurrentSubscriptions                      = metrics.NewInt("router.current_subscriptions")
	mCurrentRoutes                             = metrics.NewInt("router.current_routes")
	mTotalMessagesIncoming                     = metrics.NewInt("router.total_messages_incoming")
	mTotalMessagesIncomingBytes                = metrics.NewInt("router.total_messages_bytes_incoming")
	mTotalMessagesStoredBytes                  = metrics.NewInt("router.total_messages_bytes_stored")
	mTotalMessagesRouted                       = metrics.NewInt("router.total_messages_routed")
	mTotalOverloadedHandleChannel              = metrics.NewInt("router.total_overloaded_handle_channel")
	mTotalMessagesNotMatchingTopic             = metrics.NewInt("router.total_messages_not_matching_topic")
	mTotalMessageStoreErrors                   = metrics.NewInt("router.total_errors_message_store")
	mTotalDeliverMessageErrors                 = metrics.NewInt("router.total_errors_deliver_message")
	mTotalNotMatchedByFilters                  = metrics.NewInt("router.total_not_matched_by_filters")
)

func resetRouterMetrics() {
	mTotalSubscriptionAttempts.Set(0)
	mTotalDuplicateSubscriptionsAttempts.Set(0)
	mTotalSubscriptions.Set(0)
	mTotalUnsubscriptionAttempts.Set(0)
	mTotalInvalidTopicOnUnsubscriptionAttempts.Set(0)
	mTotalUnsubscriptions.Set(0)
	mTotalInvalidUnsubscriptionAttempts.Set(0)
	mCurrentSubscriptions.Set(0)
	mCurrentRoutes.Set(0)
	mTotalMessagesIncoming.Set(0)
	mTotalMessagesRouted.Set(0)
	mTotalOverloadedHandleChannel.Set(0)
	mTotalMessagesNotMatchingTopic.Set(0)
	mTotalDeliverMessageErrors.Set(0)
	mTotalMessageStoreErrors.Set(0)
	mTotalMessagesIncomingBytes.Set(0)
	mTotalMessagesStoredBytes.Set(0)
	mTotalNotMatchedByFilters.Set(0)
}
<|endoftext|>


<commit_before>package libmyna

import (
	"errors"
	"fmt"
	"github.com/ebfe/scard"
	"os"
	"time"
)

type Reader struct {
	ctx   *scard.Context
	name  string
	card  *scard.Card
	debug bool
}

func NewReader() (*Reader, error) {
	ctx, err := scard.EstablishContext()
	if err != nil {
		return nil, err
	}

	readers, err := ctx.ListReaders()
	if err != nil {
		return nil, err
	}

	if len(readers) == 0 {
		return nil, fmt.Errorf("リーダーが見つかりません")
	}

	if len(readers) >= 2 {
		fmt.Fprintf(os.Stderr,
			"警告: 複数のリーダーが見つかりました。最初のものを使います\n")
	}

	reader := new(Reader)
	reader.ctx = ctx
	reader.name = readers[0]
	reader.card = nil
	return reader, nil
}

func (self *Reader) SetDebug(debug bool) {
	self.debug = debug
}

func (self *Reader) Finalize() {
	self.ctx.Release()
}

func (self *Reader) GetCard() *scard.Card {
	card, _ := self.ctx.Connect(
		self.name, scard.ShareExclusive, scard.ProtocolAny)
	self.card = card
	return card
}

func (self *Reader) Connect() error {
	rs := make([]scard.ReaderState, 1)
	rs[0].Reader = self.name
	rs[0].CurrentState = scard.StateUnaware // no need
	var err error
	for i := 0; i < 5; i++ {
		err = self.ctx.GetStatusChange(rs, -1)
		if err != nil {
			return err
		}

		if rs[0].EventState&scard.StatePresent != 0 {
			card, e := self.ctx.Connect(
				self.name, scard.ShareExclusive, scard.ProtocolAny)
			if e == nil {
				self.card = card
				return nil
			} else {
				err = e
			}
		}
		fmt.Fprintf(os.Stderr, "connecting...\n")
		time.Sleep(1 * time.Second)
	}
	if err != nil {
		return err
	}
	return errors.New("カードが見つかりません")
}

func (self *Reader) SelectCardInfoAP() (*CardInfoAP, error) {
	err := self.SelectDF("D3 92 10 00 31 00 01 01 04 02")
	ap := CardInfoAP{self}
	return &ap, err
}

func (self *Reader) SelectCardInputHelperAP() (*CardInputHelperAP, error) {
	err := self.SelectDF("D3 92 10 00 31 00 01 01 04 08")
	ap := CardInputHelperAP{self}
	return &ap, err
}

func (self *Reader) SelectJPKIAP() (*JPKIAP, error) {
	err := self.SelectDF("D3 92 f0 00 26 01 00 00 00 01")
	ap := JPKIAP{self}
	return &ap, err
}

func (self *Reader) SelectDF(id string) error {
	if self.debug {
		fmt.Fprintf(os.Stderr, "# Select DF\n")
	}
	bid := ToBytes(id)
	apdu := "00 A4 04 0C" + fmt.Sprintf(" %02X % X", len(bid), bid)
	sw1, sw2, _ := self.Tx(apdu)
	if sw1 == 0x90 && sw2 == 0x00 {
		return nil
	} else {
		return NewAPDUError(sw1, sw2)
	}
}

func (self *Reader) SelectEF(id string) error {
	if self.debug {
		fmt.Fprintf(os.Stderr, "# Select EF\n")
	}
	bid := ToBytes(id)
	apdu := fmt.Sprintf("00 A4 02 0C %02X % X", len(bid), bid)
	sw1, sw2, _ := self.Tx(apdu)
	if sw1 == 0x90 && sw2 == 0x00 {
		return nil
	} else {
		return NewAPDUError(sw1, sw2)
	}
}

func (self *Reader) LookupPin() int {
	if self.debug {
		fmt.Fprintf(os.Stderr, "# Lookup PIN\n")
	}
	sw1, sw2, _ := self.Tx("00 20 00 80")
	if sw1 == 0x63 {
		return int(sw2 & 0x0F)
	} else {
		return -1
	}
}

func (self *Reader) Verify(pin string) error {
	var apdu string
	if pin == "" {
		return errors.New("PINが空です")
	}
	if self.debug {
		fmt.Fprintf(os.Stderr, "# Verify PIN\n")
	}
	bpin := []byte(pin)
	apdu = fmt.Sprintf("00 20 00 80 %02X % X", len(bpin), bpin)
	sw1, sw2, _ := self.Tx(apdu)
	if sw1 == 0x90 && sw2 == 0x00 {
		return nil
	} else if sw1 == 0x63 {
		counter := int(sw2 & 0x0F)
		if counter == 0 {
			return errors.New("暗証番号が間違っています。ブロックされました")
		}
		return fmt.Errorf("暗証番号が間違っています。のこり%d回", counter)
	} else if sw1 == 0x69 && sw2 == 0x84 {
		return errors.New("暗証番号がブロックされています。")
	} else {
		return fmt.Errorf("暗証番号が間違っています SW1=%02X SW2=%02X",
			sw1, sw2)
	}
}

func (self *Reader) ChangePin(pin string) bool {
	var apdu string
	if self.debug {
		fmt.Fprintf(os.Stderr, "# Change PIN\n")
	}
	bpin := []byte(pin)
	apdu = fmt.Sprintf("00 24 01 80 %02X % X", len(bpin), bpin)
	sw1, sw2, _ := self.Tx(apdu)
	if sw1 == 0x90 && sw2 == 0x00 {
		return true
	} else {
		return false
	}
}

func dumpBinary(bin []byte) {
	for i := 0; i < len(bin); i++ {
		if i%0x10 == 0 {
			fmt.Fprintf(os.Stderr, ">")
		}
		fmt.Fprintf(os.Stderr, " %02X", bin[i])
		if i%0x10 == 0x0f {
			fmt.Fprintln(os.Stderr)
		}
	}
	fmt.Fprintln(os.Stderr)
}

func (self *Reader) Tx(apdu string) (uint8, uint8, []byte) {
	card := self.card
	if self.debug {
		fmt.Fprintf(os.Stderr, "< %v\n", apdu)
	}
	cmd := ToBytes(apdu)
	res, err := card.Transmit(cmd)
	if err != nil {
		fmt.Fprintf(os.Stderr, "err: %s\n", err)
		return 0, 0, nil
	}

	if self.debug {
		dumpBinary(res)
	}

	l := len(res)
	if l == 2 {
		return res[0], res[1], nil
	} else if l > 2 {
		return res[l-2], res[l-1], res[:l-2]
	}
	return 0, 0, nil
}

func (self *Reader) ReadBinary(size uint16) []byte {
	if self.debug {
		fmt.Fprintf(os.Stderr, "# Read Binary\n")
	}

	var l uint8
	var apdu string
	var pos uint16
	pos = 0
	var res []byte

	for pos < size {
		if size-pos > 0xFF {
			l = 0
		} else {
			l = uint8(size - pos)
		}
		apdu = fmt.Sprintf("00 B0 %02X %02X %02X",
			pos>>8&0xFF, pos&0xFF, l)
		sw1, sw2, data := self.Tx(apdu)
		if sw1 != 0x90 || sw2 != 0x00 {
			return nil
		}
		res = append(res, data...)
		pos += uint16(len(data))
	}
	return res
}

func (self *Reader) Signature(data []byte) ([]byte, error) {
	if self.debug {
		fmt.Fprintf(os.Stderr, "# Signature\n")
	}

	apdu := fmt.Sprintf("80 2a 00 80 %02X % X 00", len(data), data)
	sw1, sw2, res := self.Tx(apdu)
	if sw1 == 0x90 && sw2 == 0x00 {
		return res, nil
	} else {
		return nil, fmt.Errorf("署名エラー(%0X, %0X)", sw1, sw2)
	}
}
<commit_msg>verfyコマンドのログをマスク<commit_after>package libmyna

import (
	"errors"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/ebfe/scard"
)

type Reader struct {
	ctx   *scard.Context
	name  string
	card  *scard.Card
	debug bool
}

func NewReader() (*Reader, error) {
	ctx, err := scard.EstablishContext()
	if err != nil {
		return nil, err
	}

	readers, err := ctx.ListReaders()
	if err != nil {
		return nil, err
	}

	if len(readers) == 0 {
		return nil, fmt.Errorf("リーダーが見つかりません")
	}

	if len(readers) >= 2 {
		fmt.Fprintf(os.Stderr,
			"警告: 複数のリーダーが見つかりました。最初のものを使います\n")
	}

	reader := new(Reader)
	reader.ctx = ctx
	reader.name = readers[0]
	reader.card = nil
	return reader, nil
}

func (self *Reader) SetDebug(debug bool) {
	self.debug = debug
}

func (self *Reader) Finalize() {
	self.ctx.Release()
}

func (self *Reader) GetCard() *scard.Card {
	card, _ := self.ctx.Connect(
		self.name, scard.ShareExclusive, scard.ProtocolAny)
	self.card = card
	return card
}

func (self *Reader) Connect() error {
	rs := make([]scard.ReaderState, 1)
	rs[0].Reader = self.name
	rs[0].CurrentState = scard.StateUnaware // no need
	var err error
	for i := 0; i < 5; i++ {
		err = self.ctx.GetStatusChange(rs, -1)
		if err != nil {
			return err
		}

		if rs[0].EventState&scard.StatePresent != 0 {
			card, e := self.ctx.Connect(
				self.name, scard.ShareExclusive, scard.ProtocolAny)
			if e == nil {
				self.card = card
				return nil
			} else {
				err = e
			}
		}
		fmt.Fprintf(os.Stderr, "connecting...\n")
		time.Sleep(1 * time.Second)
	}
	if err != nil {
		return err
	}
	return errors.New("カードが見つかりません")
}

func (self *Reader) SelectCardInfoAP() (*CardInfoAP, error) {
	err := self.SelectDF("D3 92 10 00 31 00 01 01 04 02")
	ap := CardInfoAP{self}
	return &ap, err
}

func (self *Reader) SelectCardInputHelperAP() (*CardInputHelperAP, error) {
	err := self.SelectDF("D3 92 10 00 31 00 01 01 04 08")
	ap := CardInputHelperAP{self}
	return &ap, err
}

func (self *Reader) SelectJPKIAP() (*JPKIAP, error) {
	err := self.SelectDF("D3 92 f0 00 26 01 00 00 00 01")
	ap := JPKIAP{self}
	return &ap, err
}

func (self *Reader) SelectDF(id string) error {
	if self.debug {
		fmt.Fprintf(os.Stderr, "# Select DF\n")
	}
	bid := ToBytes(id)
	apdu := "00 A4 04 0C" + fmt.Sprintf(" %02X % X", len(bid), bid)
	sw1, sw2, _ := self.Tx(apdu)
	if sw1 == 0x90 && sw2 == 0x00 {
		return nil
	} else {
		return NewAPDUError(sw1, sw2)
	}
}

func (self *Reader) SelectEF(id string) error {
	if self.debug {
		fmt.Fprintf(os.Stderr, "# Select EF\n")
	}
	bid := ToBytes(id)
	apdu := fmt.Sprintf("00 A4 02 0C %02X % X", len(bid), bid)
	sw1, sw2, _ := self.Tx(apdu)
	if sw1 == 0x90 && sw2 == 0x00 {
		return nil
	} else {
		return NewAPDUError(sw1, sw2)
	}
}

func (self *Reader) LookupPin() int {
	if self.debug {
		fmt.Fprintf(os.Stderr, "# Lookup PIN\n")
	}
	sw1, sw2, _ := self.Tx("00 20 00 80")
	if sw1 == 0x63 {
		return int(sw2 & 0x0F)
	} else {
		return -1
	}
}

func (self *Reader) Verify(pin string) error {
	var apdu string
	if pin == "" {
		return errors.New("PINが空です")
	}
	if self.debug {
		fmt.Fprintf(os.Stderr, "# Verify PIN\n")
	}
	bpin := []byte(pin)
	apdu = fmt.Sprintf("00 20 00 80 %02X % X", len(bpin), bpin)
	sw1, sw2, _ := self.Tx(apdu)
	if sw1 == 0x90 && sw2 == 0x00 {
		return nil
	} else if sw1 == 0x63 {
		counter := int(sw2 & 0x0F)
		if counter == 0 {
			return errors.New("暗証番号が間違っています。ブロックされました")
		}
		return fmt.Errorf("暗証番号が間違っています。のこり%d回", counter)
	} else if sw1 == 0x69 && sw2 == 0x84 {
		return errors.New("暗証番号がブロックされています。")
	} else {
		return fmt.Errorf("暗証番号が間違っています SW1=%02X SW2=%02X",
			sw1, sw2)
	}
}

func (self *Reader) ChangePin(pin string) bool {
	var apdu string
	if self.debug {
		fmt.Fprintf(os.Stderr, "# Change PIN\n")
	}
	bpin := []byte(pin)
	apdu = fmt.Sprintf("00 24 01 80 %02X % X", len(bpin), bpin)
	sw1, sw2, _ := self.Tx(apdu)
	if sw1 == 0x90 && sw2 == 0x00 {
		return true
	} else {
		return false
	}
}

func dumpBinary(bin []byte) {
	for i := 0; i < len(bin); i++ {
		if i%0x10 == 0 {
			fmt.Fprintf(os.Stderr, ">")
		}
		fmt.Fprintf(os.Stderr, " %02X", bin[i])
		if i%0x10 == 0x0f {
			fmt.Fprintln(os.Stderr)
		}
	}
	fmt.Fprintln(os.Stderr)
}

func (self *Reader) Tx(apdu string) (uint8, uint8, []byte) {
	card := self.card
	cmd := ToBytes(apdu)
	if self.debug {
		if cmd[0] == 0x00 && cmd[1] == 0x20 {
			len := int(cmd[4])
			mask := strings.Repeat(" XX", len)
			fmt.Fprintf(os.Stderr, "< % X XX%s\n", cmd[:4], mask)
		} else {
			fmt.Fprintf(os.Stderr, "< % X\n", cmd)
		}
	}
	res, err := card.Transmit(cmd)
	if err != nil {
		fmt.Fprintf(os.Stderr, "err: %s\n", err)
		return 0, 0, nil
	}

	if self.debug {
		dumpBinary(res)
	}

	l := len(res)
	if l == 2 {
		return res[0], res[1], nil
	} else if l > 2 {
		return res[l-2], res[l-1], res[:l-2]
	}
	return 0, 0, nil
}

func (self *Reader) ReadBinary(size uint16) []byte {
	if self.debug {
		fmt.Fprintf(os.Stderr, "# Read Binary\n")
	}

	var l uint8
	var apdu string
	var pos uint16
	pos = 0
	var res []byte

	for pos < size {
		if size-pos > 0xFF {
			l = 0
		} else {
			l = uint8(size - pos)
		}
		apdu = fmt.Sprintf("00 B0 %02X %02X %02X",
			pos>>8&0xFF, pos&0xFF, l)
		sw1, sw2, data := self.Tx(apdu)
		if sw1 != 0x90 || sw2 != 0x00 {
			return nil
		}
		res = append(res, data...)
		pos += uint16(len(data))
	}
	return res
}

func (self *Reader) Signature(data []byte) ([]byte, error) {
	if self.debug {
		fmt.Fprintf(os.Stderr, "# Signature\n")
	}

	apdu := fmt.Sprintf("80 2a 00 80 %02X % X 00", len(data), data)
	sw1, sw2, res := self.Tx(apdu)
	if sw1 == 0x90 && sw2 == 0x00 {
		return res, nil
	} else {
		return nil, fmt.Errorf("署名エラー(%0X, %0X)", sw1, sw2)
	}
}
<|endoftext|>


<commit_before>// Copyright 2009-2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package math

/*
	Floating-point mod function.
*/

// Mod returns the floating-point remainder of x/y.
// The magnitude of the result is less than y and its
// sign agrees with that of x.
//
// Special cases are:
//	Mod(±Inf, y) = NaN
//	Mod(NaN, y) = NaN
//	Mod(x, 0) = NaN
//	Mod(x, ±Inf) = x
//	Mod(x, NaN) = NaN
func Mod(x, y float64) float64

func mod(x, y float64) float64 {
	if y == 0 || IsInf(x, 0) || IsNaN(x) || IsNaN(y) {
		return NaN()
	}
	if y < 0 {
		y = -y
	}

	yfr, yexp := Frexp(y)
	sign := false
	r := x
	if x < 0 {
		r = -x
		sign = true
	}

	for r >= y {
		rfr, rexp := Frexp(r)
		if rfr < yfr {
			rexp = rexp - 1
		}
		r = r - Ldexp(y, rexp-yexp)
	}
	if sign {
		r = -r
	}
	return r
}
<commit_msg>math: use Abs in Mod rather than if x < 0 { x = -x}<commit_after>// Copyright 2009-2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package math

/*
	Floating-point mod function.
*/

// Mod returns the floating-point remainder of x/y.
// The magnitude of the result is less than y and its
// sign agrees with that of x.
//
// Special cases are:
//	Mod(±Inf, y) = NaN
//	Mod(NaN, y) = NaN
//	Mod(x, 0) = NaN
//	Mod(x, ±Inf) = x
//	Mod(x, NaN) = NaN
func Mod(x, y float64) float64

func mod(x, y float64) float64 {
	if y == 0 || IsInf(x, 0) || IsNaN(x) || IsNaN(y) {
		return NaN()
	}
	y = Abs(y)

	yfr, yexp := Frexp(y)
	r := x
	if x < 0 {
		r = -x
	}

	for r >= y {
		rfr, rexp := Frexp(r)
		if rfr < yfr {
			rexp = rexp - 1
		}
		r = r - Ldexp(y, rexp-yexp)
	}
	if x < 0 {
		r = -r
	}
	return r
}
<|endoftext|>


<commit_before>package lnwallet

import (
	"path/filepath"

	"github.com/roasbeef/btcd/chaincfg"
	"github.com/roasbeef/btcd/wire"
	_ "github.com/roasbeef/btcwallet/walletdb/bdb"
)

// networkDir returns the directory name of a network directory to hold wallet
// files.
func networkDir(dataDir string, chainParams *chaincfg.Params) string {
	netname := chainParams.Name

	// For now, we must always name the testnet data directory as "testnet"
	// and not "testnet3" or any other version, as the chaincfg testnet3
	// paramaters will likely be switched to being named "testnet3" in the
	// future.  This is done to future proof that change, and an upgrade
	// plan to move the testnet3 data directory can be worked out later.
	if chainParams.Net == wire.TestNet3 {
		netname = "testnet"
	}

	return filepath.Join(dataDir, netname)
}
<commit_msg>lnwallet: remove setup.go<commit_after><|endoftext|>


<commit_before>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package logger

import (
	"encoding/json"
	_ "fmt"
	"log"
	"net/http"
	"strconv"
	"sync"
	"time"

	"github.com/venicegeo/pz-gocommon/elasticsearch"
	"github.com/venicegeo/pz-gocommon/gocommon"
)

type LockedAdminStats struct {
	sync.Mutex
	LoggerAdminStats
}

type LogData struct {
	sync.Mutex
	esIndex elasticsearch.IIndex
	id      int
}

const schema = "LogData5"

type Service struct {
	stats   LockedAdminStats
	logData LogData
	origin  string
}

func (service *Service) Init(sys *piazza.SystemConfig, esIndex elasticsearch.IIndex) error {
	var err error

	service.stats.CreatedOn = time.Now()
	/***
	err = esIndex.Delete()
	if err != nil {
		log.Fatal(err)
	}
	if esIndex.IndexExists() {
		log.Fatal("index still exists")
	}
	err = esIndex.Create()
	if err != nil {
		log.Fatal(err)
	}
	***/

	if !esIndex.IndexExists() {
		log.Printf("Creating index: %s", esIndex.IndexName())
		err = esIndex.Create("")
		if err != nil {
			log.Fatal(err)
		}
	}

	if !esIndex.TypeExists(schema) {
		log.Printf("Creating type: %s", schema)

		mapping :=
			`{
			"LogData5":{
				"properties":{
					"service":{
						"type": "string",
						"store": true
					},
					"address":{
						"type": "string",
						"store": true
					},
					"createdOn":{
						"type": "date",
						"store": true
					},
					"severity":{
						"type": "string",
						"store": true
					},
					"message":{
						"type": "string",
						"store": true
					}
				}
			}
		}`

		err = esIndex.SetMapping(schema, piazza.JsonString(mapping))
		if err != nil {
			log.Printf("LoggerService.Init: %s", err.Error())
			return err
		}
	}

	service.logData.esIndex = esIndex

	service.origin = string(sys.Name)

	return nil
}

func (service *Service) newInternalErrorResponse(err error) *piazza.JsonResponse {
	return &piazza.JsonResponse{
		StatusCode: http.StatusInternalServerError,
		Message:    err.Error(),
		Origin:     service.origin,
	}
}

func (service *Service) newBadRequestResponse(err error) *piazza.JsonResponse {
	return &piazza.JsonResponse{
		StatusCode: http.StatusBadRequest,
		Message:    err.Error(),
		Origin:     service.origin,
	}
}

func (service *Service) GetRoot() *piazza.JsonResponse {
	resp := &piazza.JsonResponse{
		StatusCode: 200,
		Data:       "Hi. I'm pz-logger.",
	}

	err := resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func (service *Service) PostMessage(mssg *Message) *piazza.JsonResponse {
	err := mssg.Validate()
	if err != nil {
		return service.newBadRequestResponse(err)
	}

	service.logData.Lock()
	idStr := strconv.Itoa(service.logData.id)
	service.logData.id++
	service.logData.Unlock()

	_, err = service.logData.esIndex.PostData(schema, idStr, mssg)
	if err != nil {
		//log.Printf("POST failed (1): %#v %#v", err, indexResult)
		return service.newInternalErrorResponse(err)
	}

	/*	if !indexResult.Created {
		log.Printf("POST failed (2): %#v", *indexResult)
		return NewInternalErrorResponse(err)
	}*/

	service.stats.LoggerAdminStats.NumMessages++

	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       mssg,
	}

	err = resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func (service *Service) GetStats() *piazza.JsonResponse {
	service.logData.Lock()
	t := service.stats.LoggerAdminStats
	service.logData.Unlock()

	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       t,
	}

	err := resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func (service *Service) GetMessage(params *piazza.HttpQueryParams) *piazza.JsonResponse {
	var err error

	defaultPagination := &piazza.JsonPagination{
		PerPage: 10,
		Page:    0,
		Order:   piazza.PaginationOrderDescending,
		SortBy:  "createdOn",
	}
	pagination, err := piazza.NewJsonPagination(params, defaultPagination)
	if err != nil {
		return service.newBadRequestResponse(err)
	}

	dsl, err := createQueryDslAsString(pagination, params)
	if err != nil {
		return service.newBadRequestResponse(err)
	}

	log.Printf("dsl: %s", dsl)

	var searchResult *elasticsearch.SearchResult

	if dsl == "" {
		searchResult, err = service.logData.esIndex.FilterByMatchAll(schema, pagination)
	} else {
		searchResult, err = service.logData.esIndex.SearchByJSON(schema, dsl)
	}

	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	var lines []Message

	if searchResult != nil && searchResult.GetHits() != nil {
		for _, hit := range *searchResult.GetHits() {
			if hit.Source == nil {
				log.Printf("null source hit")
				continue
			}

			var msg Message
			err = json.Unmarshal(*hit.Source, &msg)
			if err != nil {
				log.Printf("UNABLE TO PARSE: %s", string(*hit.Source))
				return service.newInternalErrorResponse(err)
			}

			// just in case
			err = msg.Validate()
			if err != nil {
				log.Printf("UNABLE TO VALIDATE: %s", string(*hit.Source))
				continue
			}

			lines = append(lines, msg)
			log.Printf("ADDING %#v (%d)", lines, len(lines))
		}
	}

	pagination.Count = int(searchResult.TotalHits())
	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       lines,
		Pagination: pagination,
	}

	err = resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func createQueryDslAsString(
	pagination *piazza.JsonPagination,
	params *piazza.HttpQueryParams) (string, error) {

	must := []map[string]interface{}{}

	service, err := params.AsString("service", nil)
	if err != nil {
		return "", err
	}

	contains, err := params.AsString("contains", nil)
	if err != nil {
		return "", err
	}

	before, err := params.AsTime("before", nil)
	if err != nil {
		return "", err
	}

	after, err := params.AsTime("after", nil)
	if err != nil {
		return "", err
	}

	if service != nil {
		must = append(must, map[string]interface{}{
			"match": map[string]interface{}{
				"service": *service,
			},
		})
	}

	if contains != nil {
		must = append(must, map[string]interface{}{
			"multi_match": map[string]interface{}{
				"query":  contains,
				"fields": []string{"address", "message", "service", "severity"},
			},
		})
	}

	if after != nil || before != nil {
		rangeParams := map[string]time.Time{}

		if after != nil {
			rangeParams["gte"] = *after
		}

		if before != nil {
			rangeParams["lte"] = *before
		}

		must = append(must, map[string]interface{}{
			"range": map[string]interface{}{
				"createdOn": rangeParams,
			},
		})
	}

	if len(must) == 0 {
		return "", nil
	}

	dsl := map[string]interface{}{
		"query": map[string]interface{}{
			"filtered": map[string]interface{}{
				"query": map[string]interface{}{
					"bool": map[string]interface{}{
						"must": must,
					},
				},
			},
		},
		"size": pagination.PerPage,
		"from": pagination.PerPage * pagination.Page,
	}

	dsl["sort"] = map[string]string{
		pagination.SortBy: string(pagination.Order),
	}

	output, err := json.Marshal(dsl)
	if err != nil {
		return "", err
	}

	return string(output), nil
}
<commit_msg>debugging<commit_after>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package logger

import (
	"encoding/json"
	_ "fmt"
	"log"
	"net/http"
	"strconv"
	"sync"
	"time"

	"github.com/venicegeo/pz-gocommon/elasticsearch"
	"github.com/venicegeo/pz-gocommon/gocommon"
)

type LockedAdminStats struct {
	sync.Mutex
	LoggerAdminStats
}

type LogData struct {
	sync.Mutex
	esIndex elasticsearch.IIndex
	id      int
}

const schema = "LogData5"

type Service struct {
	stats   LockedAdminStats
	logData LogData
	origin  string
}

func (service *Service) Init(sys *piazza.SystemConfig, esIndex elasticsearch.IIndex) error {
	var err error

	service.stats.CreatedOn = time.Now()
	/***
	err = esIndex.Delete()
	if err != nil {
		log.Fatal(err)
	}
	if esIndex.IndexExists() {
		log.Fatal("index still exists")
	}
	err = esIndex.Create()
	if err != nil {
		log.Fatal(err)
	}
	***/

	if !esIndex.IndexExists() {
		log.Printf("Creating index: %s", esIndex.IndexName())
		err = esIndex.Create("")
		if err != nil {
			log.Fatal(err)
		}
	}

	if !esIndex.TypeExists(schema) {
		log.Printf("Creating type: %s", schema)

		mapping :=
			`{
			"LogData5":{
				"properties":{
					"service":{
						"type": "string",
						"store": true
					},
					"address":{
						"type": "string",
						"store": true
					},
					"createdOn":{
						"type": "date",
						"store": true
					},
					"severity":{
						"type": "string",
						"store": true
					},
					"message":{
						"type": "string",
						"store": true
					}
				}
			}
		}`

		err = esIndex.SetMapping(schema, piazza.JsonString(mapping))
		if err != nil {
			log.Printf("LoggerService.Init: %s", err.Error())
			return err
		}
	}

	service.logData.esIndex = esIndex

	service.origin = string(sys.Name)

	return nil
}

func (service *Service) newInternalErrorResponse(err error) *piazza.JsonResponse {
	return &piazza.JsonResponse{
		StatusCode: http.StatusInternalServerError,
		Message:    err.Error(),
		Origin:     service.origin,
	}
}

func (service *Service) newBadRequestResponse(err error) *piazza.JsonResponse {
	return &piazza.JsonResponse{
		StatusCode: http.StatusBadRequest,
		Message:    err.Error(),
		Origin:     service.origin,
	}
}

func (service *Service) GetRoot() *piazza.JsonResponse {
	resp := &piazza.JsonResponse{
		StatusCode: 200,
		Data:       "Hi. I'm pz-logger.",
	}

	err := resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func (service *Service) PostMessage(mssg *Message) *piazza.JsonResponse {
	err := mssg.Validate()
	if err != nil {
		return service.newBadRequestResponse(err)
	}

	service.logData.Lock()
	idStr := strconv.Itoa(service.logData.id)
	service.logData.id++
	service.logData.Unlock()

	_, err = service.logData.esIndex.PostData(schema, idStr, mssg)
	if err != nil {
		//log.Printf("POST failed (1): %#v %#v", err, indexResult)
		return service.newInternalErrorResponse(err)
	}

	/*	if !indexResult.Created {
		log.Printf("POST failed (2): %#v", *indexResult)
		return NewInternalErrorResponse(err)
	}*/

	service.stats.LoggerAdminStats.NumMessages++

	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       mssg,
	}

	err = resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func (service *Service) GetStats() *piazza.JsonResponse {
	service.logData.Lock()
	t := service.stats.LoggerAdminStats
	service.logData.Unlock()

	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       t,
	}

	err := resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func (service *Service) GetMessage(params *piazza.HttpQueryParams) *piazza.JsonResponse {
	var err error

	defaultPagination := &piazza.JsonPagination{
		PerPage: 10,
		Page:    0,
		Order:   piazza.PaginationOrderDescending,
		SortBy:  "createdOn",
	}
	pagination, err := piazza.NewJsonPagination(params, defaultPagination)
	if err != nil {
		return service.newBadRequestResponse(err)
	}

	dsl, err := createQueryDslAsString(pagination, params)
	if err != nil {
		return service.newBadRequestResponse(err)
	}

	log.Printf("dsl: %s", dsl)

	var searchResult *elasticsearch.SearchResult

	if dsl == "" {
		searchResult, err = service.logData.esIndex.FilterByMatchAll(schema, pagination)
	} else {
		searchResult, err = service.logData.esIndex.SearchByJSON(schema, dsl)
	}

	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	var lines = make([]Message, 0)

	if searchResult != nil && searchResult.GetHits() != nil {
		for _, hit := range *searchResult.GetHits() {
			if hit.Source == nil {
				log.Printf("null source hit")
				continue
			}

			var msg Message
			err = json.Unmarshal(*hit.Source, &msg)
			if err != nil {
				log.Printf("UNABLE TO PARSE: %s", string(*hit.Source))
				return service.newInternalErrorResponse(err)
			}

			// just in case
			err = msg.Validate()
			if err != nil {
				log.Printf("UNABLE TO VALIDATE: %s", string(*hit.Source))
				continue
			}

			lines = append(lines, msg)
			log.Printf("ADDING %#v (%d)", msg, len(lines))
		}
	}

	pagination.Count = int(searchResult.TotalHits())
	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       lines,
		Pagination: pagination,
	}

	err = resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}
	log.Printf("RESP: %#v", resp)
	return resp
}

func createQueryDslAsString(
	pagination *piazza.JsonPagination,
	params *piazza.HttpQueryParams) (string, error) {

	must := []map[string]interface{}{}

	service, err := params.AsString("service", nil)
	if err != nil {
		return "", err
	}

	contains, err := params.AsString("contains", nil)
	if err != nil {
		return "", err
	}

	before, err := params.AsTime("before", nil)
	if err != nil {
		return "", err
	}

	after, err := params.AsTime("after", nil)
	if err != nil {
		return "", err
	}

	if service != nil {
		must = append(must, map[string]interface{}{
			"match": map[string]interface{}{
				"service": *service,
			},
		})
	}

	if contains != nil {
		must = append(must, map[string]interface{}{
			"multi_match": map[string]interface{}{
				"query":  contains,
				"fields": []string{"address", "message", "service", "severity"},
			},
		})
	}

	if after != nil || before != nil {
		rangeParams := map[string]time.Time{}

		if after != nil {
			rangeParams["gte"] = *after
		}

		if before != nil {
			rangeParams["lte"] = *before
		}

		must = append(must, map[string]interface{}{
			"range": map[string]interface{}{
				"createdOn": rangeParams,
			},
		})
	}

	if len(must) == 0 {
		return "", nil
	}

	dsl := map[string]interface{}{
		"query": map[string]interface{}{
			"filtered": map[string]interface{}{
				"query": map[string]interface{}{
					"bool": map[string]interface{}{
						"must": must,
					},
				},
			},
		},
		"size": pagination.PerPage,
		"from": pagination.PerPage * pagination.Page,
	}

	dsl["sort"] = map[string]string{
		pagination.SortBy: string(pagination.Order),
	}

	output, err := json.Marshal(dsl)
	if err != nil {
		return "", err
	}

	return string(output), nil
}
<|endoftext|>


<commit_before>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package logger

import (
	"encoding/json"
	_ "fmt"
	"log"
	"net/http"
	"strconv"
	"sync"
	"time"

	"github.com/venicegeo/pz-gocommon/elasticsearch"
	"github.com/venicegeo/pz-gocommon/gocommon"
)

type LockedAdminStats struct {
	sync.Mutex
	LoggerAdminStats
}

type LogData struct {
	sync.Mutex
	esIndex elasticsearch.IIndex
	id      int
}

const schema = "LogData5"

type Service struct {
	stats   LockedAdminStats
	logData LogData
	origin  string
}

func (service *Service) Init(sys *piazza.SystemConfig, esIndex elasticsearch.IIndex) error {
	var err error

	service.stats.CreatedOn = time.Now()
	/***
	err = esIndex.Delete()
	if err != nil {
		log.Fatal(err)
	}
	if esIndex.IndexExists() {
		log.Fatal("index still exists")
	}
	err = esIndex.Create()
	if err != nil {
		log.Fatal(err)
	}
	***/

	if !esIndex.IndexExists() {
		log.Printf("Creating index: %s", esIndex.IndexName())
		err = esIndex.Create("")
		if err != nil {
			log.Fatal(err)
		}
	}

	if !esIndex.TypeExists(schema) {
		log.Printf("Creating type: %s", schema)

		mapping :=
			`{
			"LogData5":{
				"properties":{
					"service":{
						"type": "string",
						"store": true
					},
					"address":{
						"type": "string",
						"store": true
					},
					"createdOn":{
						"type": "date",
						"store": true
					},
					"severity":{
						"type": "string",
						"store": true
					},
					"message":{
						"type": "string",
						"store": true
					}
				}
			}
		}`

		err = esIndex.SetMapping(schema, piazza.JsonString(mapping))
		if err != nil {
			log.Printf("LoggerService.Init: %s", err.Error())
			return err
		}
	}

	service.logData.esIndex = esIndex

	service.origin = string(sys.Name)

	return nil
}

func (service *Service) newInternalErrorResponse(err error) *piazza.JsonResponse {
	return &piazza.JsonResponse{
		StatusCode: http.StatusInternalServerError,
		Message:    err.Error(),
		Origin:     service.origin,
	}
}

func (service *Service) GetRoot() *piazza.JsonResponse {
	resp := &piazza.JsonResponse{
		StatusCode: 200,
		Data:       "Hi. I'm pz-logger.",
		Origin:     service.origin,
	}

	err := resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func (service *Service) PostMessage(mssg *Message) *piazza.JsonResponse {
	err := mssg.Validate()
	if err != nil {
		return &piazza.JsonResponse{
			StatusCode: http.StatusBadRequest,
			Message:    err.Error(),
			Origin:     service.origin,
		}
	}

	service.logData.Lock()
	idStr := strconv.Itoa(service.logData.id)
	service.logData.id++
	service.logData.Unlock()

	_, err = service.logData.esIndex.PostData(schema, idStr, mssg)
	if err != nil {
		//log.Printf("POST failed (1): %#v %#v", err, indexResult)
		return service.newInternalErrorResponse(err)
	}

	/*	if !indexResult.Created {
		log.Printf("POST failed (2): %#v", *indexResult)
		return NewInternalErrorResponse(err)
	}*/

	service.stats.LoggerAdminStats.NumMessages++

	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       mssg,
		Origin:     service.origin,
	}

	err = resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func (service *Service) GetStats() *piazza.JsonResponse {
	service.logData.Lock()
	t := service.stats.LoggerAdminStats
	service.logData.Unlock()

	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       t,
		Origin:     service.origin,
	}

	err := resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func (service *Service) GetMessage(params *piazza.HttpQueryParams) *piazza.JsonResponse {
	var err error

	var formalPagination *piazza.JsonPagination
	{
		defaults := &piazza.JsonPagination{
			PerPage: 10,
			Page:    0,
			Order:   piazza.PaginationOrderDescending,
			SortBy:  "createdOn",
		}
		formalPagination, err = piazza.NewJsonPagination(params, defaults)
		if err != nil {
			return &piazza.JsonResponse{
				StatusCode: http.StatusBadRequest,
				Message:    err.Error(),
				Origin:     service.origin,
			}
		}
	}

	filterParams := service.parseFilterParams(params)

	//log.Printf("size %d, from %d, key %s, format %v",
	//	format.Size, format.From, format.Key, format.Order)

	//log.Printf("filterParams: %v\n", filterParams)

	var searchResult *elasticsearch.SearchResult

	if len(filterParams) == 0 {
		searchResult, err = service.logData.esIndex.FilterByMatchAll(schema, formalPagination)
	} else {
		var jsonString = service.createQueryDslAsString(formalPagination, filterParams)
		searchResult, err = service.logData.esIndex.SearchByJSON(schema, jsonString)
	}

	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	// TODO: unsafe truncation
	count := int(searchResult.TotalHits())
	matched := int(searchResult.NumberMatched())
	lines := make([]Message, count)

	i := 0
	for _, hit := range *searchResult.GetHits() {
		if hit == nil {
			log.Printf("null source hit")
			continue
		}
		src := *hit.Source
		//log.Printf("source hit: %s", string(src))

		tmp := &Message{}
		err = json.Unmarshal(src, tmp)
		if err != nil {
			log.Printf("UNABLE TO PARSE: %s", string(*hit.Source))
			return service.newInternalErrorResponse(err)
		}

		// still needed?
		err = tmp.Validate()
		if err != nil {
			log.Printf("UNABLE TO VALIDATE: %s", string(*hit.Source))
			continue
		}

		lines[i] = *tmp
		i++
	}

	bar := make([]interface{}, len(lines))

	for i, e := range lines {
		bar[i] = e
	}

	formalPagination.Count = matched
	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       bar,
		Pagination: formalPagination,
		Origin:     service.origin,
	}

	err = resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func (service *Service) parseFilterParams(params *piazza.HttpQueryParams) map[string]interface{} {

	var filterParams = map[string]interface{}{}

	before := params.Get("before")

	if before != "" {
		num, err := strconv.Atoi(before)
		if err == nil {
			filterParams["before"] = num
		}
	}

	after := params.Get("after")

	if after != "" {
		num, err := strconv.Atoi(after)
		if err == nil {
			filterParams["after"] = num
		}
	}

	svc := params.Get("service")

	if svc != "" {
		filterParams["service"] = svc
	}

	contains := params.Get("contains")

	if contains != "" {
		filterParams["contains"] = contains
	}

	return filterParams
}

func (service *Service) createQueryDslAsString(
	format *piazza.JsonPagination,
	params map[string]interface{},
) string {
	// fmt.Printf("%d\n", len(params))

	must := []map[string]interface{}{}

	if params["service"] != nil {
		must = append(must, map[string]interface{}{
			"match": map[string]interface{}{
				"service": params["service"],
			},
		})
	}

	if params["contains"] != nil {
		must = append(must, map[string]interface{}{
			"multi_match": map[string]interface{}{
				"query":  params["contains"],
				"fields": []string{"address", "message", "service", "severity"},
			},
		})
	}

	if params["after"] != nil || params["before"] != nil {
		rangeParams := map[string]int{}

		if params["after"] != nil {
			rangeParams["gte"] = params["after"].(int)
		}

		if params["before"] != nil {
			rangeParams["lte"] = params["before"].(int)
		}

		must = append(must, map[string]interface{}{
			"range": map[string]interface{}{
				"stamp": rangeParams,
			},
		})
	}

	dsl := map[string]interface{}{
		"query": map[string]interface{}{
			"filtered": map[string]interface{}{
				"query": map[string]interface{}{
					"bool": map[string]interface{}{
						"must": must,
					},
				},
			},
		},
		"size": format.PerPage,
		"from": format.PerPage * format.Page,
	}

	dsl["sort"] = map[string]string{
		format.SortBy: string(format.Order),
	}

	output, _ := json.Marshal(dsl)
	return string(output)
}
<commit_msg>only do it for bad requests<commit_after>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package logger

import (
	"encoding/json"
	_ "fmt"
	"log"
	"net/http"
	"strconv"
	"sync"
	"time"

	"github.com/venicegeo/pz-gocommon/elasticsearch"
	"github.com/venicegeo/pz-gocommon/gocommon"
)

type LockedAdminStats struct {
	sync.Mutex
	LoggerAdminStats
}

type LogData struct {
	sync.Mutex
	esIndex elasticsearch.IIndex
	id      int
}

const schema = "LogData5"

type Service struct {
	stats   LockedAdminStats
	logData LogData
	origin  string
}

func (service *Service) Init(sys *piazza.SystemConfig, esIndex elasticsearch.IIndex) error {
	var err error

	service.stats.CreatedOn = time.Now()
	/***
	err = esIndex.Delete()
	if err != nil {
		log.Fatal(err)
	}
	if esIndex.IndexExists() {
		log.Fatal("index still exists")
	}
	err = esIndex.Create()
	if err != nil {
		log.Fatal(err)
	}
	***/

	if !esIndex.IndexExists() {
		log.Printf("Creating index: %s", esIndex.IndexName())
		err = esIndex.Create("")
		if err != nil {
			log.Fatal(err)
		}
	}

	if !esIndex.TypeExists(schema) {
		log.Printf("Creating type: %s", schema)

		mapping :=
			`{
			"LogData5":{
				"properties":{
					"service":{
						"type": "string",
						"store": true
					},
					"address":{
						"type": "string",
						"store": true
					},
					"createdOn":{
						"type": "date",
						"store": true
					},
					"severity":{
						"type": "string",
						"store": true
					},
					"message":{
						"type": "string",
						"store": true
					}
				}
			}
		}`

		err = esIndex.SetMapping(schema, piazza.JsonString(mapping))
		if err != nil {
			log.Printf("LoggerService.Init: %s", err.Error())
			return err
		}
	}

	service.logData.esIndex = esIndex

	service.origin = string(sys.Name)

	return nil
}

func (service *Service) newInternalErrorResponse(err error) *piazza.JsonResponse {
	return &piazza.JsonResponse{
		StatusCode: http.StatusInternalServerError,
		Message:    err.Error(),
		Origin:     service.origin,
	}
}

func (service *Service) newBadRequestResponse(err error) *piazza.JsonResponse {
	return &piazza.JsonResponse{
		StatusCode: http.StatusBadRequest,
		Message:    err.Error(),
		Origin:     service.origin,
	}
}

func (service *Service) GetRoot() *piazza.JsonResponse {
	resp := &piazza.JsonResponse{
		StatusCode: 200,
		Data:       "Hi. I'm pz-logger.",
	}

	err := resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func (service *Service) PostMessage(mssg *Message) *piazza.JsonResponse {
	err := mssg.Validate()
	if err != nil {
		return newBadRequestResponse(err)
	}

	service.logData.Lock()
	idStr := strconv.Itoa(service.logData.id)
	service.logData.id++
	service.logData.Unlock()

	_, err = service.logData.esIndex.PostData(schema, idStr, mssg)
	if err != nil {
		//log.Printf("POST failed (1): %#v %#v", err, indexResult)
		return service.newInternalErrorResponse(err)
	}

	/*	if !indexResult.Created {
		log.Printf("POST failed (2): %#v", *indexResult)
		return NewInternalErrorResponse(err)
	}*/

	service.stats.LoggerAdminStats.NumMessages++

	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       mssg,
	}

	err = resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func (service *Service) GetStats() *piazza.JsonResponse {
	service.logData.Lock()
	t := service.stats.LoggerAdminStats
	service.logData.Unlock()

	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       t,
	}

	err := resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func (service *Service) GetMessage(params *piazza.HttpQueryParams) *piazza.JsonResponse {
	var err error

	var formalPagination *piazza.JsonPagination
	{
		defaults := &piazza.JsonPagination{
			PerPage: 10,
			Page:    0,
			Order:   piazza.PaginationOrderDescending,
			SortBy:  "createdOn",
		}
		formalPagination, err = piazza.NewJsonPagination(params, defaults)
		if err != nil {
			return newBadRequestResponse(err)
		}
	}

	filterParams := service.parseFilterParams(params)

	//log.Printf("size %d, from %d, key %s, format %v",
	//	format.Size, format.From, format.Key, format.Order)

	//log.Printf("filterParams: %v\n", filterParams)

	var searchResult *elasticsearch.SearchResult

	if len(filterParams) == 0 {
		searchResult, err = service.logData.esIndex.FilterByMatchAll(schema, formalPagination)
	} else {
		var jsonString = service.createQueryDslAsString(formalPagination, filterParams)
		searchResult, err = service.logData.esIndex.SearchByJSON(schema, jsonString)
	}

	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	// TODO: unsafe truncation
	count := int(searchResult.TotalHits())
	matched := int(searchResult.NumberMatched())
	lines := make([]Message, count)

	i := 0
	for _, hit := range *searchResult.GetHits() {
		if hit == nil {
			log.Printf("null source hit")
			continue
		}
		src := *hit.Source
		//log.Printf("source hit: %s", string(src))

		tmp := &Message{}
		err = json.Unmarshal(src, tmp)
		if err != nil {
			log.Printf("UNABLE TO PARSE: %s", string(*hit.Source))
			return service.newInternalErrorResponse(err)
		}

		// still needed?
		err = tmp.Validate()
		if err != nil {
			log.Printf("UNABLE TO VALIDATE: %s", string(*hit.Source))
			continue
		}

		lines[i] = *tmp
		i++
	}

	bar := make([]interface{}, len(lines))

	for i, e := range lines {
		bar[i] = e
	}

	formalPagination.Count = matched
	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       bar,
		Pagination: formalPagination,
	}

	err = resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func (service *Service) parseFilterParams(params *piazza.HttpQueryParams) map[string]interface{} {

	var filterParams = map[string]interface{}{}

	before := params.Get("before")

	if before != "" {
		num, err := strconv.Atoi(before)
		if err == nil {
			filterParams["before"] = num
		}
	}

	after := params.Get("after")

	if after != "" {
		num, err := strconv.Atoi(after)
		if err == nil {
			filterParams["after"] = num
		}
	}

	svc := params.Get("service")

	if svc != "" {
		filterParams["service"] = svc
	}

	contains := params.Get("contains")

	if contains != "" {
		filterParams["contains"] = contains
	}

	return filterParams
}

func (service *Service) createQueryDslAsString(
	format *piazza.JsonPagination,
	params map[string]interface{},
) string {
	// fmt.Printf("%d\n", len(params))

	must := []map[string]interface{}{}

	if params["service"] != nil {
		must = append(must, map[string]interface{}{
			"match": map[string]interface{}{
				"service": params["service"],
			},
		})
	}

	if params["contains"] != nil {
		must = append(must, map[string]interface{}{
			"multi_match": map[string]interface{}{
				"query":  params["contains"],
				"fields": []string{"address", "message", "service", "severity"},
			},
		})
	}

	if params["after"] != nil || params["before"] != nil {
		rangeParams := map[string]int{}

		if params["after"] != nil {
			rangeParams["gte"] = params["after"].(int)
		}

		if params["before"] != nil {
			rangeParams["lte"] = params["before"].(int)
		}

		must = append(must, map[string]interface{}{
			"range": map[string]interface{}{
				"stamp": rangeParams,
			},
		})
	}

	dsl := map[string]interface{}{
		"query": map[string]interface{}{
			"filtered": map[string]interface{}{
				"query": map[string]interface{}{
					"bool": map[string]interface{}{
						"must": must,
					},
				},
			},
		},
		"size": format.PerPage,
		"from": format.PerPage * format.Page,
	}

	dsl["sort"] = map[string]string{
		format.SortBy: string(format.Order),
	}

	output, _ := json.Marshal(dsl)
	return string(output)
}
<|endoftext|>


<commit_before>package logger

import (
	"context"
	"fmt"
	"os"
	"path"
	"runtime"
	"sort"
	"strings"
	"sync"
	"time"

	dlog "github.com/micro/go-micro/v2/debug/log"
)

func init() {
	lvl, err := GetLevel(os.Getenv("MICRO_LOG_LEVEL"))
	if err != nil {
		lvl = InfoLevel
	}

	DefaultLogger = NewHelper(NewLogger(WithLevel(lvl)))
}

type defaultLogger struct {
	sync.RWMutex
	opts Options
}

// Init(opts...) should only overwrite provided options
func (l *defaultLogger) Init(opts ...Option) error {
	for _, o := range opts {
		o(&l.opts)
	}
	return nil
}

func (l *defaultLogger) String() string {
	return "default"
}

func (l *defaultLogger) Fields(fields map[string]interface{}) Logger {
	l.Lock()
	l.opts.Fields = copyFields(fields)
	l.Unlock()
	return l
}

func copyFields(src map[string]interface{}) map[string]interface{} {
	dst := make(map[string]interface{}, len(src))
	for k, v := range src {
		dst[k] = v
	}
	return dst
}

var sourceControlSites = []string{"github.com"}

func logCallerfilePath(filepath string) string {
	for _, v := range sourceControlSites {
		if strings.Contains(filepath, v) {
			parts := strings.Split(filepath, v)
			if len(parts) > 0 {
				return path.Join(v, parts[1])
			}
		}
	}
	return filepath
}

func (l *defaultLogger) Log(level Level, v ...interface{}) {
	// TODO decide does we need to write message if log level not used?
	if !l.opts.Level.Enabled(level) {
		return
	}

	l.RLock()
	fields := copyFields(l.opts.Fields)
	l.RUnlock()

	fields["level"] = level.String()

	if _, file, line, ok := runtime.Caller(l.opts.CallerSkipCount); ok {
		fields["caller"] = fmt.Sprintf("%s:%d", logCallerfilePath(file), line)
	}

	rec := dlog.Record{
		Timestamp: time.Now(),
		Message:   fmt.Sprint(v...),
		Metadata:  make(map[string]string, len(fields)),
	}

	keys := make([]string, 0, len(fields))
	for k, v := range fields {
		keys = append(keys, k)
		rec.Metadata[k] = fmt.Sprintf("%v", v)
	}

	sort.Strings(keys)
	metadata := ""

	for _, k := range keys {
		metadata += fmt.Sprintf(" %s=%v", k, fields[k])
	}

	dlog.DefaultLog.Write(rec)

	t := rec.Timestamp.Format("2006-01-02 15:04:05")
	fmt.Printf("%s %s %v\n", t, metadata, rec.Message)
}

func (l *defaultLogger) Logf(level Level, format string, v ...interface{}) {
	//	 TODO decide does we need to write message if log level not used?
	if level < l.opts.Level {
		return
	}

	l.RLock()
	fields := copyFields(l.opts.Fields)
	l.RUnlock()

	fields["level"] = level.String()

	if _, file, line, ok := runtime.Caller(l.opts.CallerSkipCount); ok {
		fields["caller"] = fmt.Sprintf("%s:%d", logCallerfilePath(file), line)
	}

	rec := dlog.Record{
		Timestamp: time.Now(),
		Message:   fmt.Sprintf(format, v...),
		Metadata:  make(map[string]string, len(fields)),
	}

	keys := make([]string, 0, len(fields))
	for k, v := range fields {
		keys = append(keys, k)
		rec.Metadata[k] = fmt.Sprintf("%v", v)
	}

	sort.Strings(keys)
	metadata := ""

	for _, k := range keys {
		metadata += fmt.Sprintf(" %s=%v", k, fields[k])
	}

	dlog.DefaultLog.Write(rec)

	t := rec.Timestamp.Format("2006-01-02 15:04:05")
	fmt.Printf("%s %s %v\n", t, metadata, rec.Message)
}

func (n *defaultLogger) Options() Options {
	// not guard against options Context values
	n.RLock()
	opts := n.opts
	opts.Fields = copyFields(n.opts.Fields)
	n.RUnlock()
	return opts
}

// NewLogger builds a new logger based on options
func NewLogger(opts ...Option) Logger {
	// Default options
	options := Options{
		Level:           InfoLevel,
		Fields:          make(map[string]interface{}),
		Out:             os.Stderr,
		CallerSkipCount: 2,
		Context:         context.Background(),
	}

	l := &defaultLogger{opts: options}
	if err := l.Init(opts...); err != nil {
		l.Log(FatalLevel, err)
	}

	return l
}
<commit_msg>Display only logging file name as opposed to path in logs (#1580)<commit_after>package logger

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
	"sync"
	"time"

	dlog "github.com/micro/go-micro/v2/debug/log"
)

func init() {
	lvl, err := GetLevel(os.Getenv("MICRO_LOG_LEVEL"))
	if err != nil {
		lvl = InfoLevel
	}

	DefaultLogger = NewHelper(NewLogger(WithLevel(lvl)))
}

type defaultLogger struct {
	sync.RWMutex
	opts Options
}

// Init(opts...) should only overwrite provided options
func (l *defaultLogger) Init(opts ...Option) error {
	for _, o := range opts {
		o(&l.opts)
	}
	return nil
}

func (l *defaultLogger) String() string {
	return "default"
}

func (l *defaultLogger) Fields(fields map[string]interface{}) Logger {
	l.Lock()
	l.opts.Fields = copyFields(fields)
	l.Unlock()
	return l
}

func copyFields(src map[string]interface{}) map[string]interface{} {
	dst := make(map[string]interface{}, len(src))
	for k, v := range src {
		dst[k] = v
	}
	return dst
}

func logCallerfilePath(loggingFilePath string) string {
	parts := strings.Split(loggingFilePath, string(filepath.Separator))
	return parts[len(parts)-1]
}

func (l *defaultLogger) Log(level Level, v ...interface{}) {
	// TODO decide does we need to write message if log level not used?
	if !l.opts.Level.Enabled(level) {
		return
	}

	l.RLock()
	fields := copyFields(l.opts.Fields)
	l.RUnlock()

	fields["level"] = level.String()

	if _, file, line, ok := runtime.Caller(l.opts.CallerSkipCount); ok {
		fields["file"] = fmt.Sprintf("%s:%d", logCallerfilePath(file), line)
	}

	rec := dlog.Record{
		Timestamp: time.Now(),
		Message:   fmt.Sprint(v...),
		Metadata:  make(map[string]string, len(fields)),
	}

	keys := make([]string, 0, len(fields))
	for k, v := range fields {
		keys = append(keys, k)
		rec.Metadata[k] = fmt.Sprintf("%v", v)
	}

	sort.Strings(keys)
	metadata := ""

	for _, k := range keys {
		metadata += fmt.Sprintf(" %s=%v", k, fields[k])
	}

	dlog.DefaultLog.Write(rec)

	t := rec.Timestamp.Format("2006-01-02 15:04:05")
	fmt.Printf("%s %s %v\n", t, metadata, rec.Message)
}

func (l *defaultLogger) Logf(level Level, format string, v ...interface{}) {
	//	 TODO decide does we need to write message if log level not used?
	if level < l.opts.Level {
		return
	}

	l.RLock()
	fields := copyFields(l.opts.Fields)
	l.RUnlock()

	fields["level"] = level.String()

	if _, file, line, ok := runtime.Caller(l.opts.CallerSkipCount); ok {
		fields["file"] = fmt.Sprintf("%s:%d", logCallerfilePath(file), line)
	}

	rec := dlog.Record{
		Timestamp: time.Now(),
		Message:   fmt.Sprintf(format, v...),
		Metadata:  make(map[string]string, len(fields)),
	}

	keys := make([]string, 0, len(fields))
	for k, v := range fields {
		keys = append(keys, k)
		rec.Metadata[k] = fmt.Sprintf("%v", v)
	}

	sort.Strings(keys)
	metadata := ""

	for _, k := range keys {
		metadata += fmt.Sprintf(" %s=%v", k, fields[k])
	}

	dlog.DefaultLog.Write(rec)

	t := rec.Timestamp.Format("2006-01-02 15:04:05")
	fmt.Printf("%s %s %v\n", t, metadata, rec.Message)
}

func (n *defaultLogger) Options() Options {
	// not guard against options Context values
	n.RLock()
	opts := n.opts
	opts.Fields = copyFields(n.opts.Fields)
	n.RUnlock()
	return opts
}

// NewLogger builds a new logger based on options
func NewLogger(opts ...Option) Logger {
	// Default options
	options := Options{
		Level:           InfoLevel,
		Fields:          make(map[string]interface{}),
		Out:             os.Stderr,
		CallerSkipCount: 2,
		Context:         context.Background(),
	}

	l := &defaultLogger{opts: options}
	if err := l.Init(opts...); err != nil {
		l.Log(FatalLevel, err)
	}

	return l
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package logging

import (
	"encoding/json"
	"errors"
	"fmt"

	"go.uber.org/zap"
	"go.uber.org/zap/zapcore"
	corev1 "k8s.io/api/core/v1"

	"github.com/knative/pkg/logging/logkey"
)

// NewLogger creates a logger with the supplied configuration.
// In addition to the logger, it returns AtomicLevel that can
// be used to change the logging level at runtime.
// If configuration is empty, a fallback configuration is used.
// If configuration cannot be used to instantiate a logger,
// the same fallback configuration is used.
func NewLogger(configJSON string, levelOverride string) (*zap.SugaredLogger, zap.AtomicLevel) {
	logger, atomicLevel, err := newLoggerFromConfig(configJSON, levelOverride)
	if err == nil {
		return logger.Sugar(), atomicLevel
	}

	loggingCfg := zap.NewProductionConfig()
	if len(levelOverride) > 0 {
		if level, err := levelFromString(levelOverride); err == nil {
			loggingCfg.Level = zap.NewAtomicLevelAt(*level)
		}
	}

	logger, err2 := loggingCfg.Build()
	if err2 != nil {
		panic(err2)
	}

	logger.Error("Failed to parse the logging config. Falling back to default logger.",
		zap.Error(err), zap.String(logkey.JSONConfig, configJSON))
	return logger.Sugar(), loggingCfg.Level
}

// NewLoggerFromConfig creates a logger using the provided Config
func NewLoggerFromConfig(config *Config, name string) (*zap.SugaredLogger, zap.AtomicLevel) {
	logger, level := NewLogger(config.LoggingConfig, config.LoggingLevel[name].String())
	return logger.Named(name), level
}

func newLoggerFromConfig(configJSON string, levelOverride string) (*zap.Logger, zap.AtomicLevel, error) {
	if len(configJSON) == 0 {
		return nil, zap.AtomicLevel{}, errors.New("empty logging configuration")
	}

	var loggingCfg zap.Config
	if err := json.Unmarshal([]byte(configJSON), &loggingCfg); err != nil {
		return nil, zap.AtomicLevel{}, err
	}

	if len(levelOverride) > 0 {
		if level, err := levelFromString(levelOverride); err == nil {
			loggingCfg.Level = zap.NewAtomicLevelAt(*level)
		}
	}

	logger, err := loggingCfg.Build()
	if err != nil {
		return nil, zap.AtomicLevel{}, err
	}

	logger.Info("Successfully created the logger.", zap.String(logkey.JSONConfig, configJSON))
	logger.Sugar().Infof("Logging level set to %v", loggingCfg.Level)
	return logger, loggingCfg.Level, nil
}

// Config contains the configuration defined in the logging ConfigMap.
// +k8s:deepcopy-gen=true
type Config struct {
	LoggingConfig string
	LoggingLevel  map[string]zapcore.Level
}

// NewConfigFromMap creates a LoggingConfig from the supplied map,
// expecting the given list of components.
func NewConfigFromMap(data map[string]string, components ...string) (*Config, error) {
	lc := &Config{}
	if zlc, ok := data["zap-logger-config"]; ok {
		lc.LoggingConfig = zlc
	}
	lc.LoggingLevel = make(map[string]zapcore.Level)
	for _, component := range components {
		if ll, ok := data["loglevel."+component]; ok {
			if len(ll) > 0 {
				level, err := levelFromString(ll)
				if err != nil {
					return nil, err
				}
				lc.LoggingLevel[component] = *level
			}
		}
	}
	return lc, nil
}

// NewConfigFromConfigMap creates a LoggingConfig from the supplied ConfigMap,
// expecting the given list of components.
func NewConfigFromConfigMap(configMap *corev1.ConfigMap, components ...string) (*Config, error) {
	return NewConfigFromMap(configMap.Data, components...)
}

func levelFromString(level string) (*zapcore.Level, error) {
	var zapLevel zapcore.Level
	if err := zapLevel.UnmarshalText([]byte(level)); err != nil {
		return nil, fmt.Errorf("invalid logging level: %v", level)
	}
	return &zapLevel, nil
}

// UpdateLevelFromConfigMap returns a helper func that can be used to update the logging level
// when a config map is updated
func UpdateLevelFromConfigMap(logger *zap.SugaredLogger, atomicLevel zap.AtomicLevel,
	levelKey string, components ...string) func(configMap *corev1.ConfigMap) {
	return func(configMap *corev1.ConfigMap) {
		loggingConfig, err := NewConfigFromConfigMap(configMap, components...)
		if err != nil {
			logger.Error("Failed to parse the logging configmap. Previous config map will be used.", zap.Error(err))
			return
		}

		if level, ok := loggingConfig.LoggingLevel[levelKey]; ok {
			if atomicLevel.Level() != level {
				logger.Infof("Updating logging level for %v from %v to %v.", levelKey, atomicLevel.Level(), level)
				atomicLevel.SetLevel(level)
			}
		}
	}
}
<commit_msg>Use fallback named logger (#30)<commit_after>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package logging

import (
	"encoding/json"
	"errors"
	"fmt"

	"go.uber.org/zap"
	"go.uber.org/zap/zapcore"
	corev1 "k8s.io/api/core/v1"

	"github.com/knative/pkg/logging/logkey"
)

// NewLogger creates a logger with the supplied configuration.
// In addition to the logger, it returns AtomicLevel that can
// be used to change the logging level at runtime.
// If configuration is empty, a fallback configuration is used.
// If configuration cannot be used to instantiate a logger,
// the same fallback configuration is used.
func NewLogger(configJSON string, levelOverride string) (*zap.SugaredLogger, zap.AtomicLevel) {
	logger, atomicLevel, err := newLoggerFromConfig(configJSON, levelOverride)
	if err == nil {
		return logger.Sugar(), atomicLevel
	}

	loggingCfg := zap.NewProductionConfig()
	if len(levelOverride) > 0 {
		if level, err := levelFromString(levelOverride); err == nil {
			loggingCfg.Level = zap.NewAtomicLevelAt(*level)
		}
	}

	logger, err2 := loggingCfg.Build()
	if err2 != nil {
		panic(err2)
	}
	return logger.Named("fallback-logger").Sugar(), loggingCfg.Level
}

// NewLoggerFromConfig creates a logger using the provided Config
func NewLoggerFromConfig(config *Config, name string) (*zap.SugaredLogger, zap.AtomicLevel) {
	logger, level := NewLogger(config.LoggingConfig, config.LoggingLevel[name].String())
	return logger.Named(name), level
}

func newLoggerFromConfig(configJSON string, levelOverride string) (*zap.Logger, zap.AtomicLevel, error) {
	if len(configJSON) == 0 {
		return nil, zap.AtomicLevel{}, errors.New("empty logging configuration")
	}

	var loggingCfg zap.Config
	if err := json.Unmarshal([]byte(configJSON), &loggingCfg); err != nil {
		return nil, zap.AtomicLevel{}, err
	}

	if len(levelOverride) > 0 {
		if level, err := levelFromString(levelOverride); err == nil {
			loggingCfg.Level = zap.NewAtomicLevelAt(*level)
		}
	}

	logger, err := loggingCfg.Build()
	if err != nil {
		return nil, zap.AtomicLevel{}, err
	}

	logger.Info("Successfully created the logger.", zap.String(logkey.JSONConfig, configJSON))
	logger.Sugar().Infof("Logging level set to %v", loggingCfg.Level)
	return logger, loggingCfg.Level, nil
}

// Config contains the configuration defined in the logging ConfigMap.
// +k8s:deepcopy-gen=true
type Config struct {
	LoggingConfig string
	LoggingLevel  map[string]zapcore.Level
}

// NewConfigFromMap creates a LoggingConfig from the supplied map,
// expecting the given list of components.
func NewConfigFromMap(data map[string]string, components ...string) (*Config, error) {
	lc := &Config{}
	if zlc, ok := data["zap-logger-config"]; ok {
		lc.LoggingConfig = zlc
	}
	lc.LoggingLevel = make(map[string]zapcore.Level)
	for _, component := range components {
		if ll, ok := data["loglevel."+component]; ok {
			if len(ll) > 0 {
				level, err := levelFromString(ll)
				if err != nil {
					return nil, err
				}
				lc.LoggingLevel[component] = *level
			}
		}
	}
	return lc, nil
}

// NewConfigFromConfigMap creates a LoggingConfig from the supplied ConfigMap,
// expecting the given list of components.
func NewConfigFromConfigMap(configMap *corev1.ConfigMap, components ...string) (*Config, error) {
	return NewConfigFromMap(configMap.Data, components...)
}

func levelFromString(level string) (*zapcore.Level, error) {
	var zapLevel zapcore.Level
	if err := zapLevel.UnmarshalText([]byte(level)); err != nil {
		return nil, fmt.Errorf("invalid logging level: %v", level)
	}
	return &zapLevel, nil
}

// UpdateLevelFromConfigMap returns a helper func that can be used to update the logging level
// when a config map is updated
func UpdateLevelFromConfigMap(logger *zap.SugaredLogger, atomicLevel zap.AtomicLevel,
	levelKey string, components ...string) func(configMap *corev1.ConfigMap) {
	return func(configMap *corev1.ConfigMap) {
		loggingConfig, err := NewConfigFromConfigMap(configMap, components...)
		if err != nil {
			logger.Error("Failed to parse the logging configmap. Previous config map will be used.", zap.Error(err))
			return
		}

		if level, ok := loggingConfig.LoggingLevel[levelKey]; ok {
			if atomicLevel.Level() != level {
				logger.Infof("Updating logging level for %v from %v to %v.", levelKey, atomicLevel.Level(), level)
				atomicLevel.SetLevel(level)
			}
		}
	}
}
<|endoftext|>


<commit_before>/*
 Copyright 2016 Padduck, LLC

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 	http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package logging

import (
	"fmt"
	"strings"
)

type level struct {
	scale byte;
	display string;
}

var (
	DEBUG level = level{scale: 7, display: "DEBUG"};
	INFO level = level{scale: 31, display: "INFO"};
	WARN level = level{scale: 63, display: "WARN"};
	ERROR level = level{scale: 127, display: "ERROR"};
	CRITICAL level = level{scale: 255, display: "CRITICAL"};
	loggingLevel = INFO;
)

func Info(msg string, data ...interface{}) {
	log(INFO, msg, data);
}

func Warn(msg string, data ...interface{}) {
	log(WARN, msg, data);
}

func Debug(msg string, data ...interface{}) {
	log(DEBUG, msg, data);
}

func Error(msg string, data ...interface{}) {
	log(ERROR, msg, data);
}

func Critical(msg string, data ...interface{}) {
	log(CRITICAL, msg, data);
}

func Infof(msg string, data ...interface{}) {
	logf(INFO, msg, data);
}

func Warnf(msg string, data ...interface{}) {
	logf(WARN, msg, data);
}

func Debugf(msg string, data ...interface{}) {
	logf(DEBUG, msg, data);
}

func Errorf(msg string, data ...interface{}) {
	logf(ERROR, msg, data);
}

func Criticalf(msg string, data ...interface{}) {
	logf(CRITICAL, msg, data);
}

func SetLevel(lvl level) {
	loggingLevel = lvl;
}

func SetLevelByString(lvl string) {
	switch (strings.ToUpper(lvl)) {
	case "DEBUG":
		SetLevel(DEBUG);
	case "INFO":
		SetLevel(INFO);
	case "WARN":
		SetLevel(WARN);
	case "ERROR":
		SetLevel(ERROR);
	case "CRITICAL":
		SetLevel(CRITICAL);
	}
}

func log(lvl level, msg string, data ...interface{}) {
	if(lvl.scale < loggingLevel.scale) {
		return;
	}
	var dataLength = len(data[0].([]interface{}));
	if(data == nil || dataLength == 0) {
		fmt.Printf("[%s] %s\n", lvl.display, msg);
	} else {
		cast := make([]interface{}, 3);
		cast[0] = lvl.display;
		cast[1] = msg;
		if(dataLength == 1) {
			cast[2] = data[0].([]interface{})[0];
		} else {
			cast[2] = data[0].([]interface{});
		}
		fmt.Printf("[%s] %s\n%v\n", cast...);
	}
}

func logf(lvl level, msg string, data ...interface{}) {
	if(lvl.scale < loggingLevel.scale) {
		return;
	}
	if(data == nil || len(data[0].([]interface{})) == 0) {
		fmt.Printf("[%s] %s\n", lvl.display, msg);
	} else {
		fmt.Printf("[%s] %s\n", lvl.display, fmt.Sprintf(msg, data[0].([]interface{})...));
	}
}<commit_msg>Add file logging and timestamps<commit_after>/*
 Copyright 2016 Padduck, LLC

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 	http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package logging

import (
	"fmt"
	"strings"
	"os"
	"time"
	"path"
)

type level struct {
	scale   byte;
	display string;
}

var (
	DEBUG level = level{scale: 7, display: "DEBUG"};
	INFO level = level{scale: 31, display: "INFO"};
	WARN level = level{scale: 63, display: "WARN"};
	ERROR level = level{scale: 127, display: "ERROR"};
	CRITICAL level = level{scale: 255, display: "CRITICAL"};
	loggingLevel = INFO;
	logFile *os.File
)

const (
	logFileFolder = "logs"
	logFileName = "2006-01-02T15-04-05.log"
	formatNoData = "[%s] [%s] %s\n"
	formatWithData = "[%s] [%s] %s\n%v\n"
)

func init() {
	var err error;
	os.Mkdir(logFileFolder, os.ModeDir);
	logFile, err = os.OpenFile(path.Join(logFileFolder, time.Now().Format(logFileName)), os.O_WRONLY | os.O_CREATE, os.ModeType)
	if (err != nil) {
		Critical("Could not create log file", err);
	}
}

func Info(msg string, data ...interface{}) {
	log(INFO, msg, data);
}

func Warn(msg string, data ...interface{}) {
	log(WARN, msg, data);
}

func Debug(msg string, data ...interface{}) {
	log(DEBUG, msg, data);
}

func Error(msg string, data ...interface{}) {
	log(ERROR, msg, data);
}

func Critical(msg string, data ...interface{}) {
	log(CRITICAL, msg, data);
}

func Infof(msg string, data ...interface{}) {
	logf(INFO, msg, data);
}

func Warnf(msg string, data ...interface{}) {
	logf(WARN, msg, data);
}

func Debugf(msg string, data ...interface{}) {
	logf(DEBUG, msg, data);
}

func Errorf(msg string, data ...interface{}) {
	logf(ERROR, msg, data);
}

func Criticalf(msg string, data ...interface{}) {
	logf(CRITICAL, msg, data);
}

func SetLevel(lvl level) {
	loggingLevel = lvl;
}

func SetLevelByString(lvl string) {
	switch (strings.ToUpper(lvl)) {
	case "DEBUG":
		SetLevel(DEBUG);
	case "INFO":
		SetLevel(INFO);
	case "WARN":
		SetLevel(WARN);
	case "ERROR":
		SetLevel(ERROR);
	case "CRITICAL":
		SetLevel(CRITICAL);
	}
}

func log(lvl level, msg string, data ...interface{}) {
	if (lvl.scale < loggingLevel.scale) {
		return;
	}
	var dataLength = len(data[0].([]interface{}));
	if (data == nil || dataLength == 0) {
		var output = fmt.Sprintf(formatNoData, getTimestamp(), lvl.display, msg);
		fmt.Print(output);
		logFile.WriteString(output);
	} else {
		cast := make([]interface{}, 4);
		cast[0] = getTimestamp();
		cast[1] = lvl.display;
		cast[2] = msg;
		if (dataLength == 1) {
			cast[3] = data[0].([]interface{})[0];
		} else {
			cast[3] = data[0].([]interface{});
		}
		var output = fmt.Sprintf(formatWithData, cast...);
		fmt.Print(output);
		logFile.WriteString(output);
	}
}

func logf(lvl level, msg string, data ...interface{}) {
	if (lvl.scale < loggingLevel.scale) {
		return;
	}
	if (data == nil || len(data[0].([]interface{})) == 0) {
		var output = fmt.Sprintf(formatNoData, getTimestamp(), lvl.display, msg);
		fmt.Print(output);
		logFile.WriteString(output);
	} else {
		var output = fmt.Sprintf(formatNoData, getTimestamp(), lvl.display, fmt.Sprintf(msg, data[0].([]interface{})...));
		fmt.Print(output);
		logFile.WriteString(output);
	}
}

func getTimestamp() string {
	return time.Now().Format("15:04:05");
}<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package etcdserver

import (
	"crypto/sha1"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"log"
	"net/url"
	"path"
	"sort"
	"strings"
	"sync"

	"github.com/coreos/etcd/pkg/flags"
	"github.com/coreos/etcd/pkg/netutil"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/raft/raftpb"
	"github.com/coreos/etcd/store"
)

const (
	raftAttributesSuffix = "raftAttributes"
	attributesSuffix     = "attributes"
)

type ClusterInfo interface {
	// ID returns the cluster ID
	ID() types.ID
	// ClientURLs returns an aggregate set of all URLs on which this
	// cluster is listening for client requests
	ClientURLs() []string
	// Members returns a slice of members sorted by their ID
	Members() []*Member
	// Member retrieves a particular member based on ID, or nil if the
	// member does not exist in the cluster
	Member(id types.ID) *Member
	// IsIDRemoved checks whether the given ID has been removed from this
	// cluster at some point in the past
	IsIDRemoved(id types.ID) bool
}

// Cluster is a list of Members that belong to the same raft cluster
type Cluster struct {
	id      types.ID
	token   string
	members map[types.ID]*Member
	// removed contains the ids of removed members in the cluster.
	// removed id cannot be reused.
	removed map[types.ID]bool
	store   store.Store
	sync.Mutex
}

// NewClusterFromString returns a Cluster instantiated from the given cluster token
// and cluster string, by parsing members from a set of discovery-formatted
// names-to-IPs, like:
// mach0=http://1.1.1.1,mach0=http://2.2.2.2,mach1=http://3.3.3.3,mach2=http://4.4.4.4
func NewClusterFromString(token string, cluster string) (*Cluster, error) {
	c := newCluster(token)

	v, err := url.ParseQuery(strings.Replace(cluster, ",", "&", -1))
	if err != nil {
		return nil, err
	}
	for name, urls := range v {
		if len(urls) == 0 || urls[0] == "" {
			return nil, fmt.Errorf("Empty URL given for %q", name)
		}
		purls := &flags.URLsValue{}
		if err := purls.Set(strings.Join(urls, ",")); err != nil {
			return nil, err
		}
		m := NewMember(name, types.URLs(*purls), c.token, nil)
		if _, ok := c.members[m.ID]; ok {
			return nil, fmt.Errorf("Member exists with identical ID %v", m)
		}
		c.members[m.ID] = m
	}
	c.genID()
	return c, nil
}

func NewClusterFromStore(token string, st store.Store) *Cluster {
	c := newCluster(token)
	c.store = st
	c.members, c.removed = membersFromStore(c.store)
	return c
}

func NewClusterFromMembers(token string, id types.ID, membs []*Member) *Cluster {
	c := newCluster(token)
	c.id = id
	for _, m := range membs {
		c.members[m.ID] = m
	}
	return c
}

func newCluster(token string) *Cluster {
	return &Cluster{
		token:   token,
		members: make(map[types.ID]*Member),
		removed: make(map[types.ID]bool),
	}
}

func (c *Cluster) ID() types.ID { return c.id }

func (c *Cluster) Members() []*Member {
	c.Lock()
	defer c.Unlock()
	var sms SortableMemberSlice
	for _, m := range c.members {
		sms = append(sms, m.Clone())
	}
	sort.Sort(sms)
	return []*Member(sms)
}

func (c *Cluster) Member(id types.ID) *Member {
	c.Lock()
	defer c.Unlock()
	return c.members[id].Clone()
}

// MemberByName returns a Member with the given name if exists.
// If more than one member has the given name, it will panic.
func (c *Cluster) MemberByName(name string) *Member {
	c.Lock()
	defer c.Unlock()
	var memb *Member
	for _, m := range c.members {
		if m.Name == name {
			if memb != nil {
				log.Panicf("two members with the given name %q exist", name)
			}
			memb = m
		}
	}
	return memb.Clone()
}

func (c *Cluster) MemberIDs() []types.ID {
	c.Lock()
	defer c.Unlock()
	var ids []types.ID
	for _, m := range c.members {
		ids = append(ids, m.ID)
	}
	sort.Sort(types.IDSlice(ids))
	return ids
}

func (c *Cluster) IsIDRemoved(id types.ID) bool {
	c.Lock()
	defer c.Unlock()
	return c.removed[id]
}

// PeerURLs returns a list of all peer addresses.
// The returned list is sorted in ascending lexicographical order.
func (c *Cluster) PeerURLs() []string {
	c.Lock()
	defer c.Unlock()
	urls := make([]string, 0)
	for _, p := range c.members {
		for _, addr := range p.PeerURLs {
			urls = append(urls, addr)
		}
	}
	sort.Strings(urls)
	return urls
}

// ClientURLs returns a list of all client addresses.
// The returned list is sorted in ascending lexicographical order.
func (c *Cluster) ClientURLs() []string {
	c.Lock()
	defer c.Unlock()
	urls := make([]string, 0)
	for _, p := range c.members {
		for _, url := range p.ClientURLs {
			urls = append(urls, url)
		}
	}
	sort.Strings(urls)
	return urls
}

func (c *Cluster) String() string {
	c.Lock()
	defer c.Unlock()
	sl := []string{}
	for _, m := range c.members {
		for _, u := range m.PeerURLs {
			sl = append(sl, fmt.Sprintf("%s=%s", m.Name, u))
		}
	}
	sort.Strings(sl)
	return strings.Join(sl, ",")
}

func (c *Cluster) genID() {
	mIDs := c.MemberIDs()
	b := make([]byte, 8*len(mIDs))
	for i, id := range mIDs {
		binary.BigEndian.PutUint64(b[8*i:], uint64(id))
	}
	hash := sha1.Sum(b)
	c.id = types.ID(binary.BigEndian.Uint64(hash[:8]))
}

func (c *Cluster) SetID(id types.ID) { c.id = id }

func (c *Cluster) SetStore(st store.Store) { c.store = st }

func (c *Cluster) Recover() {
	c.members, c.removed = membersFromStore(c.store)
}

// ValidateConfigurationChange takes a proposed ConfChange and
// ensures that it is still valid.
func (c *Cluster) ValidateConfigurationChange(cc raftpb.ConfChange) error {
	members, removed := membersFromStore(c.store)
	id := types.ID(cc.NodeID)
	if removed[id] {
		return ErrIDRemoved
	}
	switch cc.Type {
	case raftpb.ConfChangeAddNode:
		if members[id] != nil {
			return ErrIDExists
		}
		urls := make(map[string]bool)
		for _, m := range members {
			for _, u := range m.PeerURLs {
				urls[u] = true
			}
		}
		m := new(Member)
		if err := json.Unmarshal(cc.Context, m); err != nil {
			log.Panicf("unmarshal member should never fail: %v", err)
		}
		for _, u := range m.PeerURLs {
			if urls[u] {
				return ErrPeerURLexists
			}
		}
	case raftpb.ConfChangeRemoveNode:
		if members[id] == nil {
			return ErrIDNotFound
		}
	case raftpb.ConfChangeUpdateNode:
		if members[id] == nil {
			return ErrIDNotFound
		}
		urls := make(map[string]bool)
		for _, m := range members {
			if m.ID == id {
				continue
			}
			for _, u := range m.PeerURLs {
				urls[u] = true
			}
		}
		m := new(Member)
		if err := json.Unmarshal(cc.Context, m); err != nil {
			log.Panicf("unmarshal member should never fail: %v", err)
		}
		for _, u := range m.PeerURLs {
			if urls[u] {
				return ErrPeerURLexists
			}
		}
	default:
		log.Panicf("ConfChange type should be either AddNode, RemoveNode or UpdateNode")
	}
	return nil
}

// AddMember adds a new Member into the cluster, and saves the given member's
// raftAttributes into the store. The given member should have empty attributes.
// A Member with a matching id must not exist.
func (c *Cluster) AddMember(m *Member) {
	c.Lock()
	defer c.Unlock()
	b, err := json.Marshal(m.RaftAttributes)
	if err != nil {
		log.Panicf("marshal raftAttributes should never fail: %v", err)
	}
	p := path.Join(memberStoreKey(m.ID), raftAttributesSuffix)
	if _, err := c.store.Create(p, false, string(b), false, store.Permanent); err != nil {
		log.Panicf("create raftAttributes should never fail: %v", err)
	}
	c.members[m.ID] = m
}

// RemoveMember removes a member from the store.
// The given id MUST exist, or the function panics.
func (c *Cluster) RemoveMember(id types.ID) {
	c.Lock()
	defer c.Unlock()
	if _, err := c.store.Delete(memberStoreKey(id), true, true); err != nil {
		log.Panicf("delete member should never fail: %v", err)
	}
	delete(c.members, id)
	if _, err := c.store.Create(removedMemberStoreKey(id), false, "", false, store.Permanent); err != nil {
		log.Panicf("create removedMember should never fail: %v", err)
	}
	c.removed[id] = true
}

func (c *Cluster) UpdateAttributes(id types.ID, attr Attributes) {
	c.Lock()
	defer c.Unlock()
	c.members[id].Attributes = attr
	// TODO: update store in this function
}

func (c *Cluster) UpdateRaftAttributes(id types.ID, raftAttr RaftAttributes) {
	c.Lock()
	defer c.Unlock()
	b, err := json.Marshal(raftAttr)
	if err != nil {
		log.Panicf("marshal raftAttributes should never fail: %v", err)
	}
	p := path.Join(memberStoreKey(id), raftAttributesSuffix)
	if _, err := c.store.Update(p, string(b), store.Permanent); err != nil {
		log.Panicf("update raftAttributes should never fail: %v", err)
	}
	c.members[id].RaftAttributes = raftAttr
}

// Validate ensures that there is no identical urls in the cluster peer list
func (c *Cluster) Validate() error {
	urlMap := make(map[string]bool)
	for _, m := range c.Members() {
		for _, url := range m.PeerURLs {
			if urlMap[url] {
				return fmt.Errorf("duplicate url %v in cluster config", url)
			}
			urlMap[url] = true
		}
	}
	return nil
}

func membersFromStore(st store.Store) (map[types.ID]*Member, map[types.ID]bool) {
	members := make(map[types.ID]*Member)
	removed := make(map[types.ID]bool)
	e, err := st.Get(storeMembersPrefix, true, true)
	if err != nil {
		if isKeyNotFound(err) {
			return members, removed
		}
		log.Panicf("get storeMembers should never fail: %v", err)
	}
	for _, n := range e.Node.Nodes {
		m, err := nodeToMember(n)
		if err != nil {
			log.Panicf("nodeToMember should never fail: %v", err)
		}
		members[m.ID] = m
	}

	e, err = st.Get(storeRemovedMembersPrefix, true, true)
	if err != nil {
		if isKeyNotFound(err) {
			return members, removed
		}
		log.Panicf("get storeRemovedMembers should never fail: %v", err)
	}
	for _, n := range e.Node.Nodes {
		removed[mustParseMemberIDFromKey(n.Key)] = true
	}
	return members, removed
}

// ValidateClusterAndAssignIDs validates the local cluster by matching the PeerURLs
// with the existing cluster. If the validation succeeds, it assigns the IDs
// from the existing cluster to the local cluster.
// If the validation fails, an error will be returned.
func ValidateClusterAndAssignIDs(local *Cluster, existing *Cluster) error {
	ems := existing.Members()
	lms := local.Members()
	if len(ems) != len(lms) {
		return fmt.Errorf("member count is unequal")
	}
	sort.Sort(SortableMemberSliceByPeerURLs(ems))
	sort.Sort(SortableMemberSliceByPeerURLs(lms))

	for i := range ems {
		// TODO: Remove URLStringsEqual after improvement of using hostnames #2150 #2123
		if !netutil.URLStringsEqual(ems[i].PeerURLs, lms[i].PeerURLs) {
			return fmt.Errorf("unmatched member while checking PeerURLs")
		}
		lms[i].ID = ems[i].ID
	}
	local.members = make(map[types.ID]*Member)
	for _, m := range lms {
		local.members[m.ID] = m
	}
	return nil
}
<commit_msg>etcdserver: move the mutex before what it guards<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package etcdserver

import (
	"crypto/sha1"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"log"
	"net/url"
	"path"
	"sort"
	"strings"
	"sync"

	"github.com/coreos/etcd/pkg/flags"
	"github.com/coreos/etcd/pkg/netutil"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/raft/raftpb"
	"github.com/coreos/etcd/store"
)

const (
	raftAttributesSuffix = "raftAttributes"
	attributesSuffix     = "attributes"
)

type ClusterInfo interface {
	// ID returns the cluster ID
	ID() types.ID
	// ClientURLs returns an aggregate set of all URLs on which this
	// cluster is listening for client requests
	ClientURLs() []string
	// Members returns a slice of members sorted by their ID
	Members() []*Member
	// Member retrieves a particular member based on ID, or nil if the
	// member does not exist in the cluster
	Member(id types.ID) *Member
	// IsIDRemoved checks whether the given ID has been removed from this
	// cluster at some point in the past
	IsIDRemoved(id types.ID) bool
}

// Cluster is a list of Members that belong to the same raft cluster
type Cluster struct {
	id    types.ID
	token string
	store store.Store

	sync.Mutex // guards members and removed map
	members    map[types.ID]*Member
	// removed contains the ids of removed members in the cluster.
	// removed id cannot be reused.
	removed map[types.ID]bool
}

// NewClusterFromString returns a Cluster instantiated from the given cluster token
// and cluster string, by parsing members from a set of discovery-formatted
// names-to-IPs, like:
// mach0=http://1.1.1.1,mach0=http://2.2.2.2,mach1=http://3.3.3.3,mach2=http://4.4.4.4
func NewClusterFromString(token string, cluster string) (*Cluster, error) {
	c := newCluster(token)

	v, err := url.ParseQuery(strings.Replace(cluster, ",", "&", -1))
	if err != nil {
		return nil, err
	}
	for name, urls := range v {
		if len(urls) == 0 || urls[0] == "" {
			return nil, fmt.Errorf("Empty URL given for %q", name)
		}
		purls := &flags.URLsValue{}
		if err := purls.Set(strings.Join(urls, ",")); err != nil {
			return nil, err
		}
		m := NewMember(name, types.URLs(*purls), c.token, nil)
		if _, ok := c.members[m.ID]; ok {
			return nil, fmt.Errorf("Member exists with identical ID %v", m)
		}
		c.members[m.ID] = m
	}
	c.genID()
	return c, nil
}

func NewClusterFromStore(token string, st store.Store) *Cluster {
	c := newCluster(token)
	c.store = st
	c.members, c.removed = membersFromStore(c.store)
	return c
}

func NewClusterFromMembers(token string, id types.ID, membs []*Member) *Cluster {
	c := newCluster(token)
	c.id = id
	for _, m := range membs {
		c.members[m.ID] = m
	}
	return c
}

func newCluster(token string) *Cluster {
	return &Cluster{
		token:   token,
		members: make(map[types.ID]*Member),
		removed: make(map[types.ID]bool),
	}
}

func (c *Cluster) ID() types.ID { return c.id }

func (c *Cluster) Members() []*Member {
	c.Lock()
	defer c.Unlock()
	var sms SortableMemberSlice
	for _, m := range c.members {
		sms = append(sms, m.Clone())
	}
	sort.Sort(sms)
	return []*Member(sms)
}

func (c *Cluster) Member(id types.ID) *Member {
	c.Lock()
	defer c.Unlock()
	return c.members[id].Clone()
}

// MemberByName returns a Member with the given name if exists.
// If more than one member has the given name, it will panic.
func (c *Cluster) MemberByName(name string) *Member {
	c.Lock()
	defer c.Unlock()
	var memb *Member
	for _, m := range c.members {
		if m.Name == name {
			if memb != nil {
				log.Panicf("two members with the given name %q exist", name)
			}
			memb = m
		}
	}
	return memb.Clone()
}

func (c *Cluster) MemberIDs() []types.ID {
	c.Lock()
	defer c.Unlock()
	var ids []types.ID
	for _, m := range c.members {
		ids = append(ids, m.ID)
	}
	sort.Sort(types.IDSlice(ids))
	return ids
}

func (c *Cluster) IsIDRemoved(id types.ID) bool {
	c.Lock()
	defer c.Unlock()
	return c.removed[id]
}

// PeerURLs returns a list of all peer addresses.
// The returned list is sorted in ascending lexicographical order.
func (c *Cluster) PeerURLs() []string {
	c.Lock()
	defer c.Unlock()
	urls := make([]string, 0)
	for _, p := range c.members {
		for _, addr := range p.PeerURLs {
			urls = append(urls, addr)
		}
	}
	sort.Strings(urls)
	return urls
}

// ClientURLs returns a list of all client addresses.
// The returned list is sorted in ascending lexicographical order.
func (c *Cluster) ClientURLs() []string {
	c.Lock()
	defer c.Unlock()
	urls := make([]string, 0)
	for _, p := range c.members {
		for _, url := range p.ClientURLs {
			urls = append(urls, url)
		}
	}
	sort.Strings(urls)
	return urls
}

func (c *Cluster) String() string {
	c.Lock()
	defer c.Unlock()
	sl := []string{}
	for _, m := range c.members {
		for _, u := range m.PeerURLs {
			sl = append(sl, fmt.Sprintf("%s=%s", m.Name, u))
		}
	}
	sort.Strings(sl)
	return strings.Join(sl, ",")
}

func (c *Cluster) genID() {
	mIDs := c.MemberIDs()
	b := make([]byte, 8*len(mIDs))
	for i, id := range mIDs {
		binary.BigEndian.PutUint64(b[8*i:], uint64(id))
	}
	hash := sha1.Sum(b)
	c.id = types.ID(binary.BigEndian.Uint64(hash[:8]))
}

func (c *Cluster) SetID(id types.ID) { c.id = id }

func (c *Cluster) SetStore(st store.Store) { c.store = st }

func (c *Cluster) Recover() {
	c.members, c.removed = membersFromStore(c.store)
}

// ValidateConfigurationChange takes a proposed ConfChange and
// ensures that it is still valid.
func (c *Cluster) ValidateConfigurationChange(cc raftpb.ConfChange) error {
	members, removed := membersFromStore(c.store)
	id := types.ID(cc.NodeID)
	if removed[id] {
		return ErrIDRemoved
	}
	switch cc.Type {
	case raftpb.ConfChangeAddNode:
		if members[id] != nil {
			return ErrIDExists
		}
		urls := make(map[string]bool)
		for _, m := range members {
			for _, u := range m.PeerURLs {
				urls[u] = true
			}
		}
		m := new(Member)
		if err := json.Unmarshal(cc.Context, m); err != nil {
			log.Panicf("unmarshal member should never fail: %v", err)
		}
		for _, u := range m.PeerURLs {
			if urls[u] {
				return ErrPeerURLexists
			}
		}
	case raftpb.ConfChangeRemoveNode:
		if members[id] == nil {
			return ErrIDNotFound
		}
	case raftpb.ConfChangeUpdateNode:
		if members[id] == nil {
			return ErrIDNotFound
		}
		urls := make(map[string]bool)
		for _, m := range members {
			if m.ID == id {
				continue
			}
			for _, u := range m.PeerURLs {
				urls[u] = true
			}
		}
		m := new(Member)
		if err := json.Unmarshal(cc.Context, m); err != nil {
			log.Panicf("unmarshal member should never fail: %v", err)
		}
		for _, u := range m.PeerURLs {
			if urls[u] {
				return ErrPeerURLexists
			}
		}
	default:
		log.Panicf("ConfChange type should be either AddNode, RemoveNode or UpdateNode")
	}
	return nil
}

// AddMember adds a new Member into the cluster, and saves the given member's
// raftAttributes into the store. The given member should have empty attributes.
// A Member with a matching id must not exist.
func (c *Cluster) AddMember(m *Member) {
	c.Lock()
	defer c.Unlock()
	b, err := json.Marshal(m.RaftAttributes)
	if err != nil {
		log.Panicf("marshal raftAttributes should never fail: %v", err)
	}
	p := path.Join(memberStoreKey(m.ID), raftAttributesSuffix)
	if _, err := c.store.Create(p, false, string(b), false, store.Permanent); err != nil {
		log.Panicf("create raftAttributes should never fail: %v", err)
	}
	c.members[m.ID] = m
}

// RemoveMember removes a member from the store.
// The given id MUST exist, or the function panics.
func (c *Cluster) RemoveMember(id types.ID) {
	c.Lock()
	defer c.Unlock()
	if _, err := c.store.Delete(memberStoreKey(id), true, true); err != nil {
		log.Panicf("delete member should never fail: %v", err)
	}
	delete(c.members, id)
	if _, err := c.store.Create(removedMemberStoreKey(id), false, "", false, store.Permanent); err != nil {
		log.Panicf("create removedMember should never fail: %v", err)
	}
	c.removed[id] = true
}

func (c *Cluster) UpdateAttributes(id types.ID, attr Attributes) {
	c.Lock()
	defer c.Unlock()
	c.members[id].Attributes = attr
	// TODO: update store in this function
}

func (c *Cluster) UpdateRaftAttributes(id types.ID, raftAttr RaftAttributes) {
	c.Lock()
	defer c.Unlock()
	b, err := json.Marshal(raftAttr)
	if err != nil {
		log.Panicf("marshal raftAttributes should never fail: %v", err)
	}
	p := path.Join(memberStoreKey(id), raftAttributesSuffix)
	if _, err := c.store.Update(p, string(b), store.Permanent); err != nil {
		log.Panicf("update raftAttributes should never fail: %v", err)
	}
	c.members[id].RaftAttributes = raftAttr
}

// Validate ensures that there is no identical urls in the cluster peer list
func (c *Cluster) Validate() error {
	urlMap := make(map[string]bool)
	for _, m := range c.Members() {
		for _, url := range m.PeerURLs {
			if urlMap[url] {
				return fmt.Errorf("duplicate url %v in cluster config", url)
			}
			urlMap[url] = true
		}
	}
	return nil
}

func membersFromStore(st store.Store) (map[types.ID]*Member, map[types.ID]bool) {
	members := make(map[types.ID]*Member)
	removed := make(map[types.ID]bool)
	e, err := st.Get(storeMembersPrefix, true, true)
	if err != nil {
		if isKeyNotFound(err) {
			return members, removed
		}
		log.Panicf("get storeMembers should never fail: %v", err)
	}
	for _, n := range e.Node.Nodes {
		m, err := nodeToMember(n)
		if err != nil {
			log.Panicf("nodeToMember should never fail: %v", err)
		}
		members[m.ID] = m
	}

	e, err = st.Get(storeRemovedMembersPrefix, true, true)
	if err != nil {
		if isKeyNotFound(err) {
			return members, removed
		}
		log.Panicf("get storeRemovedMembers should never fail: %v", err)
	}
	for _, n := range e.Node.Nodes {
		removed[mustParseMemberIDFromKey(n.Key)] = true
	}
	return members, removed
}

// ValidateClusterAndAssignIDs validates the local cluster by matching the PeerURLs
// with the existing cluster. If the validation succeeds, it assigns the IDs
// from the existing cluster to the local cluster.
// If the validation fails, an error will be returned.
func ValidateClusterAndAssignIDs(local *Cluster, existing *Cluster) error {
	ems := existing.Members()
	lms := local.Members()
	if len(ems) != len(lms) {
		return fmt.Errorf("member count is unequal")
	}
	sort.Sort(SortableMemberSliceByPeerURLs(ems))
	sort.Sort(SortableMemberSliceByPeerURLs(lms))

	for i := range ems {
		// TODO: Remove URLStringsEqual after improvement of using hostnames #2150 #2123
		if !netutil.URLStringsEqual(ems[i].PeerURLs, lms[i].PeerURLs) {
			return fmt.Errorf("unmatched member while checking PeerURLs")
		}
		lms[i].ID = ems[i].ID
	}
	local.members = make(map[types.ID]*Member)
	for _, m := range lms {
		local.members[m.ID] = m
	}
	return nil
}
<|endoftext|>