<commit_before>package datastore

import (
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/gocql/gocql"

	"github.com/jeffpierce/cassabon/config"
	"github.com/jeffpierce/cassabon/logging"
	"github.com/jeffpierce/cassabon/middleware"
)

// rollup contains the accumulated metrics data for a path.
type rollup struct {
	expr  string    // The text form of the path expression, to locate the definition
	count []uint64  // The number of data points accumulated (for averaging)
	value []float64 // One rollup per window definition
}

// runlist contains the paths to be written for an expression, and when to write the rollups.
type runlist struct {
	nextWriteTime []time.Time        // The next write time for each rollup bucket
	path          map[string]*rollup // The rollup data for each path matched by the expression
}

type StoreManager struct {

	// Rollup configuration.
	// Note: Does not reload on SIGHUP.
	rollupPriority []string                    // First matched expression wins
	rollup         map[string]config.RollupDef // Rollup processing definitions by path expression

	// Timer management.
	setTimeout chan time.Duration // Write a duration to this to get a notification on timeout channel
	timeout    chan struct{}      // Timeout notifications arrive on this channel

	// Database connection.
	dbClient *gocql.Session

	// Rollup data.
	byPath map[string]*rollup  // Stats, by path, for rollup accumulation
	byExpr map[string]*runlist // Stats, by path within expression, for rollup processing
}

func (sm *StoreManager) Init() {

	// Copy in the configuration (requires hard restart to refresh).
	sm.rollupPriority = config.G.RollupPriority
	sm.rollup = config.G.Rollup

	// Initialize private objects.
	sm.setTimeout = make(chan time.Duration, 0)
	sm.timeout = make(chan struct{}, 1)

	// Start the persistent goroutines.
	config.G.OnExitWG.Add(2)
	go sm.timer()
	go sm.run()

	// Kick off the timer.
	sm.setTimeout <- time.Second
}

func (sm *StoreManager) Start() {
}

func (sm *StoreManager) resetRollupData() {

	// Initialize rollup data structures.
	sm.byPath = make(map[string]*rollup)
	sm.byExpr = make(map[string]*runlist)
	baseTime := time.Now()
	for expr, rollupdef := range sm.rollup {
		// For each expression, provide a place to record all the paths that it matches.
		rl := new(runlist)
		rl.nextWriteTime = make([]time.Time, len(rollupdef.Windows))
		rl.path = make(map[string]*rollup)
		// Establish the next time boundary on which each write will take place.
		for i, v := range rollupdef.Windows {
			rl.nextWriteTime[i] = nextTimeBoundary(baseTime, v.Window)
		}
		sm.byExpr[expr] = rl
	}
}

// populateSchema ensures that all necessary Cassandra setup has been completed.
func (sm *StoreManager) populateSchema() {

	// Create the keyspace if it does not exist.
	conn := sm.dbClient.Pool.Pick(sm.dbClient.Query(""))
	if err := conn.UseKeyspace(config.G.Cassandra.Keyspace); err != nil {
		// Note: "USE <keyspace>" isn't allowed, and conn.UseKeyspace() isn't sticky.
		config.G.Log.System.LogInfo("Keyspace not found: %v", err)
		var options string
		if len(config.G.Cassandra.CreateOpts) > 0 {
			options = "," + config.G.Cassandra.CreateOpts
		}
		query := fmt.Sprintf(
			"CREATE KEYSPACE %s WITH replication = {'class':'%s'%s}",
			config.G.Cassandra.Keyspace, config.G.Cassandra.Strategy, options)
		config.G.Log.System.LogDebug(query)
		if err := sm.dbClient.Query(query).Exec(); err != nil {
			config.G.Log.System.LogFatal("Could not create keyspace: %v", err)
		}
		config.G.Log.System.LogInfo("Keyspace %q created", config.G.Cassandra.Keyspace)
	}

	// Create tables if they do not exist
	for _, table := range config.G.RollupTables {
		var ttlfloat float64
		ttl := strings.Split(table, "_")[1]
		ttlfloat, _ = strconv.ParseFloat(ttl, 64)
		query := fmt.Sprintf(
			`CREATE TABLE IF NOT EXISTS %s.%s
                (path text, timestamp timestamp, stat double, PRIMARY KEY (path, timestamp))
            WITH COMPACT STORAGE
                AND CLUSTERING ORDER BY (timestamp ASC)
                AND compaction = {'class': 'org.apache.cassandra.db.compaction.DateTieredCompactionStrategy'}
                AND compression = {'sstable_compression': 'org.apache.cassandra.io.compress.LZ4Compressor'}
                AND dclocal_read_repair_chance = 0.1
                AND default_time_to_live = %v
                AND gc_grace_seconds = 864000
                AND memtable_flush_period_in_ms = 0
                AND read_repair_chance = 0.0
                AND speculative_retry = '99.0PERCENTILE';`,
			config.G.Cassandra.Keyspace, table, int(ttlfloat*1.1))

		config.G.Log.System.LogDebug(query)

		if err := sm.dbClient.Query(query).Exec(); err != nil {
			config.G.Log.System.LogFatal("Table %q creation failed: %v", table, err)
		}
	}
}

func (sm *StoreManager) run() {

	defer config.G.OnPanic()

	// Perform first-time initialization of rollup data accumulation structures.
	sm.resetRollupData()

	// Open connection to the Cassandra database here, so we can defer the close.
	var err error
	config.G.Log.System.LogDebug("StoreManager initializing Cassandra client")
	sm.dbClient, err = middleware.CassandraSession(
		config.G.Cassandra.Hosts,
		config.G.Cassandra.Port,
		"",
	)
	if err != nil {
		// Without Cassandra client we can't do our job, so log, whine, and crash.
		config.G.Log.System.LogFatal("StoreManager unable to connect to Cassandra at %v, port %s: %v",
			config.G.Cassandra.Hosts, config.G.Cassandra.Port, err)
	}

	defer sm.dbClient.Close()
	config.G.Log.System.LogDebug("StoreManager Cassandra client initialized")

	config.G.Log.System.LogDebug("StoreManager Cassandra Keyspace configuration starting...")
	sm.populateSchema()

	for {
		select {
		case <-config.G.OnPeerChangeReq:
			config.G.Log.System.LogDebug("StoreManager::run received PEERCHANGE message")
			sm.flush(true)
			sm.resetRollupData()
			config.G.OnPeerChangeRsp <- struct{}{} // Unblock sender
		case <-config.G.OnExit:
			config.G.Log.System.LogDebug("StoreManager::run received QUIT message")
			sm.flush(true)
			config.G.OnExitWG.Done()
			return
		case metric := <-config.G.Channels.DataStore:
			sm.accumulate(metric)
		case <-sm.timeout:
			sm.flush(false)
		}
	}
}

// timer sends a message on the "timeout" channel after the specified duration.
func (sm *StoreManager) timer() {
	for {
		select {
		case <-config.G.OnExit:
			config.G.Log.System.LogDebug("StoreManager::timer received QUIT message")
			config.G.OnExitWG.Done()
			return
		case duration := <-sm.setTimeout:
			// Block in this state until a new entry is received.
			select {
			case <-config.G.OnExit:
				// Nothing; do handling above on next iteration.
			case <-time.After(duration):
				select {
				case sm.timeout <- struct{}{}:
					// Timeout sent.
				default:
					// Do not block.
				}
			}
		}
	}
}

// accumulate records a metric according to the rollup definitions.
func (sm *StoreManager) accumulate(metric config.CarbonMetric) {
	config.G.Log.System.LogDebug("StoreManager::accumulate %s=%v", metric.Path, metric.Value)

	// Locate the metric in the map.
	var currentRollup *rollup
	var found bool
	if currentRollup, found = sm.byPath[metric.Path]; !found {

		// Determine which expression matches this path.
		var expr string
		for _, expr = range sm.rollupPriority {
			if expr != config.ROLLUP_CATCHALL {
				if sm.rollup[expr].Expression.MatchString(metric.Path) {
					break
				}
			}
			// Catchall always appears last, and is therefore the default value.
		}

		// Initialize, and insert the new rollup into both maps.
		currentRollup = new(rollup)
		currentRollup.expr = expr
		currentRollup.count = make([]uint64, len(sm.rollup[expr].Windows))
		currentRollup.value = make([]float64, len(sm.rollup[expr].Windows))
		sm.byPath[metric.Path] = currentRollup
		sm.byExpr[expr].path[metric.Path] = currentRollup

		// Send the entry off for writing to the path index.
		config.G.Channels.IndexStore <- metric
	}

	// Apply the incoming metric to each rollup bucket.
	switch sm.rollup[currentRollup.expr].Method {
	case config.AVERAGE:
		for i, v := range currentRollup.value {
			currentRollup.value[i] = (v*float64(currentRollup.count[i]) + metric.Value) /
				float64(currentRollup.count[i]+1)
		}
	case config.MAX:
		for i, v := range currentRollup.value {
			if v < metric.Value {
				currentRollup.value[i] = metric.Value
			}
		}
	case config.MIN:
		for i, v := range currentRollup.value {
			if v > metric.Value || currentRollup.count[i] == 0 {
				currentRollup.value[i] = metric.Value
			}
		}
	case config.SUM:
		for i, v := range currentRollup.value {
			currentRollup.value[i] = v + metric.Value
		}
	}

	// Note that we added a data point into each bucket.
	for i, _ := range currentRollup.count {
		currentRollup.count[i]++
	}
}

// flush persists the accumulated metrics to the database.
func (sm *StoreManager) flush(terminating bool) {
	config.G.Log.System.LogDebug("StoreManager::flush terminating=%v", terminating)

	// Report the current length of the list of unique paths seen.
	logging.Statsd.Client.Gauge("path.count", int64(len(sm.byPath)), 1.0)

	// Use a consistent current time for all tests in this cycle.
	baseTime := time.Now()

	// Use a reasonable default value for setting the next timer delay.
	nextFlush := baseTime.Add(time.Minute)

	// Walk the set of expressions, looking for closed rollup windows.
	for expr, rl := range sm.byExpr {

		// Inspect each rollup window defined for this expression.
		for i, windowEnd := range rl.nextWriteTime {

			// If the window has closed, or if terminating, process and clear the data.
			if windowEnd.Before(baseTime) || terminating {

				var statTime time.Time
				if terminating {
					statTime = baseTime
				} else {
					statTime = windowEnd
				}

				// Iterate over all the paths that match the current expression.
				for path, rollup := range rl.path {

					if rollup.count[i] > 0 {
						// Data has accumulated while this window was open; write it.
						sm.write(expr, sm.rollup[expr].Windows[i], path, statTime, rollup.value[i])
					}

					// Ensure the bucket is empty for the next open window.
					rollup.count[i] = 0
					rollup.value[i] = 0
				}

				// Set a new window closing time for the just-cleared window.
				rl.nextWriteTime[i] = nextTimeBoundary(baseTime, sm.rollup[expr].Windows[i].Window)
			}
			// ASSERT: rl.nextWriteTime[i] time is in the future (later than baseTime).

			// Adjust the timer delay downwards if this window closing time is
			// earlier than all others seen so far.
			if nextFlush.After(rl.nextWriteTime[i]) {
				nextFlush = rl.nextWriteTime[i]
			}
		}
	}

	// Set a timer to expire when the earliest future window closing occurs.
	if !terminating {

		// Convert window closing time to a duration, and do a sanity check.
		delay := nextFlush.Sub(baseTime)
		if delay.Nanoseconds() < 0 {
			delay = time.Millisecond
		}

		// Perform a non-blocking write to the timeout channel.
		select {
		case sm.setTimeout <- delay:
			// Notification sent
		default:
			// Do not block if channel is at capacity
		}
	}
}

// flush persists the accumulated metrics to the database.
func (sm *StoreManager) write(expr string, w config.RollupWindow, path string, ts time.Time, value float64) {

	config.G.Log.Carbon.LogInfo("match=%q tbl=%s ts=%v path=%s val=%.4f win=%v ret=%v ",
		expr, w.Table, ts.Format("15:04:05.000"), path, value, w.Window, w.Retention)

	query := fmt.Sprintf(
		`INSERT INTO %s.%s (path, timestamp, stat) VALUES (?, ?, ?)`,
		config.G.Cassandra.Keyspace, w.Table)

	if err := sm.dbClient.Query(query, path, ts, value).Exec(); err != nil {
		// Could not write to Cassandra cluster...we should scream loudly about this.  Possibly a failure case?.
		config.G.Log.System.LogError("Cassandra write error: %v", err)
	}
}
<commit_msg>Log table creations; don't create if they already exist<commit_after>package datastore

import (
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/gocql/gocql"

	"github.com/jeffpierce/cassabon/config"
	"github.com/jeffpierce/cassabon/logging"
	"github.com/jeffpierce/cassabon/middleware"
)

// rollup contains the accumulated metrics data for a path.
type rollup struct {
	expr  string    // The text form of the path expression, to locate the definition
	count []uint64  // The number of data points accumulated (for averaging)
	value []float64 // One rollup per window definition
}

// runlist contains the paths to be written for an expression, and when to write the rollups.
type runlist struct {
	nextWriteTime []time.Time        // The next write time for each rollup bucket
	path          map[string]*rollup // The rollup data for each path matched by the expression
}

type StoreManager struct {

	// Rollup configuration.
	// Note: Does not reload on SIGHUP.
	rollupPriority []string                    // First matched expression wins
	rollup         map[string]config.RollupDef // Rollup processing definitions by path expression

	// Timer management.
	setTimeout chan time.Duration // Write a duration to this to get a notification on timeout channel
	timeout    chan struct{}      // Timeout notifications arrive on this channel

	// Database connection.
	dbClient *gocql.Session

	// Rollup data.
	byPath map[string]*rollup  // Stats, by path, for rollup accumulation
	byExpr map[string]*runlist // Stats, by path within expression, for rollup processing
}

func (sm *StoreManager) Init() {

	// Copy in the configuration (requires hard restart to refresh).
	sm.rollupPriority = config.G.RollupPriority
	sm.rollup = config.G.Rollup

	// Initialize private objects.
	sm.setTimeout = make(chan time.Duration, 0)
	sm.timeout = make(chan struct{}, 1)

	// Start the persistent goroutines.
	config.G.OnExitWG.Add(2)
	go sm.timer()
	go sm.run()

	// Kick off the timer.
	sm.setTimeout <- time.Second
}

func (sm *StoreManager) Start() {
}

func (sm *StoreManager) resetRollupData() {

	// Initialize rollup data structures.
	sm.byPath = make(map[string]*rollup)
	sm.byExpr = make(map[string]*runlist)
	baseTime := time.Now()
	for expr, rollupdef := range sm.rollup {
		// For each expression, provide a place to record all the paths that it matches.
		rl := new(runlist)
		rl.nextWriteTime = make([]time.Time, len(rollupdef.Windows))
		rl.path = make(map[string]*rollup)
		// Establish the next time boundary on which each write will take place.
		for i, v := range rollupdef.Windows {
			rl.nextWriteTime[i] = nextTimeBoundary(baseTime, v.Window)
		}
		sm.byExpr[expr] = rl
	}
}

// populateSchema ensures that all necessary Cassandra setup has been completed.
func (sm *StoreManager) populateSchema() {

	// Create the keyspace if it does not exist.
	conn := sm.dbClient.Pool.Pick(sm.dbClient.Query(""))
	if err := conn.UseKeyspace(config.G.Cassandra.Keyspace); err != nil {
		// Note: "USE <keyspace>" isn't allowed, and conn.UseKeyspace() isn't sticky.
		config.G.Log.System.LogInfo("Keyspace not found: %v", err)
		var options string
		if len(config.G.Cassandra.CreateOpts) > 0 {
			options = "," + config.G.Cassandra.CreateOpts
		}
		query := fmt.Sprintf(
			"CREATE KEYSPACE %s WITH replication = {'class':'%s'%s}",
			config.G.Cassandra.Keyspace, config.G.Cassandra.Strategy, options)
		config.G.Log.System.LogDebug(query)
		if err := sm.dbClient.Query(query).Exec(); err != nil {
			config.G.Log.System.LogFatal("Could not create keyspace: %v", err)
		}
		config.G.Log.System.LogInfo("Keyspace %q created", config.G.Cassandra.Keyspace)
	}

	// Create tables if they do not exist
	ksmd, _ := sm.dbClient.KeyspaceMetadata(config.G.Cassandra.Keyspace)
	for _, table := range config.G.RollupTables {
		if ksmd != nil {
			if _, found := ksmd.Tables[table]; found {
				continue
			}
		}
		var ttlfloat float64
		ttl := strings.Split(table, "_")[1]
		ttlfloat, _ = strconv.ParseFloat(ttl, 64)
		query := fmt.Sprintf(
			`CREATE TABLE IF NOT EXISTS %s.%s
                (path text, timestamp timestamp, stat double, PRIMARY KEY (path, timestamp))
            WITH COMPACT STORAGE
                AND CLUSTERING ORDER BY (timestamp ASC)
                AND compaction = {'class': 'org.apache.cassandra.db.compaction.DateTieredCompactionStrategy'}
                AND compression = {'sstable_compression': 'org.apache.cassandra.io.compress.LZ4Compressor'}
                AND dclocal_read_repair_chance = 0.1
                AND default_time_to_live = %v
                AND gc_grace_seconds = 864000
                AND memtable_flush_period_in_ms = 0
                AND read_repair_chance = 0.0
                AND speculative_retry = '99.0PERCENTILE';`,
			config.G.Cassandra.Keyspace, table, int(ttlfloat*1.1))

		config.G.Log.System.LogDebug(query)
		config.G.Log.System.LogInfo("Creating table %q", table)

		if err := sm.dbClient.Query(query).Exec(); err != nil {
			config.G.Log.System.LogFatal("Table %q creation failed: %v", table, err)
		}
	}
}

func (sm *StoreManager) run() {

	defer config.G.OnPanic()

	// Perform first-time initialization of rollup data accumulation structures.
	sm.resetRollupData()

	// Open connection to the Cassandra database here, so we can defer the close.
	var err error
	config.G.Log.System.LogDebug("StoreManager initializing Cassandra client")
	sm.dbClient, err = middleware.CassandraSession(
		config.G.Cassandra.Hosts,
		config.G.Cassandra.Port,
		"",
	)
	if err != nil {
		// Without Cassandra client we can't do our job, so log, whine, and crash.
		config.G.Log.System.LogFatal("StoreManager unable to connect to Cassandra at %v, port %s: %v",
			config.G.Cassandra.Hosts, config.G.Cassandra.Port, err)
	}

	defer sm.dbClient.Close()
	config.G.Log.System.LogDebug("StoreManager Cassandra client initialized")

	config.G.Log.System.LogDebug("StoreManager Cassandra Keyspace configuration starting...")
	sm.populateSchema()

	for {
		select {
		case <-config.G.OnPeerChangeReq:
			config.G.Log.System.LogDebug("StoreManager::run received PEERCHANGE message")
			sm.flush(true)
			sm.resetRollupData()
			config.G.OnPeerChangeRsp <- struct{}{} // Unblock sender
		case <-config.G.OnExit:
			config.G.Log.System.LogDebug("StoreManager::run received QUIT message")
			sm.flush(true)
			config.G.OnExitWG.Done()
			return
		case metric := <-config.G.Channels.DataStore:
			sm.accumulate(metric)
		case <-sm.timeout:
			sm.flush(false)
		}
	}
}

// timer sends a message on the "timeout" channel after the specified duration.
func (sm *StoreManager) timer() {
	for {
		select {
		case <-config.G.OnExit:
			config.G.Log.System.LogDebug("StoreManager::timer received QUIT message")
			config.G.OnExitWG.Done()
			return
		case duration := <-sm.setTimeout:
			// Block in this state until a new entry is received.
			select {
			case <-config.G.OnExit:
				// Nothing; do handling above on next iteration.
			case <-time.After(duration):
				select {
				case sm.timeout <- struct{}{}:
					// Timeout sent.
				default:
					// Do not block.
				}
			}
		}
	}
}

// accumulate records a metric according to the rollup definitions.
func (sm *StoreManager) accumulate(metric config.CarbonMetric) {
	config.G.Log.System.LogDebug("StoreManager::accumulate %s=%v", metric.Path, metric.Value)

	// Locate the metric in the map.
	var currentRollup *rollup
	var found bool
	if currentRollup, found = sm.byPath[metric.Path]; !found {

		// Determine which expression matches this path.
		var expr string
		for _, expr = range sm.rollupPriority {
			if expr != config.ROLLUP_CATCHALL {
				if sm.rollup[expr].Expression.MatchString(metric.Path) {
					break
				}
			}
			// Catchall always appears last, and is therefore the default value.
		}

		// Initialize, and insert the new rollup into both maps.
		currentRollup = new(rollup)
		currentRollup.expr = expr
		currentRollup.count = make([]uint64, len(sm.rollup[expr].Windows))
		currentRollup.value = make([]float64, len(sm.rollup[expr].Windows))
		sm.byPath[metric.Path] = currentRollup
		sm.byExpr[expr].path[metric.Path] = currentRollup

		// Send the entry off for writing to the path index.
		config.G.Channels.IndexStore <- metric
	}

	// Apply the incoming metric to each rollup bucket.
	switch sm.rollup[currentRollup.expr].Method {
	case config.AVERAGE:
		for i, v := range currentRollup.value {
			currentRollup.value[i] = (v*float64(currentRollup.count[i]) + metric.Value) /
				float64(currentRollup.count[i]+1)
		}
	case config.MAX:
		for i, v := range currentRollup.value {
			if v < metric.Value {
				currentRollup.value[i] = metric.Value
			}
		}
	case config.MIN:
		for i, v := range currentRollup.value {
			if v > metric.Value || currentRollup.count[i] == 0 {
				currentRollup.value[i] = metric.Value
			}
		}
	case config.SUM:
		for i, v := range currentRollup.value {
			currentRollup.value[i] = v + metric.Value
		}
	}

	// Note that we added a data point into each bucket.
	for i, _ := range currentRollup.count {
		currentRollup.count[i]++
	}
}

// flush persists the accumulated metrics to the database.
func (sm *StoreManager) flush(terminating bool) {
	config.G.Log.System.LogDebug("StoreManager::flush terminating=%v", terminating)

	// Report the current length of the list of unique paths seen.
	logging.Statsd.Client.Gauge("path.count", int64(len(sm.byPath)), 1.0)

	// Use a consistent current time for all tests in this cycle.
	baseTime := time.Now()

	// Use a reasonable default value for setting the next timer delay.
	nextFlush := baseTime.Add(time.Minute)

	// Walk the set of expressions, looking for closed rollup windows.
	for expr, rl := range sm.byExpr {

		// Inspect each rollup window defined for this expression.
		for i, windowEnd := range rl.nextWriteTime {

			// If the window has closed, or if terminating, process and clear the data.
			if windowEnd.Before(baseTime) || terminating {

				var statTime time.Time
				if terminating {
					statTime = baseTime
				} else {
					statTime = windowEnd
				}

				// Iterate over all the paths that match the current expression.
				for path, rollup := range rl.path {

					if rollup.count[i] > 0 {
						// Data has accumulated while this window was open; write it.
						sm.write(expr, sm.rollup[expr].Windows[i], path, statTime, rollup.value[i])
					}

					// Ensure the bucket is empty for the next open window.
					rollup.count[i] = 0
					rollup.value[i] = 0
				}

				// Set a new window closing time for the just-cleared window.
				rl.nextWriteTime[i] = nextTimeBoundary(baseTime, sm.rollup[expr].Windows[i].Window)
			}
			// ASSERT: rl.nextWriteTime[i] time is in the future (later than baseTime).

			// Adjust the timer delay downwards if this window closing time is
			// earlier than all others seen so far.
			if nextFlush.After(rl.nextWriteTime[i]) {
				nextFlush = rl.nextWriteTime[i]
			}
		}
	}

	// Set a timer to expire when the earliest future window closing occurs.
	if !terminating {

		// Convert window closing time to a duration, and do a sanity check.
		delay := nextFlush.Sub(baseTime)
		if delay.Nanoseconds() < 0 {
			delay = time.Millisecond
		}

		// Perform a non-blocking write to the timeout channel.
		select {
		case sm.setTimeout <- delay:
			// Notification sent
		default:
			// Do not block if channel is at capacity
		}
	}
}

// flush persists the accumulated metrics to the database.
func (sm *StoreManager) write(expr string, w config.RollupWindow, path string, ts time.Time, value float64) {

	config.G.Log.Carbon.LogInfo("match=%q tbl=%s ts=%v path=%s val=%.4f win=%v ret=%v ",
		expr, w.Table, ts.Format("15:04:05.000"), path, value, w.Window, w.Retention)

	query := fmt.Sprintf(
		`INSERT INTO %s.%s (path, timestamp, stat) VALUES (?, ?, ?)`,
		config.G.Cassandra.Keyspace, w.Table)

	if err := sm.dbClient.Query(query, path, ts, value).Exec(); err != nil {
		// Could not write to Cassandra cluster...we should scream loudly about this.  Possibly a failure case?.
		config.G.Log.System.LogError("Cassandra write error: %v", err)
	}
}
<|endoftext|>


<commit_before>package datastore

import (
	"time"

	"github.com/jeffpierce/cassabon/config"
)

// rollups contains the accumulated metrics data for a path.
type rollups struct {
	expr  string    // The text form of the path expression, to locate the definition
	count []uint64  // The number of data points accumulated (for averaging)
	value []float64 // One rollup per window definition
}

type StoreManager struct {

	// Timer management.
	setTimeout chan time.Duration // Write a duration to this to get a notification on timeout channel
	timeout    chan struct{}      // Timeout notifications arrive on this channel

	// Rollup data.
	metricData map[string]rollups // Stats, by path, broken out by rollup window duration.
	maxTimeout time.Duration      // The duration of the shortest rollup window.
}

func (sm *StoreManager) Init() {

	// Initialize private objects.
	sm.setTimeout = make(chan time.Duration, 0)
	sm.timeout = make(chan struct{}, 1)

	// Initialize rollup data structures.
	sm.metricData = make(map[string]rollups)
	sm.maxTimeout = 5

	// Start the persistent goroutines.
	config.G.OnExitWG.Add(2)
	go sm.timer()
	go sm.insert()

	// Kick off the timer.
	sm.setTimeout <- time.Duration(sm.maxTimeout) * time.Second
}

func (sm *StoreManager) Start() {
	config.G.OnReload2WG.Add(1)
	go sm.run()
}

func (sm *StoreManager) run() {

	// Wait for metrics entries to arrive, and process them.
	for {
		select {
		case <-config.G.OnReload2:
			config.G.Log.System.LogDebug("StoreManager::run received QUIT message")
			config.G.OnReload2WG.Done()
			return
		case metric := <-config.G.Channels.DataStore:
			config.G.Log.System.LogDebug("StoreManager received metric: %v", metric)

			// Send the entry off for writing to the path index.
			config.G.Channels.IndexStore <- metric

			// Send the entry off for writing to the stats store.
			config.G.Channels.StatStore <- metric
		}
	}
}

// timer sends a message on the "timeout" channel after the specified duration.
func (sm *StoreManager) timer() {
	for {
		select {
		case <-config.G.OnExit:
			config.G.Log.System.LogDebug("StoreManager::timer received QUIT message")
			config.G.OnExitWG.Done()
			return
		case duration := <-sm.setTimeout:
			// Block in this state until a new entry is received.
			select {
			case <-config.G.OnExit:
				// Nothing; do handling above on next iteration.
			case <-time.After(duration):
				select {
				case sm.timeout <- struct{}{}:
					// Timeout sent.
				default:
					// Do not block.
				}
			}
		}
	}
}

// insert accumulates metrics at each rollup level, and writes them out at defined intervals.
func (sm *StoreManager) insert() {

	// TODO: Open connection to the Cassandra database here, so we can defer the close.

	for {
		select {
		case <-config.G.OnExit:
			config.G.Log.System.LogDebug("StoreManager::insert received QUIT message")
			sm.flush()
			config.G.OnExitWG.Done()
			return
		case metric := <-config.G.Channels.StatStore:
			config.G.Log.System.LogDebug("StoreManager::insert received metric: %v", metric)
			sm.accumulate(metric)
		case <-sm.timeout:
			config.G.Log.System.LogDebug("StoreManager::insert received timeout")
			sm.flush()
			select {
			case sm.setTimeout <- time.Duration(sm.maxTimeout) * time.Second:
				// Notification sent
			default:
				// Do not block if channel is at capacity
			}
		}
	}
}

// accumulate records a metric according to the rollup definitions.
func (sm *StoreManager) accumulate(metric config.CarbonMetric) {
	config.G.Log.System.LogDebug("StoreManager::accumulate")

	// Locate the metric in the map.
	var currentRollups rollups
	var found bool
	if currentRollups, found = sm.metricData[metric.Path]; !found {

		// Determine which expression matches this path.
		for _, expr := range config.G.RollupPriority {
			if expr != config.CATCHALL_EXPRESSION {
				if config.G.Rollup[expr].Expression.MatchString(metric.Path) {
					currentRollups.expr = expr
					break
				}
			} else {
				currentRollups.expr = config.CATCHALL_EXPRESSION
			}
		}

		currentRollups.count = make([]uint64, len(config.G.Rollup[currentRollups.expr].Windows))
		currentRollups.value = make([]float64, len(config.G.Rollup[currentRollups.expr].Windows))
	}

	// Apply the incoming metric to each rollup bucket.
	switch config.G.Rollup[currentRollups.expr].Method {
	case config.AVERAGE:
		for i, v := range currentRollups.value {
			currentRollups.value[i] = (v*float64(currentRollups.count[i]) + metric.Value) /
				float64(currentRollups.count[i]+1)
		}
	case config.MAX:
		for i, v := range currentRollups.value {
			if v < metric.Value {
				currentRollups.value[i] = metric.Value
			}
		}
	case config.MIN:
		for i, v := range currentRollups.value {
			if v > metric.Value {
				currentRollups.value[i] = metric.Value
			}
		}
	case config.SUM:
		for i, v := range currentRollups.value {
			currentRollups.value[i] = v + metric.Value
		}
	}
	for i, _ := range currentRollups.count {
		currentRollups.count[i]++
	}

	// Save the updated structure.
	sm.metricData[metric.Path] = currentRollups
}

// flush persists the accumulated metrics to the database.
func (sm *StoreManager) flush() {
	config.G.Log.System.LogDebug("StoreManager::flush")
}
<commit_msg>Add representation to permit walking paths by expression matched.<commit_after>package datastore

import (
	"time"

	"github.com/jeffpierce/cassabon/config"
)

// rollup contains the accumulated metrics data for a path.
type rollup struct {
	expr  string    // The text form of the path expression, to locate the definition
	count []uint64  // The number of data points accumulated (for averaging)
	value []float64 // One rollup per window definition
}

// runlist contains the paths to be written for an expression, and when to write the rollups.
type runlist struct {
	nextWrite []time.Time        // The next write time for each rollup bucket
	path      map[string]*rollup // The rollup data for each path matched by the expression
}

type StoreManager struct {

	// Timer management.
	setTimeout chan time.Duration // Write a duration to this to get a notification on timeout channel
	timeout    chan struct{}      // Timeout notifications arrive on this channel

	// Rollup data.
	byPath     map[string]*rollup  // Stats, by path, for rollup accumulation
	byExpr     map[string]*runlist // Stats, by path within expression, for rollup processing
	maxTimeout time.Duration       // The duration of the shortest rollup window.
}

func (sm *StoreManager) Init() {

	// Initialize private objects.
	sm.setTimeout = make(chan time.Duration, 0)
	sm.timeout = make(chan struct{}, 1)

	// Initialize rollup data structures.
	sm.byPath = make(map[string]*rollup)
	sm.byExpr = make(map[string]*runlist)
	for expr, rollupdef := range config.G.Rollup {
		// For each expression, provide a place to record all the paths that it matches.
		rl := new(runlist)
		rl.nextWrite = make([]time.Time, len(rollupdef.Windows))
		rl.path = make(map[string]*rollup)
		sm.byExpr[expr] = rl
	}
	sm.maxTimeout = 5

	// Start the persistent goroutines.
	config.G.OnExitWG.Add(2)
	go sm.timer()
	go sm.insert()

	// Kick off the timer.
	sm.setTimeout <- time.Duration(sm.maxTimeout) * time.Second
}

func (sm *StoreManager) Start() {
	config.G.OnReload2WG.Add(1)
	go sm.run()
}

func (sm *StoreManager) run() {

	// Wait for metrics entries to arrive, and process them.
	for {
		select {
		case <-config.G.OnReload2:
			config.G.Log.System.LogDebug("StoreManager::run received QUIT message")
			config.G.OnReload2WG.Done()
			return
		case metric := <-config.G.Channels.DataStore:
			config.G.Log.System.LogDebug("StoreManager received metric: %v", metric)

			// Send the entry off for writing to the path index.
			config.G.Channels.IndexStore <- metric

			// Send the entry off for writing to the stats store.
			config.G.Channels.StatStore <- metric
		}
	}
}

// timer sends a message on the "timeout" channel after the specified duration.
func (sm *StoreManager) timer() {
	for {
		select {
		case <-config.G.OnExit:
			config.G.Log.System.LogDebug("StoreManager::timer received QUIT message")
			config.G.OnExitWG.Done()
			return
		case duration := <-sm.setTimeout:
			// Block in this state until a new entry is received.
			select {
			case <-config.G.OnExit:
				// Nothing; do handling above on next iteration.
			case <-time.After(duration):
				select {
				case sm.timeout <- struct{}{}:
					// Timeout sent.
				default:
					// Do not block.
				}
			}
		}
	}
}

// insert accumulates metrics at each rollup level, and writes them out at defined intervals.
func (sm *StoreManager) insert() {

	// TODO: Open connection to the Cassandra database here, so we can defer the close.

	for {
		select {
		case <-config.G.OnExit:
			config.G.Log.System.LogDebug("StoreManager::insert received QUIT message")
			sm.flush()
			config.G.OnExitWG.Done()
			return
		case metric := <-config.G.Channels.StatStore:
			config.G.Log.System.LogDebug("StoreManager::insert received metric: %v", metric)
			sm.accumulate(metric)
		case <-sm.timeout:
			config.G.Log.System.LogDebug("StoreManager::insert received timeout")
			sm.flush()
			select {
			case sm.setTimeout <- time.Duration(sm.maxTimeout) * time.Second:
				// Notification sent
			default:
				// Do not block if channel is at capacity
			}
		}
	}
}

// accumulate records a metric according to the rollup definitions.
func (sm *StoreManager) accumulate(metric config.CarbonMetric) {
	config.G.Log.System.LogDebug("StoreManager::accumulate")

	// Locate the metric in the map.
	var currentRollup *rollup
	var found bool
	if currentRollup, found = sm.byPath[metric.Path]; !found {

		// Determine which expression matches this path.
		var expr string
		for _, expr = range config.G.RollupPriority {
			if expr != config.CATCHALL_EXPRESSION {
				if config.G.Rollup[expr].Expression.MatchString(metric.Path) {
					break
				}
			}
			// Catchall always appears last, and is therefore the default value.
		}

		// Initialize, and insert the new rollup into both maps.
		currentRollup = new(rollup)
		currentRollup.expr = expr
		currentRollup.count = make([]uint64, len(config.G.Rollup[expr].Windows))
		currentRollup.value = make([]float64, len(config.G.Rollup[expr].Windows))
		sm.byPath[metric.Path] = currentRollup
		sm.byExpr[expr].path[metric.Path] = currentRollup
	}

	// Apply the incoming metric to each rollup bucket.
	switch config.G.Rollup[currentRollup.expr].Method {
	case config.AVERAGE:
		for i, v := range currentRollup.value {
			currentRollup.value[i] = (v*float64(currentRollup.count[i]) + metric.Value) /
				float64(currentRollup.count[i]+1)
		}
	case config.MAX:
		for i, v := range currentRollup.value {
			if v < metric.Value {
				currentRollup.value[i] = metric.Value
			}
		}
	case config.MIN:
		for i, v := range currentRollup.value {
			if v > metric.Value {
				currentRollup.value[i] = metric.Value
			}
		}
	case config.SUM:
		for i, v := range currentRollup.value {
			currentRollup.value[i] = v + metric.Value
		}
	}

	// Note that we added a data point into each bucket.
	for i, _ := range currentRollup.count {
		currentRollup.count[i]++
	}
}

// flush persists the accumulated metrics to the database.
func (sm *StoreManager) flush() {
	config.G.Log.System.LogDebug("StoreManager::flush")
}
<|endoftext|>


<commit_before>
package main

import (
	"fmt"
)

type Fetcher interface {
	Fetch(url string) (body string, urls []string, err error)
}

func Crawl(url string, depth int, fetcher Fetcher, ch chan string, visited_ch chan map[string]bool) {

	defer close(ch)

	if depth <= 0 {
		return
	}

	visited := <- visited_ch
	_, found := visited[url]
	visited[url] = true
	visited_ch <- visited

	if found {
		return
	}

	body, urls, err := fetcher.Fetch(url)

	if err != nil {
		ch <- fmt.Sprintln(err)
		return
	}

	ch <- fmt.Sprintf("found: %s %q\n", url, body)

	chans := make([]chan string, len(urls))
	for i, u := range urls {
		chans[i] = make(chan string)
		go Crawl(u, depth-1, fetcher, chans[i], visited_ch)
	}

	for i := range chans {
		for s := range chans[i] {
			ch <- s
		}
	}

	return
}

func main() {
	ch := make(chan string)

	visited_ch := make(chan map[string]bool, 1)
	visited_ch <- make(map[string]bool)

	go Crawl("http://golang.org/", 4, fetcher, ch, visited_ch)

	for s := range ch {
		fmt.Print(s)
	}
}

// fakeFetcher is a Fetcher that returns canned results.
type fakeFetcher map[string]*fakeResult

type fakeResult struct {
	body string
	urls []string
}

func (f fakeFetcher) Fetch(url string) (string, []string, error) {
	if res, ok := f[url]; ok {
		return res.body, res.urls, nil
	}
	return "", nil, fmt.Errorf("not found: %s", url)
}

// fetcher is a populated fakeFetcher.
var fetcher = fakeFetcher{
	"http://golang.org/": &fakeResult{
		"The Go Programming Language",
		[]string{
			"http://golang.org/pkg/",
			"http://golang.org/cmd/",
		},
	},
	"http://golang.org/pkg/": &fakeResult{
		"Packages",
		[]string{
			"http://golang.org/",
			"http://golang.org/cmd/",
			"http://golang.org/pkg/fmt/",
			"http://golang.org/pkg/os/",
		},
	},
	"http://golang.org/pkg/fmt/": &fakeResult{
		"Package fmt",
		[]string{
			"http://golang.org/",
			"http://golang.org/pkg/",
		},
	},
	"http://golang.org/pkg/os/": &fakeResult{
		"Package os",
		[]string{
			"http://golang.org/",
			"http://golang.org/pkg/",
		},
	},
}
<commit_msg>Updated "Web Crawler" exercise description<commit_after>/* Taken from the Go Tour (Exercise: Web Crawler)
 *
 * In this exercise you'll use Go's concurrency features to parallelize a web crawler.
 *
 * Modify the Crawl function to fetch URLs in parallel without fetching the same URL twice.
 *
 */

package main

import (
	"fmt"
)

type Fetcher interface {
	Fetch(url string) (body string, urls []string, err error)
}

func Crawl(url string, depth int, fetcher Fetcher, ch chan string, visited_ch chan map[string]bool) {

	defer close(ch)

	if depth <= 0 {
		return
	}

	visited := <- visited_ch
	_, found := visited[url]
	visited[url] = true
	visited_ch <- visited

	if found {
		return
	}

	body, urls, err := fetcher.Fetch(url)

	if err != nil {
		ch <- fmt.Sprintln(err)
		return
	}

	ch <- fmt.Sprintf("found: %s %q\n", url, body)

	chans := make([]chan string, len(urls))
	for i, u := range urls {
		chans[i] = make(chan string)
		go Crawl(u, depth-1, fetcher, chans[i], visited_ch)
	}

	for i := range chans {
		for s := range chans[i] {
			ch <- s
		}
	}

	return
}

func main() {
	ch := make(chan string)

	visited_ch := make(chan map[string]bool, 1)
	visited_ch <- make(map[string]bool)

	go Crawl("http://golang.org/", 4, fetcher, ch, visited_ch)

	for s := range ch {
		fmt.Print(s)
	}
}

// fakeFetcher is a Fetcher that returns canned results.
type fakeFetcher map[string]*fakeResult

type fakeResult struct {
	body string
	urls []string
}

func (f fakeFetcher) Fetch(url string) (string, []string, error) {
	if res, ok := f[url]; ok {
		return res.body, res.urls, nil
	}
	return "", nil, fmt.Errorf("not found: %s", url)
}

// fetcher is a populated fakeFetcher.
var fetcher = fakeFetcher{
	"http://golang.org/": &fakeResult{
		"The Go Programming Language",
		[]string{
			"http://golang.org/pkg/",
			"http://golang.org/cmd/",
		},
	},
	"http://golang.org/pkg/": &fakeResult{
		"Packages",
		[]string{
			"http://golang.org/",
			"http://golang.org/cmd/",
			"http://golang.org/pkg/fmt/",
			"http://golang.org/pkg/os/",
		},
	},
	"http://golang.org/pkg/fmt/": &fakeResult{
		"Package fmt",
		[]string{
			"http://golang.org/",
			"http://golang.org/pkg/",
		},
	},
	"http://golang.org/pkg/os/": &fakeResult{
		"Package os",
		[]string{
			"http://golang.org/",
			"http://golang.org/pkg/",
		},
	},
}
<|endoftext|>


<commit_before><commit_msg>Fixed #103<commit_after><|endoftext|>


<commit_before>// Copyright (c) 2016 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package firewall

import (
	"github.com/golang/glog"
	"github.com/jinzhu/gorm"
	"github.com/romana/core/common"
	"sync"
)

type firewallStore struct {
	common.DbStore
	mu sync.Mutex
}

// Entities implements Entities method of
// Service interface.
func (firewallStore *firewallStore) Entities() []interface{} {
	retval := make([]interface{}, 1)
	retval[0] = new(IPtablesRule)
	return retval
}

// CreateSchemaPostProcess implements  common.ServiceStore.CreateSchemaPostProcess()
func (fs firewallStore) CreateSchemaPostProcess() error {
	return nil
}

func (fs firewallStore) GetDb() *gorm.DB {
	glog.Info("In GetDb()")
	return fs.Db
}

// IPtablesRule represents a single iptables rule managed by the agent.
// TODO rename FirewallRule
type IPtablesRule struct {
	ID    uint64 `sql:"AUTO_INCREMENT"`
	Body  string
	State string
}

func (firewallStore *firewallStore) addIPtablesRule(rule *IPtablesRule) error {
	glog.Info("Acquiring store mutex for addIPtablesRule")
	if rule == nil {
		glog.Error("In addIPtablesRule(), received nil rule")
		panic()
	}

	firewallStore.mu.Lock()
	defer func() {
		glog.Info("Releasing store mutex for addIPtablesRule")
		firewallStore.mu.Unlock()
	}()
	glog.Info("Acquired store mutex for addIPtablesRule")

	// db := firewallStore.DbStore.Db
	db := firewallStore.GetDb()
	glog.Info("In addIPtablesRule() after GetDb")
	if db == nil {
		glog.Error("In addIPtablesRule(), db is nil")
		panic()
	}

	firewallStore.DbStore.Db.Create(rule)
	glog.Info("In addIPtablesRule() after Db.Create")
	if db.Error != nil {
		return db.Error
	}
	firewallStore.DbStore.Db.NewRecord(*rule)
	err := common.MakeMultiError(db.GetErrors())
	if err != nil {
		return err
	}
	if db.Error != nil {
		return db.Error
	}
	return nil
}

func (firewallStore *firewallStore) listIPtablesRules() ([]IPtablesRule, error) {
	glog.Info("Acquiring store mutex for listIPtablesRules")
	firewallStore.mu.Lock()
	defer func() {
		glog.Info("Releasing store mutex for listIPtablesRules")
		firewallStore.mu.Unlock()
	}()
	glog.Info("Acquired store mutex for listIPtablesRules")

	var iPtablesRule []IPtablesRule
	firewallStore.DbStore.Db.Find(&iPtablesRule)
	err := common.MakeMultiError(firewallStore.DbStore.Db.GetErrors())
	if err != nil {
		return nil, err
	}
	return iPtablesRule, nil
}

func (firewallStore *firewallStore) deleteIPtablesRule(rule *IPtablesRule) error {
	glog.Info("Acquiring store mutex for deleteIPtablesRule")
	firewallStore.mu.Lock()
	defer func() {
		glog.Info("Releasing store mutex for deleteIPtablesRule")
		firewallStore.mu.Unlock()
	}()
	glog.Info("Acquired store mutex for deleteIPtablesRule")

	db := firewallStore.DbStore.Db
	firewallStore.DbStore.Db.Delete(rule)
	err := common.MakeMultiError(db.GetErrors())
	if err != nil {
		return err
	}
	if db.Error != nil {
		return db.Error
	}

	return nil
}

func (firewallStore *firewallStore) findIPtablesRules(subString string) (*[]IPtablesRule, error) {
	glog.Info("Acquiring store mutex for findIPtablesRule")
	firewallStore.mu.Lock()
	defer func() {
		glog.Info("Releasing store mutex for findIPtablesRule")
		firewallStore.mu.Unlock()
	}()
	glog.Info("Acquired store mutex for findIPtablesRule")

	var rules []IPtablesRule
	db := firewallStore.DbStore.Db
	firewallStore.DbStore.Db.Where("body LIKE = %?%", subString).Find(&rules)
	err := common.MakeMultiError(db.GetErrors())
	if err != nil {
		return nil, err
	}
	if db.Error != nil {
		return nil, db.Error
	}
	return &rules, nil
}

// opSwitchIPtables represents action to be taken in switchIPtablesRule
type opSwitchIPtables int

const (
	setRuleActive opSwitchIPtables = iota
	setRuleInactive
	toggleRule
)

func (op opSwitchIPtables) String() string {
	var result string

	switch op {
	case setRuleActive:
		result = "active"
	case setRuleInactive:
		result = "inactive"
	case toggleRule:
		result = "toggleRule"
	}

	return result
}

// switchIPtablesRule changes IPtablesRule state.
func (firewallStore *firewallStore) switchIPtablesRule(rule *IPtablesRule, op opSwitchIPtables) error {

	// Fast track return if nothing to be done
	if rule.State == op.String() {
		glog.Infof("switchIPtablesRule nothing to be done for %s", rule.State)
		return nil
	}

	glog.Info("Acquiring store mutex for switchIPtablesRule")
	firewallStore.mu.Lock()
	defer func() {
		glog.Info("Releasing store mutex for switchIPtablesRule")
		firewallStore.mu.Unlock()
	}()
	glog.Info("Acquired store mutex for switchIPtablesRule")

	// if toggle requested then reverse current state
	if op == toggleRule {
		if rule.State == setRuleInactive.String() {
			rule.State = setRuleActive.String()
		} else {
			rule.State = setRuleInactive.String()
		}
		// otherwise just assign op value
	} else {
		rule.State = op.String()
	}

	db := firewallStore.DbStore.Db
	firewallStore.DbStore.Db.Save(rule)
	err := common.MakeMultiError(db.GetErrors())
	if err != nil {
		return err
	}
	if db.Error != nil {
		return db.Error
	}

	return nil
}
<commit_msg>Fix Firewall database (2)<commit_after>// Copyright (c) 2016 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package firewall

import (
	"github.com/golang/glog"
	"github.com/jinzhu/gorm"
	"github.com/romana/core/common"
	"sync"
)

type firewallStore struct {
	common.DbStore
	mu sync.Mutex
}

// Entities implements Entities method of
// Service interface.
func (firewallStore *firewallStore) Entities() []interface{} {
	retval := make([]interface{}, 1)
	retval[0] = new(IPtablesRule)
	return retval
}

// CreateSchemaPostProcess implements  common.ServiceStore.CreateSchemaPostProcess()
func (fs firewallStore) CreateSchemaPostProcess() error {
	return nil
}

func (fs firewallStore) GetDb() *gorm.DB {
	glog.Info("In GetDb()")
	return fs.Db
}

// IPtablesRule represents a single iptables rule managed by the agent.
// TODO rename FirewallRule
type IPtablesRule struct {
	ID    uint64 `sql:"AUTO_INCREMENT"`
	Body  string
	State string
}

func (firewallStore *firewallStore) addIPtablesRule(rule *IPtablesRule) error {
	glog.Info("Acquiring store mutex for addIPtablesRule")
	if rule == nil {
		panic("In addIPtablesRule(), received nil rule")
	}

	firewallStore.mu.Lock()
	defer func() {
		glog.Info("Releasing store mutex for addIPtablesRule")
		firewallStore.mu.Unlock()
	}()
	glog.Info("Acquired store mutex for addIPtablesRule")

	// db := firewallStore.DbStore.Db
	db := firewallStore.GetDb()
	glog.Info("In addIPtablesRule() after GetDb")
	if db == nil {
		panic("In addIPtablesRule(), db is nil")
	}

	firewallStore.DbStore.Db.Create(rule)
	glog.Info("In addIPtablesRule() after Db.Create")
	if db.Error != nil {
		return db.Error
	}
	firewallStore.DbStore.Db.NewRecord(*rule)
	err := common.MakeMultiError(db.GetErrors())
	if err != nil {
		return err
	}
	if db.Error != nil {
		return db.Error
	}
	return nil
}

func (firewallStore *firewallStore) listIPtablesRules() ([]IPtablesRule, error) {
	glog.Info("Acquiring store mutex for listIPtablesRules")
	firewallStore.mu.Lock()
	defer func() {
		glog.Info("Releasing store mutex for listIPtablesRules")
		firewallStore.mu.Unlock()
	}()
	glog.Info("Acquired store mutex for listIPtablesRules")

	var iPtablesRule []IPtablesRule
	firewallStore.DbStore.Db.Find(&iPtablesRule)
	err := common.MakeMultiError(firewallStore.DbStore.Db.GetErrors())
	if err != nil {
		return nil, err
	}
	return iPtablesRule, nil
}

func (firewallStore *firewallStore) deleteIPtablesRule(rule *IPtablesRule) error {
	glog.Info("Acquiring store mutex for deleteIPtablesRule")
	firewallStore.mu.Lock()
	defer func() {
		glog.Info("Releasing store mutex for deleteIPtablesRule")
		firewallStore.mu.Unlock()
	}()
	glog.Info("Acquired store mutex for deleteIPtablesRule")

	db := firewallStore.DbStore.Db
	firewallStore.DbStore.Db.Delete(rule)
	err := common.MakeMultiError(db.GetErrors())
	if err != nil {
		return err
	}
	if db.Error != nil {
		return db.Error
	}

	return nil
}

func (firewallStore *firewallStore) findIPtablesRules(subString string) (*[]IPtablesRule, error) {
	glog.Info("Acquiring store mutex for findIPtablesRule")
	firewallStore.mu.Lock()
	defer func() {
		glog.Info("Releasing store mutex for findIPtablesRule")
		firewallStore.mu.Unlock()
	}()
	glog.Info("Acquired store mutex for findIPtablesRule")

	var rules []IPtablesRule
	db := firewallStore.DbStore.Db
	firewallStore.DbStore.Db.Where("body LIKE = %?%", subString).Find(&rules)
	err := common.MakeMultiError(db.GetErrors())
	if err != nil {
		return nil, err
	}
	if db.Error != nil {
		return nil, db.Error
	}
	return &rules, nil
}

// opSwitchIPtables represents action to be taken in switchIPtablesRule
type opSwitchIPtables int

const (
	setRuleActive opSwitchIPtables = iota
	setRuleInactive
	toggleRule
)

func (op opSwitchIPtables) String() string {
	var result string

	switch op {
	case setRuleActive:
		result = "active"
	case setRuleInactive:
		result = "inactive"
	case toggleRule:
		result = "toggleRule"
	}

	return result
}

// switchIPtablesRule changes IPtablesRule state.
func (firewallStore *firewallStore) switchIPtablesRule(rule *IPtablesRule, op opSwitchIPtables) error {

	// Fast track return if nothing to be done
	if rule.State == op.String() {
		glog.Infof("switchIPtablesRule nothing to be done for %s", rule.State)
		return nil
	}

	glog.Info("Acquiring store mutex for switchIPtablesRule")
	firewallStore.mu.Lock()
	defer func() {
		glog.Info("Releasing store mutex for switchIPtablesRule")
		firewallStore.mu.Unlock()
	}()
	glog.Info("Acquired store mutex for switchIPtablesRule")

	// if toggle requested then reverse current state
	if op == toggleRule {
		if rule.State == setRuleInactive.String() {
			rule.State = setRuleActive.String()
		} else {
			rule.State = setRuleInactive.String()
		}
		// otherwise just assign op value
	} else {
		rule.State = op.String()
	}

	db := firewallStore.DbStore.Db
	firewallStore.DbStore.Db.Save(rule)
	err := common.MakeMultiError(db.GetErrors())
	if err != nil {
		return err
	}
	if db.Error != nil {
		return db.Error
	}

	return nil
}
<|endoftext|>


<commit_before>package api

// TokenAuth is used to perform token backend operations on Vault
type TokenAuth struct {
	c *Client
}

// Token is used to return the client for token-backend API calls
func (a *Auth) Token() *TokenAuth {
	return &TokenAuth{c: a.c}
}

func (c *TokenAuth) Create(opts *TokenCreateRequest) (*Secret, error) {
	r := c.c.NewRequest("POST", "/v1/auth/token/create")
	if err := r.SetJSONBody(opts); err != nil {
		return nil, err
	}

	resp, err := c.c.RawRequest(r)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	return ParseSecret(resp.Body)
}

func (c *TokenAuth) CreateWithRole(opts *TokenCreateRequest, roleName string) (*Secret, error) {
	r := c.c.NewRequest("POST", "/v1/auth/token/create/"+roleName)
	if err := r.SetJSONBody(opts); err != nil {
		return nil, err
	}

	resp, err := c.c.RawRequest(r)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	return ParseSecret(resp.Body)
}

func (c *TokenAuth) Lookup(token string) (*Secret, error) {
	r := c.c.NewRequest("POST", "/v1/auth/token/lookup")
	if err := r.SetJSONBody(map[string]interface{}{
		"token": token,
	}); err != nil {
		return nil, err
	}

	resp, err := c.c.RawRequest(r)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	return ParseSecret(resp.Body)
}

func (c *TokenAuth) LookupAccessor(accessor string) (*Secret, error) {
	r := c.c.NewRequest("POST", "/v1/auth/token/lookup-accessor")
	if err := r.SetJSONBody(map[string]interface{}{
		"accessor": accessor,
	}); err != nil {
		return nil, err
	}
	resp, err := c.c.RawRequest(r)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	return ParseSecret(resp.Body)
}

func (c *TokenAuth) LookupSelf() (*Secret, error) {
	r := c.c.NewRequest("GET", "/v1/auth/token/lookup-self")

	resp, err := c.c.RawRequest(r)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	return ParseSecret(resp.Body)
}

func (c *TokenAuth) Renew(token string, increment int) (*Secret, error) {
	r := c.c.NewRequest("PUT", "/v1/auth/token/renew")
	if err := r.SetJSONBody(map[string]interface{}{
		"token":     token,
		"increment": increment,
	}); err != nil {
		return nil, err
	}

	resp, err := c.c.RawRequest(r)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	return ParseSecret(resp.Body)
}

func (c *TokenAuth) RenewSelf(increment int) (*Secret, error) {
	r := c.c.NewRequest("PUT", "/v1/auth/token/renew-self")

	body := map[string]interface{}{"increment": increment}
	if err := r.SetJSONBody(body); err != nil {
		return nil, err
	}

	resp, err := c.c.RawRequest(r)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	return ParseSecret(resp.Body)
}

// RevokeAccessor revokes a token associated with the given accessor
// along with all the child tokens.
func (c *TokenAuth) RevokeAccessor(accessor string) error {
	r := c.c.NewRequest("POST", "/v1/auth/token/revoke-accessor")
	if err := r.SetJSONBody(map[string]interface{}{
		"accessor": accessor,
	}); err != nil {
		return err
	}
	resp, err := c.c.RawRequest(r)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	return nil
}

// RevokeOrphan revokes a token without revoking the tree underneath it (so
// child tokens are orphaned rather than revoked)
func (c *TokenAuth) RevokeOrphan(token string) error {
	r := c.c.NewRequest("PUT", "/v1/auth/token/revoke-orphan")
	if err := r.SetJSONBody(map[string]interface{}{
		"token": token,
	}); err != nil {
		return err
	}

	resp, err := c.c.RawRequest(r)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	return nil
}

// RevokeSelf revokes the token making the call
func (c *TokenAuth) RevokeSelf() error {
	r := c.c.NewRequest("PUT", "/v1/auth/token/revoke-self")
	resp, err := c.c.RawRequest(r)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	return nil
}

// RevokeTree is the "normal" revoke operation that revokes the given token and
// the entire tree underneath -- all of its child tokens, their child tokens,
// etc.
func (c *TokenAuth) RevokeTree(token string) error {
	r := c.c.NewRequest("PUT", "/v1/auth/token/revoke")
	if err := r.SetJSONBody(map[string]interface{}{
		"token": token,
	}); err != nil {
		return err
	}

	resp, err := c.c.RawRequest(r)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	return nil
}

// TokenCreateRequest is the options structure for creating a token.
type TokenCreateRequest struct {
	ID              string            `json:"id,omitempty"`
	Policies        []string          `json:"policies,omitempty"`
	Metadata        map[string]string `json:"meta,omitempty"`
	Lease           string            `json:"lease,omitempty"`
	TTL             string            `json:"ttl,omitempty"`
	ExplicitMaxTTL  string            `json:"explicit_max_ttl,omitempty"`
	Period          string            `json:"period,omitempty"`
	NoParent        bool              `json:"no_parent,omitempty"`
	NoDefaultPolicy bool              `json:"no_default_policy,omitempty"`
	DisplayName     string            `json:"display_name"`
	NumUses         int               `json:"num_uses"`
	Renewable       *bool             `json:"renewable,omitempty"`
}
<commit_msg>Add golang api method for creating orphan tokens (#1834)<commit_after>package api

// TokenAuth is used to perform token backend operations on Vault
type TokenAuth struct {
	c *Client
}

// Token is used to return the client for token-backend API calls
func (a *Auth) Token() *TokenAuth {
	return &TokenAuth{c: a.c}
}

func (c *TokenAuth) Create(opts *TokenCreateRequest) (*Secret, error) {
	r := c.c.NewRequest("POST", "/v1/auth/token/create")
	if err := r.SetJSONBody(opts); err != nil {
		return nil, err
	}

	resp, err := c.c.RawRequest(r)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	return ParseSecret(resp.Body)
}

func (c *TokenAuth) CreateOrphan(opts *TokenCreateRequest) (*Secret, error) {
	r := c.c.NewRequest("POST", "/v1/auth/token/create-orphan")
	if err := r.SetJSONBody(opts); err != nil {
		return nil, err
	}

	resp, err := c.c.RawRequest(r)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	return ParseSecret(resp.Body)
}

func (c *TokenAuth) CreateWithRole(opts *TokenCreateRequest, roleName string) (*Secret, error) {
	r := c.c.NewRequest("POST", "/v1/auth/token/create/"+roleName)
	if err := r.SetJSONBody(opts); err != nil {
		return nil, err
	}

	resp, err := c.c.RawRequest(r)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	return ParseSecret(resp.Body)
}

func (c *TokenAuth) Lookup(token string) (*Secret, error) {
	r := c.c.NewRequest("POST", "/v1/auth/token/lookup")
	if err := r.SetJSONBody(map[string]interface{}{
		"token": token,
	}); err != nil {
		return nil, err
	}

	resp, err := c.c.RawRequest(r)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	return ParseSecret(resp.Body)
}

func (c *TokenAuth) LookupAccessor(accessor string) (*Secret, error) {
	r := c.c.NewRequest("POST", "/v1/auth/token/lookup-accessor")
	if err := r.SetJSONBody(map[string]interface{}{
		"accessor": accessor,
	}); err != nil {
		return nil, err
	}
	resp, err := c.c.RawRequest(r)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	return ParseSecret(resp.Body)
}

func (c *TokenAuth) LookupSelf() (*Secret, error) {
	r := c.c.NewRequest("GET", "/v1/auth/token/lookup-self")

	resp, err := c.c.RawRequest(r)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	return ParseSecret(resp.Body)
}

func (c *TokenAuth) Renew(token string, increment int) (*Secret, error) {
	r := c.c.NewRequest("PUT", "/v1/auth/token/renew")
	if err := r.SetJSONBody(map[string]interface{}{
		"token":     token,
		"increment": increment,
	}); err != nil {
		return nil, err
	}

	resp, err := c.c.RawRequest(r)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	return ParseSecret(resp.Body)
}

func (c *TokenAuth) RenewSelf(increment int) (*Secret, error) {
	r := c.c.NewRequest("PUT", "/v1/auth/token/renew-self")

	body := map[string]interface{}{"increment": increment}
	if err := r.SetJSONBody(body); err != nil {
		return nil, err
	}

	resp, err := c.c.RawRequest(r)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	return ParseSecret(resp.Body)
}

// RevokeAccessor revokes a token associated with the given accessor
// along with all the child tokens.
func (c *TokenAuth) RevokeAccessor(accessor string) error {
	r := c.c.NewRequest("POST", "/v1/auth/token/revoke-accessor")
	if err := r.SetJSONBody(map[string]interface{}{
		"accessor": accessor,
	}); err != nil {
		return err
	}
	resp, err := c.c.RawRequest(r)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	return nil
}

// RevokeOrphan revokes a token without revoking the tree underneath it (so
// child tokens are orphaned rather than revoked)
func (c *TokenAuth) RevokeOrphan(token string) error {
	r := c.c.NewRequest("PUT", "/v1/auth/token/revoke-orphan")
	if err := r.SetJSONBody(map[string]interface{}{
		"token": token,
	}); err != nil {
		return err
	}

	resp, err := c.c.RawRequest(r)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	return nil
}

// RevokeSelf revokes the token making the call
func (c *TokenAuth) RevokeSelf() error {
	r := c.c.NewRequest("PUT", "/v1/auth/token/revoke-self")
	resp, err := c.c.RawRequest(r)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	return nil
}

// RevokeTree is the "normal" revoke operation that revokes the given token and
// the entire tree underneath -- all of its child tokens, their child tokens,
// etc.
func (c *TokenAuth) RevokeTree(token string) error {
	r := c.c.NewRequest("PUT", "/v1/auth/token/revoke")
	if err := r.SetJSONBody(map[string]interface{}{
		"token": token,
	}); err != nil {
		return err
	}

	resp, err := c.c.RawRequest(r)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	return nil
}

// TokenCreateRequest is the options structure for creating a token.
type TokenCreateRequest struct {
	ID              string            `json:"id,omitempty"`
	Policies        []string          `json:"policies,omitempty"`
	Metadata        map[string]string `json:"meta,omitempty"`
	Lease           string            `json:"lease,omitempty"`
	TTL             string            `json:"ttl,omitempty"`
	ExplicitMaxTTL  string            `json:"explicit_max_ttl,omitempty"`
	Period          string            `json:"period,omitempty"`
	NoParent        bool              `json:"no_parent,omitempty"`
	NoDefaultPolicy bool              `json:"no_default_policy,omitempty"`
	DisplayName     string            `json:"display_name"`
	NumUses         int               `json:"num_uses"`
	Renewable       *bool             `json:"renewable,omitempty"`
}
<|endoftext|>


<commit_before>package scheduler

import (
	"fmt"
	"net/http"
	"net/url"
	"strings"

	"github.com/coreos/fleet/client"
	"github.com/coreos/fleet/schema"
)

// JobName represents the (unique) name of a job. The convention is <app>.<type>.<instance>.service:
//
//	my-sweet-app.v1.web.1
type JobName string

// Image represents a container image, which is tied to a repository.
type Image struct {
	Repo string
	ID   string
}

func (i Image) String() string {
	return fmt.Sprintf("%s:%s", i.Repo, i.ID)
}

// Execute represents a command to execute inside and image.
type Execute struct {
	Command string
	Image   Image
}

// Job is a job that can be scheduled.
type Job struct {
	// The unique name of the job.
	Name JobName

	// The name to expose for service discovery if applicable.
	Service string

	// A map of environment variables to set.
	Environment map[string]string

	// The command to run.
	Execute Execute
}

// State represents the state of a job.
type State int

// Various states that a job can be in.
const (
	StatePending State = iota
	StateRunning
	StateFailed
)

// JobState represents the status of a job.
type JobState struct {
	MachineID string
	Name      JobName
	State     string // TODO use State type
}

// Scheduler is an interface that represents something that can schedule Jobs
// onto the cluster.
type Scheduler interface {
	// Schedule schedules a job to run on the cluster.
	Schedule(*Job) error

	// Unschedule unschedules a job from the cluster by its name.
	Unschedule(JobName) error

	// List JobStates
	JobStates() ([]*JobState, error)
}

// NewScheduler is a factory method for generating a new Scheduler instance.
func NewScheduler(fleet string) (Scheduler, error) {
	if fleet == "" {
		return newScheduler(), nil
	}

	return newFleetScheduler(fleet)
}

// scheduler is a fake implementation of the Scheduler interface.
type scheduler struct{}

func newScheduler() *scheduler {
	return &scheduler{}
}

// Schedule implements Scheduler Schedule.
func (s *scheduler) Schedule(j *Job) error {
	return nil
}

// Unschedule implements Scheduler Unschedule.
func (s *scheduler) Unschedule(n JobName) error {
	return nil
}

func (s *scheduler) JobStates() ([]*JobState, error) {
	return nil, nil
}

// FleetScheduler is an implementation of the Scheduler interface that schedules
// jobs onto a coreos cluster via the fleet api.
type FleetScheduler struct {
	client client.API
}

// newFleetScheduler returns a new FleetScheduler with a configured fleet api
// client.
func newFleetScheduler(fleet string) (*FleetScheduler, error) {
	u, err := url.Parse(fleet)
	if err != nil {
		return nil, err
	}

	c, err := client.NewHTTPClient(
		http.DefaultClient,
		*u,
	)
	if err != nil {
		panic(err)
	}

	return &FleetScheduler{
		client: c,
	}, nil
}

// Schedule implements Scheduler Schedule and builds an appropritate systemd
// unit definition to run the container.
func (s *FleetScheduler) Schedule(j *Job) error {
	u := s.buildUnit(j)
	return s.client.CreateUnit(u)
}

func (s *FleetScheduler) Unschedule(n JobName) error {
	return s.client.DestroyUnit(unitNameFromJobName(n))
}

func (s *FleetScheduler) JobStates() ([]*JobState, error) {
	states, err := s.client.UnitStates()
	if err != nil {
		return nil, err
	}

	js := make([]*JobState, len(states))
	for i, st := range states {
		js[i] = &JobState{
			MachineID: st.MachineID,
			Name:      jobNameFromUnitName(st.Name),
			State:     st.SystemdActiveState,
		}
	}

	return js, nil
}

func unitNameFromJobName(n JobName) string {
	return string(n) + ".service"
}

func jobNameFromUnitName(un string) JobName {
	return JobName(strings.TrimSuffix(un, ".service"))
}

// buildUnit builds a Unit file that looks like this:
//
// [Unit]
// Description=app.v1.web.1
// After=discovery.service
//
// [Service]
// TimeoutStartSec=30m
//
// ExecStartPre=/bin/bash -c "/usr/bin/docker inspect remind101/app &> /dev/null || /usr/bin/docker pull remind101/app"
// ExecStartPre=/bin/bash -c "/usr/bin/docker rm app.v1.web.1 &> /dev/null; exit 0"
// ExecStart=/usr/bin/docker run --name app.v1.web.1 --rm -h %H remind101/app
// ExecStop=/usr/bin/docker stop app.v1.web.1

func (s *FleetScheduler) buildUnit(j *Job) *schema.Unit {
	img := j.Execute.Image.String()
	opts := []*schema.UnitOption{
		{
			Section: "Unit",
			Name:    "Description",
			Value:   string(j.Name),
		},
		{
			Section: "Unit",
			Name:    "After",
			Value:   "discovery.service",
		},
		{
			Section: "Service",
			Name:    "TimeoutStartSec",
			Value:   "30m",
		},
		{
			Section: "Service",
			Name:    "Restart",
			Value:   "on-failure",
		},
		{
			Section: "Service",
			Name:    "KillMode",
			Value:   "none",
		},
		{
			Section: "Service",
			Name:    "ExecStartPre",
			Value:   fmt.Sprintf(`-/bin/bash -c "/usr/bin/docker inspect %s &> /dev/null || /usr/bin/docker pull %s"`, img, img),
		},
		{
			Section: "Service",
			Name:    "ExecStartPre",
			Value:   fmt.Sprintf(`-/bin/bash -c "/usr/bin/docker rm %s &> /dev/null; exit 0"`, j.Name),
		},
		{
			Section: "Service",
			Name:    "ExecStart",
			Value:   fmt.Sprintf(`/usr/bin/docker run --name %s %s --rm -h %%H -P %s %s`, j.Name, env(j), img, j.Execute.Command),
		},
		{
			Section: "Service",
			Name:    "ExecStop",
			Value:   fmt.Sprintf(`/usr/bin/docker stop %s`, j.Name),
		},
		{
			Section: "X-Fleet",
			Name:    "MachineMetadata",
			Value:   "role=empire_minion",
		},
	}

	return &schema.Unit{
		Name:         string(j.Name) + ".service",
		DesiredState: "launched",
		Options:      opts,
	}
}

func env(j *Job) string {
	envs := []string{}
	for k, v := range j.Environment {
		envs = append(envs, fmt.Sprintf("-e %s=%s", k, v))
	}

	if j.Service != "" {
		envs = append(envs, fmt.Sprintf("-e SERVICE_NAME=%s", j.Service))
	}

	return strings.Join(envs, " ")
}
<commit_msg>Simpler ExecStartPre.<commit_after>package scheduler

import (
	"fmt"
	"net/http"
	"net/url"
	"strings"

	"github.com/coreos/fleet/client"
	"github.com/coreos/fleet/schema"
)

// JobName represents the (unique) name of a job. The convention is <app>.<type>.<instance>.service:
//
//	my-sweet-app.v1.web.1
type JobName string

// Image represents a container image, which is tied to a repository.
type Image struct {
	Repo string
	ID   string
}

func (i Image) String() string {
	return fmt.Sprintf("%s:%s", i.Repo, i.ID)
}

// Execute represents a command to execute inside and image.
type Execute struct {
	Command string
	Image   Image
}

// Job is a job that can be scheduled.
type Job struct {
	// The unique name of the job.
	Name JobName

	// The name to expose for service discovery if applicable.
	Service string

	// A map of environment variables to set.
	Environment map[string]string

	// The command to run.
	Execute Execute
}

// State represents the state of a job.
type State int

// Various states that a job can be in.
const (
	StatePending State = iota
	StateRunning
	StateFailed
)

// JobState represents the status of a job.
type JobState struct {
	MachineID string
	Name      JobName
	State     string // TODO use State type
}

// Scheduler is an interface that represents something that can schedule Jobs
// onto the cluster.
type Scheduler interface {
	// Schedule schedules a job to run on the cluster.
	Schedule(*Job) error

	// Unschedule unschedules a job from the cluster by its name.
	Unschedule(JobName) error

	// List JobStates
	JobStates() ([]*JobState, error)
}

// NewScheduler is a factory method for generating a new Scheduler instance.
func NewScheduler(fleet string) (Scheduler, error) {
	if fleet == "" {
		return newScheduler(), nil
	}

	return newFleetScheduler(fleet)
}

// scheduler is a fake implementation of the Scheduler interface.
type scheduler struct{}

func newScheduler() *scheduler {
	return &scheduler{}
}

// Schedule implements Scheduler Schedule.
func (s *scheduler) Schedule(j *Job) error {
	return nil
}

// Unschedule implements Scheduler Unschedule.
func (s *scheduler) Unschedule(n JobName) error {
	return nil
}

func (s *scheduler) JobStates() ([]*JobState, error) {
	return nil, nil
}

// FleetScheduler is an implementation of the Scheduler interface that schedules
// jobs onto a coreos cluster via the fleet api.
type FleetScheduler struct {
	client client.API
}

// newFleetScheduler returns a new FleetScheduler with a configured fleet api
// client.
func newFleetScheduler(fleet string) (*FleetScheduler, error) {
	u, err := url.Parse(fleet)
	if err != nil {
		return nil, err
	}

	c, err := client.NewHTTPClient(
		http.DefaultClient,
		*u,
	)
	if err != nil {
		panic(err)
	}

	return &FleetScheduler{
		client: c,
	}, nil
}

// Schedule implements Scheduler Schedule and builds an appropritate systemd
// unit definition to run the container.
func (s *FleetScheduler) Schedule(j *Job) error {
	u := s.buildUnit(j)
	return s.client.CreateUnit(u)
}

func (s *FleetScheduler) Unschedule(n JobName) error {
	return s.client.DestroyUnit(unitNameFromJobName(n))
}

func (s *FleetScheduler) JobStates() ([]*JobState, error) {
	states, err := s.client.UnitStates()
	if err != nil {
		return nil, err
	}

	js := make([]*JobState, len(states))
	for i, st := range states {
		js[i] = &JobState{
			MachineID: st.MachineID,
			Name:      jobNameFromUnitName(st.Name),
			State:     st.SystemdActiveState,
		}
	}

	return js, nil
}

func unitNameFromJobName(n JobName) string {
	return string(n) + ".service"
}

func jobNameFromUnitName(un string) JobName {
	return JobName(strings.TrimSuffix(un, ".service"))
}

// buildUnit builds a Unit file that looks like this:
//
// [Unit]
// Description=app.v1.web.1
// After=discovery.service
//
// [Service]
// TimeoutStartSec=30m
//
// ExecStartPre=-/bin/bash -c "/usr/bin/docker inspect remind101/app &> /dev/null || /usr/bin/docker pull remind101/app"
// ExecStartPre=-/bin/bash -c "/usr/bin/docker rm app.v1.web.1 &> /dev/null; exit 0"
// ExecStart=/usr/bin/docker run --name app.v1.web.1 --rm -h %H remind101/app
// ExecStop=/usr/bin/docker stop app.v1.web.1

func (s *FleetScheduler) buildUnit(j *Job) *schema.Unit {
	img := j.Execute.Image.String()
	opts := []*schema.UnitOption{
		{
			Section: "Unit",
			Name:    "Description",
			Value:   string(j.Name),
		},
		{
			Section: "Unit",
			Name:    "After",
			Value:   "discovery.service",
		},
		{
			Section: "Service",
			Name:    "TimeoutStartSec",
			Value:   "30m",
		},
		{
			Section: "Service",
			Name:    "Restart",
			Value:   "on-failure",
		},
		{
			Section: "Service",
			Name:    "KillMode",
			Value:   "none",
		},
		{
			Section: "Service",
			Name:    "ExecStartPre",
			Value:   fmt.Sprintf(`-/usr/bin/docker pull %s`, img),
		},
		{
			Section: "Service",
			Name:    "ExecStartPre",
			Value:   fmt.Sprintf(`-/usr/bin/docker rm %s &> /dev/null`, j.Name),
		},
		{
			Section: "Service",
			Name:    "ExecStart",
			Value:   fmt.Sprintf(`/usr/bin/docker run --name %s %s --rm -h %%H -P %s %s`, j.Name, env(j), img, j.Execute.Command),
		},
		{
			Section: "Service",
			Name:    "ExecStop",
			Value:   fmt.Sprintf(`/usr/bin/docker stop %s`, j.Name),
		},
		{
			Section: "X-Fleet",
			Name:    "MachineMetadata",
			Value:   "role=empire_minion",
		},
	}

	return &schema.Unit{
		Name:         string(j.Name) + ".service",
		DesiredState: "launched",
		Options:      opts,
	}
}

func env(j *Job) string {
	envs := []string{}
	for k, v := range j.Environment {
		envs = append(envs, fmt.Sprintf("-e %s=%s", k, v))
	}

	if j.Service != "" {
		envs = append(envs, fmt.Sprintf("-e SERVICE_NAME=%s", j.Service))
	}

	return strings.Join(envs, " ")
}
<|endoftext|>


<commit_before>package id3

import (
	"bytes"
	"encoding/binary"
	"io"
	"os"
)

type Decoder struct {
	r io.Reader
	h TagHeader
}

func NewDecoder(r io.Reader) *Decoder {
	return &Decoder{r: r}
}

// ParseHeader parses only the ID3 header.
func (d *Decoder) ParseHeader() (TagHeader, error) {
	header, err := d.readHeader()
	if err != nil {
		return TagHeader{}, err
	}

	d.h = header
	d.r = io.LimitReader(d.r, int64(header.Size)+tagHeaderSize)

	return header, nil
}

// Parse parses a tag.
//
// Parse will always return a valid tag. In the case of an error, the
// tag will be empty.
func (d *Decoder) Parse() (*Tag, error) {
	// TODO return how many bytes we read into the reader; so people
	// know where the audio begins
	tag := NewTag()
	header, err := d.ParseHeader()
	if err != nil {
		return tag, err
	}
	tag.Header = header

	// FIXME consider moving this to ParseHeader
	if header.Flags.ExtendedHeader() {
		panic("not implemented: cannot parse extended header")
	}

	if header.Flags.Unsynchronisation() {
		panic("not implemented: cannot parse unsynchronised tag")
	}

	for {
		frame, err := d.ParseFrame()
		if err != nil {
			if err == io.EOF {
				break
			}

			return tag, err
		}
		tag.Frames[frame.ID()] = append(tag.Frames[frame.ID()], frame)
	}

	if header.Version < 0x0400 {
		tag.upgrade()
	}

	return tag, nil
}

func readBinary(r io.Reader, args ...interface{}) (err error) {
	for _, arg := range args {
		err = binary.Read(r, binary.BigEndian, arg)
		if err != nil {
			break
		}
	}

	return
}

// readHeader reads an ID3v2 header. It expects the reader to be
// seeked to the beginning of the header.
func (d *Decoder) readHeader() (header TagHeader, err error) {
	var (
		bytes struct {
			Magic   [3]byte
			Version [2]byte
			Flags   byte
			Size    [4]byte
		}
	)

	err = binary.Read(d.r, binary.BigEndian, &bytes)
	if err != nil {
		return header, err
	}
	if bytes.Magic != [3]byte{0x49, 0x44, 0x33} {
		return TagHeader{}, notATagHeader{bytes.Magic}
	}
	version := Version(int16(bytes.Version[0])<<8 | int16(bytes.Version[1]))
	if bytes.Version[0] > 4 || bytes.Version[0] < 3 {
		return TagHeader{}, UnsupportedVersion{version}
	}

	header.Version = version
	header.Flags = HeaderFlags(bytes.Flags)
	header.Size = desynchsafeInt(bytes.Size)

	return header, nil
}

// ParseFrame reads the next ID3 frame.
func (d *Decoder) ParseFrame() (Frame, error) {
	var (
		headerBytes struct {
			ID    [4]byte
			Size  [4]byte
			Flags [2]byte
		}
		header FrameHeader
	)

	err := binary.Read(d.r, binary.BigEndian, &headerBytes)
	if err != nil {
		if err == io.ErrUnexpectedEOF {
			// If we couldn't read the header assume we were at the
			// end of the tag.
			return nil, io.EOF
		}
		return nil, err
	}

	// We're in the padding, return io.EOF
	if headerBytes.ID == [4]byte{0, 0, 0, 0} {
		return nil, io.EOF
	}

	for _, byte := range headerBytes.ID {
		// Allow 0-9
		if byte >= 48 && byte <= 57 {
			continue
		}

		// Allow A-Z
		if byte >= 65 && byte <= 90 {
			continue
		}

		return nil, NotAFrameHeader{headerBytes}
	}

	header.id = FrameType(headerBytes.ID[:])
	header.flags = FrameFlags(int16(headerBytes.Flags[0])<<8 | int16(headerBytes.Flags[1]))
	frameSize := desynchsafeInt(headerBytes.Size)

	if header.flags.Compressed() {
		panic("not implemented: cannot read compressed frame")
		// TODO: Read decompressed size (4 bytes)
	}

	if header.flags.Encrypted() {
		panic("not implemented: cannot read encrypted frame")
		// TODO: Read encryption method (1 byte)
	}

	if header.flags.Grouped() {
		panic("not implemented: cannot read grouped frame")
		// TODO: Read group identifier (1 byte)
	}

	if header.id[0] == 'T' && header.id != "TXXX" {
		var encoding Encoding
		frame := TextInformationFrame{FrameHeader: header}
		information := make([]byte, frameSize-1)
		err := readBinary(d.r, &encoding, &information)
		if err != nil {
			return nil, err
		}

		frame.Text = string(encoding.toUTF8(information))

		return frame, nil
	}

	if header.id[0] == 'W' && header.id != "WXXX" {
		frame := URLLinkFrame{FrameHeader: header}
		url := make([]byte, frameSize)
		_, err = d.r.Read(url)
		if err != nil {
			return nil, err
		}
		frame.URL = string(iso88591.toUTF8(url))

		return frame, nil
	}

	switch header.id {
	case "TXXX":
		return d.readTXXXFrame(header, frameSize)
	case "WXXX":
		return d.readWXXXFrame(header, frameSize)
	case "UFID":
		return d.readUFIDFrame(header, frameSize)
	case "COMM":
		return d.readCOMMFrame(header, frameSize)
	case "PRIV":
		return d.readPRIVFrame(header, frameSize)
	case "APIC":
		return d.readAPICFrame(header, frameSize)
	case "MCDI":
		return d.readMCDIFrame(header, frameSize)
	case "USLT":
		return d.readUFIDFrame(header, frameSize)
	default:
		data := make([]byte, frameSize)
		n, err := d.r.Read(data)

		return UnsupportedFrame{
			FrameHeader: header,
			Data:        data[:n],
		}, err
	}
}

// New creates a new file from an existing *os.File and Tag. If you
// plan to save tags the file needs to be opened read and write.
func NewFile(file *os.File, tag *Tag) (*File, error) {
	stat, err := file.Stat()
	if err != nil {
		return nil, err
	}

	f := &File{
		f:        file,
		fileSize: stat.Size(),
		Tag:      tag,
	}

	f.audioReader = io.NewSectionReader(file, tagHeaderSize+int64(tag.Header.Size), f.fileSize-int64(tag.Header.Size))

	return f, nil
}

// Open opens the file with the given name in RW mode and parses its
// tag. If there is no tag, (*File).HasTag() will return false.
//
// Call Close() to close the underlying *os.File when done.
func Open(name string) (*File, error) {
	// TODO improve documentation. HasTag() will only be false until
	// the first save; and there will be an empty tag to work with.
	f, err := os.OpenFile(name, os.O_RDWR, 0)
	if err != nil {
		return nil, err
	}

	d := NewDecoder(f)
	tag, err := d.Parse()
	if err != nil {
		if _, ok := err.(notATagHeader); !ok {
			return nil, err
		}
	}
	file, err := NewFile(f, tag)
	if err != nil {
		return nil, err
	}

	return file, nil
}

// HasTag returns true when the underlying file has a tag.
func (f *File) HasTag() bool {
	return f.Tag.Header.Version > 0
}

func (d *Decoder) readTXXXFrame(header FrameHeader, frameSize int) (Frame, error) {
	var encoding Encoding
	frame := UserTextInformationFrame{FrameHeader: header}
	rest := make([]byte, frameSize-1)

	err := readBinary(d.r, &encoding, &rest)
	if err != nil {
		return nil, err
	}
	parts := splitNullN(rest, encoding, 2)

	frame.Description = string(encoding.toUTF8(parts[0]))
	frame.Text = string(encoding.toUTF8(parts[1]))

	return frame, nil
}

func (d *Decoder) readWXXXFrame(header FrameHeader, frameSize int) (Frame, error) {
	var encoding Encoding
	frame := UserDefinedURLLinkFrame{FrameHeader: header}
	rest := make([]byte, frameSize-1)

	err := readBinary(d.r, &encoding, &rest)
	if err != nil {
		return nil, err
	}

	parts := splitNullN(rest, encoding, 2)
	frame.Description = string(encoding.toUTF8(parts[0]))
	frame.URL = string(iso88591ToUTF8(parts[1]))

	return frame, nil
}

func (d *Decoder) readUFIDFrame(header FrameHeader, frameSize int) (Frame, error) {
	frame := UniqueFileIdentifierFrame{FrameHeader: header}
	rest := make([]byte, frameSize)

	err := binary.Read(d.r, binary.BigEndian, rest)
	if err != nil {
		return nil, err
	}

	parts := bytes.SplitN(rest, []byte{0}, 2)
	frame.Owner = string(iso88591.toUTF8(parts[0]))
	frame.Identifier = parts[1]

	return frame, nil
}

func (d *Decoder) readCOMMFrame(header FrameHeader, frameSize int) (Frame, error) {
	frame := CommentFrame{FrameHeader: header}
	var (
		encoding Encoding
		language [3]byte
		rest     []byte
	)
	rest = make([]byte, frameSize-4)

	err := readBinary(d.r, &encoding, &language, &rest)
	if err != nil {
		return nil, err
	}

	parts := splitNullN(rest, encoding, 2)

	frame.Language = string(language[:])

	frame.Description = string(encoding.toUTF8(parts[0]))
	frame.Text = string(encoding.toUTF8(parts[1]))

	return frame, nil
}

func (d *Decoder) readPRIVFrame(header FrameHeader, frameSize int) (Frame, error) {
	frame := PrivateFrame{FrameHeader: header}
	data := make([]byte, frameSize)
	_, err := d.r.Read(data)
	if err != nil {
		return frame, err
	}

	parts := bytes.SplitN(data, nul, 2)
	frame.Owner = parts[0]
	frame.Data = parts[1]

	return frame, nil
}

func (d *Decoder) readAPICFrame(header FrameHeader, frameSize int) (Frame, error) {
	frame := PictureFrame{FrameHeader: header}
	var (
		encoding Encoding
		rest     []byte
	)
	rest = make([]byte, frameSize-1)
	err := readBinary(d.r, &encoding, &rest)
	if err != nil {
		return frame, err
	}

	parts1 := bytes.SplitN(rest, nul, 2)
	parts2 := splitNullN(parts1[1][1:], encoding, 2)

	frame.MIMEType = string(iso88591.toUTF8(parts1[0]))
	frame.PictureType = PictureType(parts1[1][0])
	frame.Description = string(encoding.toUTF8(parts2[0]))
	frame.Data = parts2[1]

	return frame, nil
}

func (d *Decoder) readMCDIFrame(header FrameHeader, frameSize int) (Frame, error) {
	frame := MusicCDIdentifierFrame{FrameHeader: header}
	frame.TOC = make([]byte, frameSize)
	_, err := d.r.Read(frame.TOC)
	return frame, err
}

func (d *Decoder) readUSLTFrame(header FrameHeader, frameSize int) (Frame, error) {
	frame := UnsynchronisedLyricsFrame{FrameHeader: header}
	var (
		encoding Encoding
		language [3]byte
		rest     []byte
	)
	rest = make([]byte, frameSize-4)

	err := readBinary(d.r, &encoding, &language, rest)
	if err != nil {
		return frame, err
	}

	parts := splitNullN(rest, encoding, 2)
	frame.Language = string(language[:])

	frame.Description = string(encoding.toUTF8(parts[0]))
	frame.Lyrics = string(encoding.toUTF8(parts[1]))

	return frame, nil
}
<commit_msg>handle reading only up to tag size more gracefully<commit_after>package id3

import (
	"bytes"
	"encoding/binary"
	"io"
	"os"
)

type Decoder struct {
	r io.Reader
	h TagHeader
}

func NewDecoder(r io.Reader) *Decoder {
	return &Decoder{r: r}
}

// ParseHeader parses only the ID3 header.
func (d *Decoder) ParseHeader() (TagHeader, error) {
	header, err := d.readHeader()
	if err != nil {
		return TagHeader{}, err
	}

	d.h = header
	// FIXME why are we adding tagHeaderSize? we already read the header
	d.r = io.LimitReader(d.r, int64(header.Size)+tagHeaderSize)

	return header, nil
}

func (d *Decoder) remaining() int64 {
	return d.r.(*io.LimitedReader).N
}

// Parse parses a tag.
//
// Parse will always return a valid tag. In the case of an error, the
// tag will be empty.
func (d *Decoder) Parse() (*Tag, error) {
	// TODO return how many bytes we read into the reader; so people
	// know where the audio begins
	tag := NewTag()
	header, err := d.ParseHeader()
	if err != nil {
		return tag, err
	}
	tag.Header = header

	// FIXME consider moving this to ParseHeader
	if header.Flags.ExtendedHeader() {
		panic("not implemented: cannot parse extended header")
	}

	if header.Flags.Unsynchronisation() {
		panic("not implemented: cannot parse unsynchronised tag")
	}

	for {
		frame, err := d.ParseFrame()
		if err != nil {
			if err == io.EOF {
				break
			}

			return tag, err
		}
		tag.Frames[frame.ID()] = append(tag.Frames[frame.ID()], frame)
	}

	if header.Version < 0x0400 {
		tag.upgrade()
	}

	return tag, nil
}

func readBinary(r io.Reader, args ...interface{}) (err error) {
	for _, arg := range args {
		err = binary.Read(r, binary.BigEndian, arg)
		if err != nil {
			break
		}
	}

	return
}

// readHeader reads an ID3v2 header. It expects the reader to be
// seeked to the beginning of the header.
func (d *Decoder) readHeader() (header TagHeader, err error) {
	var bytes struct {
		Magic   [3]byte
		Version [2]byte
		Flags   byte
		Size    [4]byte
	}

	err = binary.Read(d.r, binary.BigEndian, &bytes)
	if err != nil {
		return header, err
	}
	if bytes.Magic != [3]byte{0x49, 0x44, 0x33} {
		return TagHeader{}, notATagHeader{bytes.Magic}
	}
	version := Version(int16(bytes.Version[0])<<8 | int16(bytes.Version[1]))
	if bytes.Version[0] > 4 || bytes.Version[0] < 3 {
		return TagHeader{}, UnsupportedVersion{version}
	}

	header.Version = version
	header.Flags = HeaderFlags(bytes.Flags)
	header.Size = desynchsafeInt(bytes.Size)

	return header, nil
}

// ParseFrame reads the next ID3 frame.
func (d *Decoder) ParseFrame() (Frame, error) {
	if d.remaining() == 0 {
		return nil, io.EOF
	}

	var (
		headerBytes struct {
			ID    [4]byte
			Size  [4]byte
			Flags [2]byte
		}
		header FrameHeader
	)

	err := binary.Read(d.r, binary.BigEndian, &headerBytes)
	if err != nil {
		return nil, err
	}

	// We're in the padding, return io.EOF
	if headerBytes.ID == [4]byte{0, 0, 0, 0} {
		return nil, io.EOF
	}

	for _, byte := range headerBytes.ID {
		// Allow 0-9
		if byte >= 48 && byte <= 57 {
			continue
		}

		// Allow A-Z
		if byte >= 65 && byte <= 90 {
			continue
		}

		return nil, NotAFrameHeader{headerBytes}
	}

	header.id = FrameType(headerBytes.ID[:])
	header.flags = FrameFlags(int16(headerBytes.Flags[0])<<8 | int16(headerBytes.Flags[1]))
	frameSize := desynchsafeInt(headerBytes.Size)

	if header.flags.Compressed() {
		panic("not implemented: cannot read compressed frame")
		// TODO: Read decompressed size (4 bytes)
	}

	if header.flags.Encrypted() {
		panic("not implemented: cannot read encrypted frame")
		// TODO: Read encryption method (1 byte)
	}

	if header.flags.Grouped() {
		panic("not implemented: cannot read grouped frame")
		// TODO: Read group identifier (1 byte)
	}

	if header.id[0] == 'T' && header.id != "TXXX" {
		var encoding Encoding
		frame := TextInformationFrame{FrameHeader: header}
		information := make([]byte, frameSize-1)
		err := readBinary(d.r, &encoding, &information)
		if err != nil {
			return nil, err
		}

		frame.Text = string(encoding.toUTF8(information))

		return frame, nil
	}

	if header.id[0] == 'W' && header.id != "WXXX" {
		frame := URLLinkFrame{FrameHeader: header}
		url := make([]byte, frameSize)
		_, err = d.r.Read(url)
		if err != nil {
			return nil, err
		}
		frame.URL = string(iso88591.toUTF8(url))

		return frame, nil
	}

	switch header.id {
	case "TXXX":
		return d.readTXXXFrame(header, frameSize)
	case "WXXX":
		return d.readWXXXFrame(header, frameSize)
	case "UFID":
		return d.readUFIDFrame(header, frameSize)
	case "COMM":
		return d.readCOMMFrame(header, frameSize)
	case "PRIV":
		return d.readPRIVFrame(header, frameSize)
	case "APIC":
		return d.readAPICFrame(header, frameSize)
	case "MCDI":
		return d.readMCDIFrame(header, frameSize)
	case "USLT":
		return d.readUFIDFrame(header, frameSize)
	default:
		data := make([]byte, frameSize)
		n, err := d.r.Read(data)

		return UnsupportedFrame{
			FrameHeader: header,
			Data:        data[:n],
		}, err
	}
}

// New creates a new file from an existing *os.File and Tag. If you
// plan to save tags the file needs to be opened read and write.
func NewFile(file *os.File, tag *Tag) (*File, error) {
	stat, err := file.Stat()
	if err != nil {
		return nil, err
	}

	f := &File{
		f:        file,
		fileSize: stat.Size(),
		Tag:      tag,
	}

	f.audioReader = io.NewSectionReader(file, tagHeaderSize+int64(tag.Header.Size), f.fileSize-int64(tag.Header.Size))

	return f, nil
}

// Open opens the file with the given name in RW mode and parses its
// tag. If there is no tag, (*File).HasTag() will return false.
//
// Call Close() to close the underlying *os.File when done.
func Open(name string) (*File, error) {
	// TODO improve documentation. HasTag() will only be false until
	// the first save; and there will be an empty tag to work with.
	f, err := os.OpenFile(name, os.O_RDWR, 0)
	if err != nil {
		return nil, err
	}

	d := NewDecoder(f)
	tag, err := d.Parse()
	if err != nil {
		if _, ok := err.(notATagHeader); !ok {
			return nil, err
		}
	}
	file, err := NewFile(f, tag)
	if err != nil {
		return nil, err
	}

	return file, nil
}

// HasTag returns true when the underlying file has a tag.
func (f *File) HasTag() bool {
	return f.Tag.Header.Version > 0
}

func (d *Decoder) readTXXXFrame(header FrameHeader, frameSize int) (Frame, error) {
	var encoding Encoding
	frame := UserTextInformationFrame{FrameHeader: header}
	rest := make([]byte, frameSize-1)

	err := readBinary(d.r, &encoding, &rest)
	if err != nil {
		return nil, err
	}
	parts := splitNullN(rest, encoding, 2)

	frame.Description = string(encoding.toUTF8(parts[0]))
	frame.Text = string(encoding.toUTF8(parts[1]))

	return frame, nil
}

func (d *Decoder) readWXXXFrame(header FrameHeader, frameSize int) (Frame, error) {
	var encoding Encoding
	frame := UserDefinedURLLinkFrame{FrameHeader: header}
	rest := make([]byte, frameSize-1)

	err := readBinary(d.r, &encoding, &rest)
	if err != nil {
		return nil, err
	}

	parts := splitNullN(rest, encoding, 2)
	frame.Description = string(encoding.toUTF8(parts[0]))
	frame.URL = string(iso88591ToUTF8(parts[1]))

	return frame, nil
}

func (d *Decoder) readUFIDFrame(header FrameHeader, frameSize int) (Frame, error) {
	frame := UniqueFileIdentifierFrame{FrameHeader: header}
	rest := make([]byte, frameSize)

	err := binary.Read(d.r, binary.BigEndian, rest)
	if err != nil {
		return nil, err
	}

	parts := bytes.SplitN(rest, []byte{0}, 2)
	frame.Owner = string(iso88591.toUTF8(parts[0]))
	frame.Identifier = parts[1]

	return frame, nil
}

func (d *Decoder) readCOMMFrame(header FrameHeader, frameSize int) (Frame, error) {
	frame := CommentFrame{FrameHeader: header}
	var (
		encoding Encoding
		language [3]byte
		rest     []byte
	)
	rest = make([]byte, frameSize-4)

	err := readBinary(d.r, &encoding, &language, &rest)
	if err != nil {
		return nil, err
	}

	parts := splitNullN(rest, encoding, 2)

	frame.Language = string(language[:])

	frame.Description = string(encoding.toUTF8(parts[0]))
	frame.Text = string(encoding.toUTF8(parts[1]))

	return frame, nil
}

func (d *Decoder) readPRIVFrame(header FrameHeader, frameSize int) (Frame, error) {
	frame := PrivateFrame{FrameHeader: header}
	data := make([]byte, frameSize)
	_, err := d.r.Read(data)
	if err != nil {
		return frame, err
	}

	parts := bytes.SplitN(data, nul, 2)
	frame.Owner = parts[0]
	frame.Data = parts[1]

	return frame, nil
}

func (d *Decoder) readAPICFrame(header FrameHeader, frameSize int) (Frame, error) {
	frame := PictureFrame{FrameHeader: header}
	var (
		encoding Encoding
		rest     []byte
	)
	rest = make([]byte, frameSize-1)
	err := readBinary(d.r, &encoding, &rest)
	if err != nil {
		return frame, err
	}

	parts1 := bytes.SplitN(rest, nul, 2)
	parts2 := splitNullN(parts1[1][1:], encoding, 2)

	frame.MIMEType = string(iso88591.toUTF8(parts1[0]))
	frame.PictureType = PictureType(parts1[1][0])
	frame.Description = string(encoding.toUTF8(parts2[0]))
	frame.Data = parts2[1]

	return frame, nil
}

func (d *Decoder) readMCDIFrame(header FrameHeader, frameSize int) (Frame, error) {
	frame := MusicCDIdentifierFrame{FrameHeader: header}
	frame.TOC = make([]byte, frameSize)
	_, err := d.r.Read(frame.TOC)
	return frame, err
}

func (d *Decoder) readUSLTFrame(header FrameHeader, frameSize int) (Frame, error) {
	frame := UnsynchronisedLyricsFrame{FrameHeader: header}
	var (
		encoding Encoding
		language [3]byte
		rest     []byte
	)
	rest = make([]byte, frameSize-4)

	err := readBinary(d.r, &encoding, &language, rest)
	if err != nil {
		return frame, err
	}

	parts := splitNullN(rest, encoding, 2)
	frame.Language = string(language[:])

	frame.Description = string(encoding.toUTF8(parts[0]))
	frame.Lyrics = string(encoding.toUTF8(parts[1]))

	return frame, nil
}
<|endoftext|>