<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"net"
	"rtsengine"
	"time"

	"github.com/JoelOtter/termloop"

	tl "github.com/JoelOtter/termloop"
)

// ScreenController on our screen
type ScreenController struct {
	*tl.Entity
	level *tl.BaseLevel

	Xdiff int
	Ydiff int

	screenWidth  int
	screenHeight int

	ui *ReferenceUI

	// Last Unit Selection Information
	UnitID int
	MouseX int
	MouseY int
}

// Tick satisfies Entity interface.
func (controller *ScreenController) Tick(event tl.Event) {
	switch event.Key { // If so, switch on the pressed key.
	case tl.KeyArrowRight:
		controller.Ydiff = 1
		controller.Xdiff = 0
		controller.ui.scrollView()

	case tl.KeyArrowLeft:
		controller.Ydiff = -1
		controller.Xdiff = 0
		controller.ui.scrollView()

	case tl.KeyArrowUp:
		controller.Xdiff = -1
		controller.Ydiff = 0
		controller.ui.scrollView()

	case tl.KeyArrowDown:
		controller.Xdiff = 1
		controller.Ydiff = 0
		controller.ui.scrollView()

	case tl.MouseRight, tl.MouseLeft:
		controller.HandleMouseDown(&event)
	}
	/*
			switch event.Type {
			case tl.EventResize:
				log.Print("resized")
				//fmt.Printf("We resized to width(%d) height(%d)", event.Width, event.Height)
				//panic(nil)
			default:
				break
			}

		}
	*/
}

// HandleMouseDown for entire screen.
func (controller *ScreenController) HandleMouseDown(event *tl.Event) {
	acre := controller.ui.findAcre(event.MouseX, event.MouseY)

	// Does our acre exist?
	if acre != nil {
		if acre.UnitID > 0 { // has a Unit?
			// Moveable unit?
			switch acre.Unit {
			case rtsengine.UnitInfantry,
				rtsengine.UnitCavalry,
				rtsengine.UnitPeasant,
				rtsengine.UnitShip,
				rtsengine.UnitCatapult,
				rtsengine.UnitArcher:
				// New Unit Selection?
				//fmt.Println("Found the unit")
				if controller.UnitID != acre.UnitID {
					controller.UnitID = acre.UnitID
					controller.MouseX = event.MouseX
					controller.MouseY = event.MouseY
				}
			}
		}
	} else if controller.UnitID > 0 { // was there a previous selection of a unit?
		id := controller.UnitID
		controller.UnitID = 0
		controller.MouseX = event.MouseX
		controller.MouseY = event.MouseY
		//fmt.Println("Path to unit")
		controller.ui.pathUnitToLocation(id, controller.MouseX, controller.MouseY)
	}
}

// Draw the screen.
func (controller *ScreenController) Draw(screen *tl.Screen) {
	screenWidth, screenHeight := screen.Size()

	controller.screenWidth = screenWidth
	controller.screenHeight = screenHeight
	// We need to make sure and call Draw on the underlying Entity.
	controller.Entity.Draw(screen)
}

// Acre is an entity on our screen.
type Acre struct {
	*tl.Entity
	level *tl.BaseLevel

	// Terrain
	LocalTerrain rtsengine.Terrain

	// Unit information if any
	UnitID int
	Unit   rtsengine.UnitType
	Life   int

	// Screen coordinates
	X int
	Y int
}

// ReferenceUI which is the master struct for our Game UI
type ReferenceUI struct {
	// TermLoop
	game             *tl.Game
	level            *tl.BaseLevel
	screenController ScreenController

	// Communication
	Connection  net.Conn
	JSONDecoder *json.Decoder
	JSONEncoder *json.Encoder

	// Current viewable acres
	acreMap map[int]*Acre

	// This player information
	playerName string // name of this player
	playerID   int    // unique ID of this player
}

// Start will instantiate the UI and attempt to talk to a rtsengine.
func (ui *ReferenceUI) Start() {
	//time.Sleep(time.Second * 60)

	ui.acreMap = make(map[int]*Acre)

	ui.game = termloop.NewGame()

	screen := ui.game.Screen()
	screenWidth, screenHeight := screen.Size()
	fmt.Printf("w %d h %d\n", screenWidth, screenHeight)

	ui.level = tl.NewBaseLevel(tl.Cell{
		Bg: tl.ColorGreen,
		Fg: tl.ColorBlack,
		Ch: ' ',
	})

	ui.screenController = ScreenController{
		Entity:       tl.NewEntity(1, 1, 1, 1),
		level:        ui.level,
		screenWidth:  screenWidth,
		screenHeight: screenHeight,
		ui:           ui,
	}

	// Set the character at position (0, 0) on the entity.
	ui.screenController.SetCell(0, 0, &tl.Cell{Fg: tl.ColorRed, Ch: ' '})
	ui.level.AddEntity(&ui.screenController)

	// Dial up the rtsengine.
	conn, err := net.Dial("tcp", "localhost:8080")
	if err != nil {
		fmt.Print(err)
		return
	}

	ui.Connection = conn
	ui.JSONEncoder = json.NewEncoder(ui.Connection)
	ui.JSONDecoder = json.NewDecoder(ui.Connection)

	// Lake
	//level.AddEntity(tl.NewRectangle(10, 10, 50, 20, tl.ColorBlue))

	// Start the game.
	ui.game.Screen().SetLevel(ui.level)

	go ui.communicationPreamble()
	go ui.listenForWireCommands()

	ui.game.Start()

	//time.Sleep(time.Second * 60)
}

// listenForWireCommands will listen for commands from the rtsengine.
func (ui *ReferenceUI) listenForWireCommands() {
	time.Sleep(time.Second * 2)
	for {
		var packetArray []rtsengine.WirePacket
		if err := ui.JSONDecoder.Decode(&packetArray); err == io.EOF {
			fmt.Println("\n\nEOF was detected. Connection lost.")
			return
		}

		switch packetArray[0].Command {
		case rtsengine.WhoAmI:
			ui.playerName = packetArray[0].PlayerName
			ui.playerID = packetArray[0].PlayerID

		case rtsengine.MoveUnit:
			acre, ok := ui.acreMap[packetArray[0].UnitID]
			if ok {
				acre.X = packetArray[0].CurrentColumn
				acre.Y = packetArray[0].CurrentRow
				acre.SetPosition(acre.X, acre.Y)
			}

		// Set the View to equial the entire world. Used for testing.
		case rtsengine.FullView:
			ui.screenController.screenWidth = packetArray[0].ViewWidth
			ui.screenController.screenHeight = packetArray[0].ViewHeight

		case rtsengine.PartialRefreshPlayerToUI:
			ui.handleRefreshPlayerToUI(packetArray)
		}

	}
}

func (ui *ReferenceUI) handleRefreshPlayerToUI(packetArray []rtsengine.WirePacket) {
	var m map[int]*Acre
	m = make(map[int]*Acre)

	for _, p := range packetArray {
		acre, ok := ui.acreMap[p.UnitID]
		if ok {
			delete(ui.acreMap, p.UnitID)
			acre.X = p.CurrentColumn
			acre.Y = p.CurrentRow
			acre.SetPosition(acre.X, acre.Y)
		} else {
			acre = &Acre{
				Entity:       tl.NewEntity(p.CurrentColumn, p.CurrentRow, 1, 1),
				level:        ui.level,
				LocalTerrain: p.LocalTerrain,
				X:            p.CurrentColumn,
				Y:            p.CurrentRow,
				Life:         p.Life,
				UnitID:       p.UnitID,
				Unit:         p.Unit,
			}

			var cell tl.Cell

			switch acre.LocalTerrain {
			case rtsengine.Snow:
				cell = tl.Cell{Fg: tl.ColorWhite, Ch: ' '}

			case rtsengine.Sand:
				cell = tl.Cell{Fg: tl.ColorYellow, Ch: ' '}

			case rtsengine.Water:
				cell = tl.Cell{Fg: tl.ColorBlue, Ch: ' '}

			case rtsengine.Mountains:
				//r, _ := utf8.DecodeRuneInString("\xF0\x9F\x97\xBB")
				cell = tl.Cell{Fg: tl.ColorBlack, Ch: '^'}

			case rtsengine.Trees:
				//r, _ := utf8.DecodeRuneInString("\xF0\x9F\x8C\xB2")
				//r, _ := utf8.DecodeRuneInString("\xC2\xAE")
				cell = tl.Cell{Fg: tl.ColorWhite, Ch: 'T'}
			}

			switch acre.Unit {
			case rtsengine.UnitFarm:
				cell = tl.Cell{Fg: tl.ColorYellow, Ch: 'F'}

			case rtsengine.UnitCavalry:
				cell = tl.Cell{Fg: tl.ColorCyan, Ch: 'Z'}

			case rtsengine.UnitInfantry:
				cell = tl.Cell{Fg: tl.ColorBlue, Ch: 'S'}

			case rtsengine.UnitArmory:
				cell = tl.Cell{Fg: tl.ColorBlue, Ch: 'A'}

			case rtsengine.UnitCastle:
				cell = tl.Cell{Fg: tl.ColorBlue, Ch: 'C'}

			case rtsengine.UnitGoldMine:
				cell = tl.Cell{Fg: tl.ColorYellow, Ch: 'G'}

			case rtsengine.UnitHomeStead:
				cell = tl.Cell{Fg: tl.ColorBlue, Ch: 'H'}

			case rtsengine.UnitPeasant:
				cell = tl.Cell{Fg: tl.ColorBlue, Ch: 'P'}

			case rtsengine.UnitStoneQuarry:
				cell = tl.Cell{Fg: tl.ColorWhite, Ch: 's'}

			case rtsengine.UnitShip:
				cell = tl.Cell{Fg: tl.ColorBlue, Ch: 's'}

			case rtsengine.UnitTower:
				cell = tl.Cell{Fg: tl.ColorBlue, Ch: 't'}

			case rtsengine.UnitWoodPile:
				cell = tl.Cell{Fg: tl.ColorBlack, Ch: 'w'}

			case rtsengine.UnitFence:
				//r, _ := utf8.DecodeRuneInString("\xE2\xAC\x9B")
				cell = tl.Cell{Fg: tl.ColorBlack, Ch: 'X'}
			}
			acre.SetCell(0, 0, &cell)
			ui.level.AddEntity(acre)
		}
		m[p.UnitID] = acre
	}

	// Remove anything left over. That means it is no longer within the view.
	for _, v := range ui.acreMap {
		ui.level.RemoveEntity(v)
	}

	ui.acreMap = m
}

// communicationPreamble will start the communication with the rtsengine.
func (ui *ReferenceUI) communicationPreamble() {
	// give the UI some time to initialize.
	time.Sleep(time.Second * 2)

	var packet rtsengine.WirePacket

	// WhoAmI? Get user information for this connection.
	packet.Command = rtsengine.WhoAmI
	err := ui.JSONEncoder.Encode(&packet)
	if err != nil {
		fmt.Println("Unexpected wire error", err)
		return
	}

	// Send full View to set our UI to the entire view of the game for testing.
	packet.Command = rtsengine.SetView

	// They are flipped for our engine.
	packet.ViewHeight, packet.ViewWidth = ui.game.Screen().Size()
	err = ui.JSONEncoder.Encode(&packet)
	if err != nil {
		fmt.Println("Unexpected wire error", err)
		return
	}

	// Force a partial refresh to place the initial acres and units on our screen.
	packet.Command = rtsengine.PartialRefreshPlayerToUI
	err = ui.JSONEncoder.Encode(&packet)
	if err != nil {
		fmt.Println("Unexpected wire error", err)
		return
	}

}

// pathUnitToLocation will request that unitID be pathed to X,Y
func (ui *ReferenceUI) pathUnitToLocation(UnitID int, X int, Y int) {
	var packet rtsengine.WirePacket

	// Send full View to set our UI to the entire view of the game for testing.
	packet.Command = rtsengine.PathUnitToLocation
	packet.CurrentRow = Y
	packet.CurrentColumn = X
	packet.UnitID = UnitID
	err := ui.JSONEncoder.Encode(&packet)
	if err != nil {
		fmt.Println("Unexpected wire error", err)
		return
	}
}

// scrollView will scroll this player view within the world.
func (ui *ReferenceUI) scrollView() {
	var packet rtsengine.WirePacket

	// Send full View to set our UI to the entire view of the game for testing.
	packet.Command = rtsengine.ScrollView
	packet.CurrentRow = ui.screenController.Xdiff
	packet.CurrentColumn = ui.screenController.Ydiff
	err := ui.JSONEncoder.Encode(&packet)
	if err != nil {
		fmt.Println("Unexpected wire error", err)
		return
	}
}

// findAcre will find the acre at X and Y and return nil if not found.
// Presently this is an O(N) search.
func (ui *ReferenceUI) findAcre(X int, Y int) *Acre {
	for _, v := range ui.acreMap {
		if v.X == X && v.Y == Y {
			return v
		}
	}
	return nil
}

// Assume grass is the default.
func main() {
	ui := ReferenceUI{}

	ui.Start()
}
<commit_msg>Renamed X/Y in the Acre structure to Row Column to alley confusion.<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"net"
	"rtsengine"
	"time"

	"github.com/JoelOtter/termloop"

	tl "github.com/JoelOtter/termloop"
)

// ScreenController on our screen
type ScreenController struct {
	*tl.Entity
	level *tl.BaseLevel

	Xdiff int
	Ydiff int

	screenWidth  int
	screenHeight int

	ui *ReferenceUI

	// Last Unit Selection Information
	UnitID int
	MouseX int
	MouseY int
}

// Tick satisfies Entity interface.
func (controller *ScreenController) Tick(event tl.Event) {
	switch event.Key { // If so, switch on the pressed key.
	case tl.KeyArrowRight:
		controller.Ydiff = 1
		controller.Xdiff = 0
		controller.ui.scrollView()

	case tl.KeyArrowLeft:
		controller.Ydiff = -1
		controller.Xdiff = 0
		controller.ui.scrollView()

	case tl.KeyArrowUp:
		controller.Xdiff = -1
		controller.Ydiff = 0
		controller.ui.scrollView()

	case tl.KeyArrowDown:
		controller.Xdiff = 1
		controller.Ydiff = 0
		controller.ui.scrollView()

	case tl.MouseRight, tl.MouseLeft:
		controller.HandleMouseDown(&event)
	}
	/*
			switch event.Type {
			case tl.EventResize:
				log.Print("resized")
				//fmt.Printf("We resized to width(%d) height(%d)", event.Width, event.Height)
				//panic(nil)
			default:
				break
			}

		}
	*/
}

// HandleMouseDown for entire screen.
func (controller *ScreenController) HandleMouseDown(event *tl.Event) {
	acre := controller.ui.findAcre(event.MouseX, event.MouseY)

	// Does our acre exist?
	if acre != nil {
		if acre.UnitID > 0 { // has a Unit?
			// Moveable unit?
			switch acre.Unit {
			case rtsengine.UnitInfantry,
				rtsengine.UnitCavalry,
				rtsengine.UnitPeasant,
				rtsengine.UnitShip,
				rtsengine.UnitCatapult,
				rtsengine.UnitArcher:
				// New Unit Selection?
				//fmt.Println("Found the unit")
				if controller.UnitID != acre.UnitID {
					controller.UnitID = acre.UnitID
					controller.MouseX = event.MouseX
					controller.MouseY = event.MouseY
				}
			}
		}
	} else if controller.UnitID > 0 { // was there a previous selection of a unit?
		id := controller.UnitID
		controller.UnitID = 0
		controller.MouseX = event.MouseX
		controller.MouseY = event.MouseY
		//fmt.Println("Path to unit")
		controller.ui.pathUnitToLocation(id, controller.MouseX, controller.MouseY)
	}
}

// Draw the screen.
func (controller *ScreenController) Draw(screen *tl.Screen) {
	screenWidth, screenHeight := screen.Size()

	controller.screenWidth = screenWidth
	controller.screenHeight = screenHeight
	// We need to make sure and call Draw on the underlying Entity.
	controller.Entity.Draw(screen)
}

// Acre is an entity on our screen.
type Acre struct {
	*tl.Entity
	level *tl.BaseLevel

	// Terrain
	LocalTerrain rtsengine.Terrain

	// Unit information if any
	UnitID int
	Unit   rtsengine.UnitType
	Life   int

	// Screen coordinates
	Column int
	Row    int
}

// ReferenceUI which is the master struct for our Game UI
type ReferenceUI struct {
	// TermLoop
	game             *tl.Game
	level            *tl.BaseLevel
	screenController ScreenController

	// Communication
	Connection  net.Conn
	JSONDecoder *json.Decoder
	JSONEncoder *json.Encoder

	// Current viewable acres
	acreMap map[int]*Acre

	// This player information
	playerName string // name of this player
	playerID   int    // unique ID of this player
}

// Start will instantiate the UI and attempt to talk to a rtsengine.
func (ui *ReferenceUI) Start() {
	//time.Sleep(time.Second * 60)

	ui.acreMap = make(map[int]*Acre)

	ui.game = termloop.NewGame()

	screen := ui.game.Screen()
	screenWidth, screenHeight := screen.Size()
	fmt.Printf("w %d h %d\n", screenWidth, screenHeight)

	ui.level = tl.NewBaseLevel(tl.Cell{
		Bg: tl.ColorGreen,
		Fg: tl.ColorBlack,
		Ch: ' ',
	})

	ui.screenController = ScreenController{
		Entity:       tl.NewEntity(1, 1, 1, 1),
		level:        ui.level,
		screenWidth:  screenWidth,
		screenHeight: screenHeight,
		ui:           ui,
	}

	// Set the character at position (0, 0) on the entity.
	ui.screenController.SetCell(0, 0, &tl.Cell{Fg: tl.ColorRed, Ch: ' '})
	ui.level.AddEntity(&ui.screenController)

	// Dial up the rtsengine.
	conn, err := net.Dial("tcp", "localhost:8080")
	if err != nil {
		fmt.Print(err)
		return
	}

	ui.Connection = conn
	ui.JSONEncoder = json.NewEncoder(ui.Connection)
	ui.JSONDecoder = json.NewDecoder(ui.Connection)

	// Lake
	//level.AddEntity(tl.NewRectangle(10, 10, 50, 20, tl.ColorBlue))

	// Start the game.
	ui.game.Screen().SetLevel(ui.level)

	go ui.communicationPreamble()
	go ui.listenForWireCommands()

	ui.game.Start()

	//time.Sleep(time.Second * 60)
}

// listenForWireCommands will listen for commands from the rtsengine.
func (ui *ReferenceUI) listenForWireCommands() {
	time.Sleep(time.Second * 2)
	for {
		var packetArray []rtsengine.WirePacket
		if err := ui.JSONDecoder.Decode(&packetArray); err == io.EOF {
			fmt.Println("\n\nEOF was detected. Connection lost.")
			return
		}

		switch packetArray[0].Command {
		case rtsengine.WhoAmI:
			ui.playerName = packetArray[0].PlayerName
			ui.playerID = packetArray[0].PlayerID

		case rtsengine.MoveUnit:
			acre, ok := ui.acreMap[packetArray[0].UnitID]
			if ok {
				acre.Column = packetArray[0].CurrentColumn
				acre.Row = packetArray[0].CurrentRow
				acre.SetPosition(acre.Column, acre.Row)
			}

		// Set the View to equial the entire world. Used for testing.
		case rtsengine.FullView:
			ui.screenController.screenWidth = packetArray[0].ViewWidth
			ui.screenController.screenHeight = packetArray[0].ViewHeight

		case rtsengine.PartialRefreshPlayerToUI:
			ui.handleRefreshPlayerToUI(packetArray)
		}

	}
}

func (ui *ReferenceUI) handleRefreshPlayerToUI(packetArray []rtsengine.WirePacket) {
	var m map[int]*Acre
	m = make(map[int]*Acre)

	for _, p := range packetArray {
		acre, ok := ui.acreMap[p.UnitID]
		if ok {
			delete(ui.acreMap, p.UnitID)
			acre.Column = p.CurrentColumn
			acre.Row = p.CurrentRow
			acre.SetPosition(acre.Column, acre.Row)
		} else {
			acre = &Acre{
				Entity:       tl.NewEntity(p.CurrentColumn, p.CurrentRow, 1, 1),
				level:        ui.level,
				LocalTerrain: p.LocalTerrain,
				Column:       p.CurrentColumn,
				Row:          p.CurrentRow,
				Life:         p.Life,
				UnitID:       p.UnitID,
				Unit:         p.Unit,
			}

			var cell tl.Cell

			switch acre.LocalTerrain {
			case rtsengine.Snow:
				cell = tl.Cell{Fg: tl.ColorWhite, Ch: ' '}

			case rtsengine.Sand:
				cell = tl.Cell{Fg: tl.ColorYellow, Ch: ' '}

			case rtsengine.Water:
				cell = tl.Cell{Fg: tl.ColorBlue, Ch: ' '}

			case rtsengine.Mountains:
				//r, _ := utf8.DecodeRuneInString("\xF0\x9F\x97\xBB")
				cell = tl.Cell{Fg: tl.ColorBlack, Ch: '^'}

			case rtsengine.Trees:
				//r, _ := utf8.DecodeRuneInString("\xF0\x9F\x8C\xB2")
				//r, _ := utf8.DecodeRuneInString("\xC2\xAE")
				cell = tl.Cell{Fg: tl.ColorWhite, Ch: 'T'}
			}

			switch acre.Unit {
			case rtsengine.UnitFarm:
				cell = tl.Cell{Fg: tl.ColorYellow, Ch: 'F'}

			case rtsengine.UnitCavalry:
				cell = tl.Cell{Fg: tl.ColorCyan, Ch: 'Z'}

			case rtsengine.UnitInfantry:
				cell = tl.Cell{Fg: tl.ColorBlue, Ch: 'S'}

			case rtsengine.UnitArmory:
				cell = tl.Cell{Fg: tl.ColorBlue, Ch: 'A'}

			case rtsengine.UnitCastle:
				cell = tl.Cell{Fg: tl.ColorBlue, Ch: 'C'}

			case rtsengine.UnitGoldMine:
				cell = tl.Cell{Fg: tl.ColorYellow, Ch: 'G'}

			case rtsengine.UnitHomeStead:
				cell = tl.Cell{Fg: tl.ColorBlue, Ch: 'H'}

			case rtsengine.UnitPeasant:
				cell = tl.Cell{Fg: tl.ColorBlue, Ch: 'P'}

			case rtsengine.UnitStoneQuarry:
				cell = tl.Cell{Fg: tl.ColorWhite, Ch: 's'}

			case rtsengine.UnitShip:
				cell = tl.Cell{Fg: tl.ColorBlue, Ch: 's'}

			case rtsengine.UnitTower:
				cell = tl.Cell{Fg: tl.ColorBlue, Ch: 't'}

			case rtsengine.UnitWoodPile:
				cell = tl.Cell{Fg: tl.ColorBlack, Ch: 'w'}

			case rtsengine.UnitFence:
				//r, _ := utf8.DecodeRuneInString("\xE2\xAC\x9B")
				cell = tl.Cell{Fg: tl.ColorBlack, Ch: 'X'}
			}
			acre.SetCell(0, 0, &cell)
			ui.level.AddEntity(acre)
		}
		m[p.UnitID] = acre
	}

	// Remove anything left over. That means it is no longer within the view.
	for _, v := range ui.acreMap {
		ui.level.RemoveEntity(v)
	}

	ui.acreMap = m
}

// communicationPreamble will start the communication with the rtsengine.
func (ui *ReferenceUI) communicationPreamble() {
	// give the UI some time to initialize.
	time.Sleep(time.Second * 2)

	var packet rtsengine.WirePacket

	// WhoAmI? Get user information for this connection.
	packet.Command = rtsengine.WhoAmI
	err := ui.JSONEncoder.Encode(&packet)
	if err != nil {
		fmt.Println("Unexpected wire error", err)
		return
	}

	// Send full View to set our UI to the entire view of the game for testing.
	packet.Command = rtsengine.SetView

	// They are flipped for our engine.
	packet.ViewHeight, packet.ViewWidth = ui.game.Screen().Size()
	err = ui.JSONEncoder.Encode(&packet)
	if err != nil {
		fmt.Println("Unexpected wire error", err)
		return
	}

	// Force a partial refresh to place the initial acres and units on our screen.
	packet.Command = rtsengine.PartialRefreshPlayerToUI
	err = ui.JSONEncoder.Encode(&packet)
	if err != nil {
		fmt.Println("Unexpected wire error", err)
		return
	}

}

// pathUnitToLocation will request that unitID be pathed to X,Y
func (ui *ReferenceUI) pathUnitToLocation(UnitID int, X int, Y int) {
	var packet rtsengine.WirePacket

	// Send full View to set our UI to the entire view of the game for testing.
	packet.Command = rtsengine.PathUnitToLocation
	packet.CurrentRow = Y
	packet.CurrentColumn = X
	packet.UnitID = UnitID
	err := ui.JSONEncoder.Encode(&packet)
	if err != nil {
		fmt.Println("Unexpected wire error", err)
		return
	}
}

// scrollView will scroll this player view within the world.
func (ui *ReferenceUI) scrollView() {
	var packet rtsengine.WirePacket

	// Send full View to set our UI to the entire view of the game for testing.
	packet.Command = rtsengine.ScrollView
	packet.CurrentRow = ui.screenController.Xdiff
	packet.CurrentColumn = ui.screenController.Ydiff
	err := ui.JSONEncoder.Encode(&packet)
	if err != nil {
		fmt.Println("Unexpected wire error", err)
		return
	}
}

// findAcre will find the acre at X and Y and return nil if not found.
// Presently this is an O(N) search.
func (ui *ReferenceUI) findAcre(X int, Y int) *Acre {
	for _, v := range ui.acreMap {
		if v.Column == X && v.Row == Y {
			return v
		}
	}
	return nil
}

// Assume grass is the default.
func main() {
	ui := ReferenceUI{}

	ui.Start()
}
<|endoftext|>


<commit_before>package server

import (
	"crypto/rand"
	"encoding/base64"
	goerr "errors"
	"fmt"
	"log"
	"net/http"
	"path"
	"time"

	"github.com/pachyderm/pachyderm/src/client/auth"
	"github.com/pachyderm/pachyderm/src/client/pkg/errors"
	"github.com/pachyderm/pachyderm/src/server/pkg/backoff"
	col "github.com/pachyderm/pachyderm/src/server/pkg/collection"
	"github.com/pachyderm/pachyderm/src/server/pkg/watch"

	"github.com/coreos/go-oidc"
	logrus "github.com/sirupsen/logrus"
	"golang.org/x/net/context"
	"golang.org/x/oauth2"
)

const threeMinutes = 3 * 60 // Passed to col.PutTTL (so value is in seconds)

// various oidc invalid argument errors. Use 'goerror' instead of internal
// 'errors' library b/c stack trace isn't useful
var (
	notConfigured = goerr.New("OIDC ID provider configuration not found")
	watchFailed   = goerr.New("error converting OIDC state token (has it expired?)")
	tokenDeleted  = goerr.New("error during authorization: OIDC state token expired")
)

// InternalOIDCProvider contains information about the configured OIDC ID
// provider, as well as auth information identifying Pachyderm in the ID
// provider (ClientID and ClientSecret), which Pachyderm needs to perform
// authorization with it.
type InternalOIDCProvider struct {
	// a points back to the owning auth/server.apiServer, currently just so that
	// InternalOIDCProvider can get an etcd client from it to read/write OIDC
	// state tokens to etcd during authorization
	a *apiServer

	// Provider generates the ID provider login URL returned by GetOIDCLogin
	Provider *oidc.Provider

	// Issuer is the address of the OIDC ID provider (where we exchange
	// authorization codes for access tokens and get users' email addresses in
	// Authorize())
	Issuer string

	// ClientID is Pachyderm's identifier in the OIDC ID provider (generated by
	// the ID provider, and passed to Pachyderm by the cluster administrator via
	// SetConfig)
	ClientID string

	// ClientSecret is a shared secret with the ID provider, for doing the
	// auth-code -> access-token exchange.
	ClientSecret string

	// RedirectURI is used by GetOIDCLogin to generate a login URL that redirects
	// users back to Pachyderm (must be provided by the cluster administrator via
	// SetConfig, as only they know their network topology & Pachyderm's address
	// within it, and must be included in login URLs)
	RedirectURI string

	// States is an etcd collection containing the state information associated
	// with every in-progress authentication flow. /authorization-code/callback
	// places users' ID tokens in here when they authenticate successfully, and
	// Authenticate() retrieves those ID tokens, converts them to Pachyderm
	// tokens, and returns users' Pachyderm tokens back to them—all scoped to the
	// state token identifying the login flow
	States col.Collection
}

// CryptoString returns a cryptographically random, URL safe string with length
// at least n
//
// TODO(msteffen): move away from UUIDv4 towards this (current implementation of
// UUIDv4 produces UUIDs via CSPRNG, but the UUIDv4 spec doesn't guarantee that
// behavior, and we shouldn't assume it going forward)
func CryptoString(n int) string {
	var numBytes int
	for n >= base64.RawURLEncoding.EncodedLen(numBytes) {
		numBytes++
	}
	b := make([]byte, numBytes)
	_, err := rand.Read(b)
	if err != nil {
		panic("could not generate cryptographically secure random string!")
	}

	return base64.RawURLEncoding.EncodeToString(b)
}

// NewOIDCSP creates a new InternalOIDCProvider object from the given parameters
func (a *apiServer) NewOIDCSP(issuer, clientID, clientSecret, redirectURI string) (*InternalOIDCProvider, error) {
	o := &InternalOIDCProvider{
		a:            a,
		Issuer:       issuer,
		ClientID:     clientID,
		ClientSecret: clientSecret,
		RedirectURI:  redirectURI,
		States: col.NewCollection(
			a.env.GetEtcdClient(),
			path.Join(oidcAuthnPrefix),
			nil,
			&auth.SessionInfo{}, // TODO make OIDCSessionInfo
			nil,
			nil,
		),
	}
	var err error
	o.Provider, err = oidc.NewProvider(
		// Due to the implementation of go-oidc, this context is used for RPCs
		// (fetching certificates) within all future OIDC authentication flows.
		// Thus, it must not have a timeout. We ideally should create a new
		// context.WithCancel() and cancel that new context if/when o.Provider is
		// updated, but we don't have a convenient place to put that cancel() call
		// and the effect of this omission is limited to in-flight authentication
		// flows at the moment that o.Provider updated, so we're ignoring it.
		context.Background(),
		issuer)
	if err != nil {
		return nil, err
	}
	return o, nil
}

// GetOIDCLoginURL uses the given state to generate a login URL for the OIDC provider object
func (o *InternalOIDCProvider) GetOIDCLoginURL(ctx context.Context) (string, string, error) {
	if o == nil {
		return "", "", notConfigured
	}
	// TODO(msteffen, adelelopez): We *think* this 'if' block can't run anymore:
	// (if o != nil, then o.Provider != nil)
	// remove if no one reports seeing this error in 1.11.0.
	if o.Provider == nil {
		o.Provider, err = oidc.NewProvider(context.Background(), o.Issuer)
		if err != nil {
			return "", "", fmt.Errorf("provider could not be found: %v", err)
		}
	}
	state := CryptoString(30)
	nonce := CryptoString(30)
	conf := oauth2.Config{
		ClientID:     o.ClientID,
		ClientSecret: o.ClientSecret,
		RedirectURL:  o.RedirectURI,
		Endpoint:     o.Provider.Endpoint(),
		// "openid" is a required scope for OpenID Connect flows.
		// "profile" and "email" are necessary for using the email as an identifier
		Scopes: []string{oidc.ScopeOpenID, "profile", "email"},
	}

	if _, err := col.NewSTM(ctx, o.a.env.GetEtcdClient(), func(stm col.STM) error {
		return o.States.ReadWrite(stm).PutTTL(state, &auth.SessionInfo{
			Nonce: nonce, // read & verified by /authorization-code/callback
		}, threeMinutes)
	}); err != nil {
		return "", "", errors.Wrap(err, "could not create OIDC login session")
	}

	url := conf.AuthCodeURL(state,
		oauth2.SetAuthURLParam("response_type", "code"),
		oauth2.SetAuthURLParam("nonce", nonce))
	return url, state, nil
}

// OIDCStateToEmail takes the state session created for the OIDC session
// and uses it discover the email of the user who obtained the
// code (or verify that the code belongs to them). This is how
// Pachyderm currently implements authorization in a production cluster
func (o *InternalOIDCProvider) OIDCStateToEmail(ctx context.Context, state string) (string, error) {
	// reestablish watch in a loop, in case there's a watch error
	var idToken string
	if err := backoff.RetryNotify(func() error {
		watcher, err := o.States.ReadOnly(ctx).WatchOne(state)
		if err != nil {
			return watchFailed
		}
		defer watcher.Close()

		// lookup the token from the given state
		for e := range watcher.Watch() {
			if e.Type == watch.EventError {
				// reestablish watch
				return errors.Wrapf(e.Err, "error watching OIDC state token during conversion")
			} else if e.Type == watch.EventDelete {
				return tokenDeleted
			}

			// see if there's an ID token attached to the OIDC state now
			var si auth.SessionInfo
			if err := si.Unmarshal(e.Value); err != nil {
				// retry watch (maybe a valid SessionInfo will appear later?)
				return errors.Wrapf(e.Err, "error unmarshalling OIDC SessionInfo during conversion")
			}
			if si.IDToken != "" {
				idToken = si.IDToken
				return nil // success
			}
			logrus.Errorf("ID token unset in OIDC conversion watch event")
		}
		return nil
	}, backoff.New60sBackOff(), func(err error, _ time.Duration) error {
		logrus.Errorf("error watching OIDC state token during authorization: %v", err)
		if err == watchFailed || err == tokenDeleted {
			return err // don't retry, just return the error
		}
		return nil
	}); err != nil {
		return "", err
	}

	// Use the ID token passed from the authorization callback as our token source
	ts := oauth2.StaticTokenSource(
		&oauth2.Token{
			AccessToken: idToken,
		},
	)
	userInfo, err := o.Provider.UserInfo(ctx, ts)
	if err != nil {
		return "", err
	}
	logrus.Infof("recovered user info with email: '%v'", userInfo.Email)
	return userInfo.Email, nil
}

func (a *apiServer) handleExchange(w http.ResponseWriter, req *http.Request) {
	ctx := context.Background()

	var err error
	sp := a.getOIDCSP()
	if sp == nil {
		http.Error(w, "OIDC has not been configured or was disabled", http.StatusConflict)
		return
	}
	conf := &oauth2.Config{
		ClientID:     sp.ClientID,
		ClientSecret: sp.ClientSecret,
		RedirectURL:  sp.RedirectURI,
		Scopes:       []string{"openid", "email", "profile"},
		Endpoint:     sp.Provider.Endpoint(),
	}

	code := req.URL.Query()["code"][0]
	state := req.URL.Query()["state"][0]

	logrus.Infof("session state and code are obtained")

	// Use the authorization code that is pushed to the redirect
	tok, err := conf.Exchange(ctx, code)
	if err != nil {
		logrus.Errorf("failed to exchange code: %v", err)
		http.Error(w, "Unauthorized", http.StatusUnauthorized)
		return
	}

	logrus.Info("exchanged OIDC code for token")

	var verifier = sp.Provider.Verifier(&oidc.Config{ClientID: conf.ClientID})
	// Extract the ID Token from OAuth2 token.
	rawIDToken, ok := tok.Extra("id_token").(string)
	if !ok {
		logrus.Errorf("missing id token")
		http.Error(w, "Unauthorized", http.StatusUnauthorized)
		return
	}

	logrus.Infof("raw ID Token: %v", rawIDToken)

	// Parse and verify ID Token payload.
	idToken, err := verifier.Verify(ctx, rawIDToken)
	if err != nil {
		logrus.Errorf("could not verify token: %v", err)
		http.Error(w, "Unauthorized", http.StatusUnauthorized)
		return
	}

	// Verify the ID token, and if it's valid, add it to this state's SessionInfo
	// in etcd, so that any concurrent Authorize() calls can discover it and give
	// the caller a Pachyderm token.
	_, err = col.NewSTM(ctx, a.env.GetEtcdClient(), func(stm col.STM) error {
		var si auth.SessionInfo
		return sp.States.ReadWrite(stm).Update(state, &si, func() error {
			if idToken.Nonce != si.Nonce {
				logrus.Errorf("IDP nonce %v did not match Pachyderm's session nonce %v",
					idToken.Nonce, si.Nonce)
				http.Error(w, "Unauthorized", http.StatusUnauthorized)
				return nil // no error in etcd read, even though the request is erroring
			}
			logrus.Infof("nonce is %v", idToken.Nonce)

			si.IDToken = rawIDToken
			logrus.Infof("saving state with access token")
			return nil
		})
	})

	fmt.Fprintf(w, "You are now logged in. Go back to the terminal to use Pachyderm!")
}

func (a *apiServer) serveOIDC() {
	// serve OIDC handler to exchange the auth code
	http.HandleFunc("/authorization-code/callback", a.handleExchange)
	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%v", OidcPort), nil))
}
<commit_msg>Remove obsolete comment<commit_after>package server

import (
	"crypto/rand"
	"encoding/base64"
	goerr "errors"
	"fmt"
	"log"
	"net/http"
	"path"
	"time"

	"github.com/pachyderm/pachyderm/src/client/auth"
	"github.com/pachyderm/pachyderm/src/client/pkg/errors"
	"github.com/pachyderm/pachyderm/src/server/pkg/backoff"
	col "github.com/pachyderm/pachyderm/src/server/pkg/collection"
	"github.com/pachyderm/pachyderm/src/server/pkg/watch"

	"github.com/coreos/go-oidc"
	logrus "github.com/sirupsen/logrus"
	"golang.org/x/net/context"
	"golang.org/x/oauth2"
)

const threeMinutes = 3 * 60 // Passed to col.PutTTL (so value is in seconds)

// various oidc invalid argument errors. Use 'goerror' instead of internal
// 'errors' library b/c stack trace isn't useful
var (
	notConfigured = goerr.New("OIDC ID provider configuration not found")
	watchFailed   = goerr.New("error converting OIDC state token (has it expired?)")
	tokenDeleted  = goerr.New("error during authorization: OIDC state token expired")
)

// InternalOIDCProvider contains information about the configured OIDC ID
// provider, as well as auth information identifying Pachyderm in the ID
// provider (ClientID and ClientSecret), which Pachyderm needs to perform
// authorization with it.
type InternalOIDCProvider struct {
	// a points back to the owning auth/server.apiServer, currently just so that
	// InternalOIDCProvider can get an etcd client from it to read/write OIDC
	// state tokens to etcd during authorization
	a *apiServer

	// Provider generates the ID provider login URL returned by GetOIDCLogin
	Provider *oidc.Provider

	// Issuer is the address of the OIDC ID provider (where we exchange
	// authorization codes for access tokens and get users' email addresses in
	// Authorize())
	Issuer string

	// ClientID is Pachyderm's identifier in the OIDC ID provider (generated by
	// the ID provider, and passed to Pachyderm by the cluster administrator via
	// SetConfig)
	ClientID string

	// ClientSecret is a shared secret with the ID provider, for doing the
	// auth-code -> access-token exchange.
	ClientSecret string

	// RedirectURI is used by GetOIDCLogin to generate a login URL that redirects
	// users back to Pachyderm (must be provided by the cluster administrator via
	// SetConfig, as only they know their network topology & Pachyderm's address
	// within it, and must be included in login URLs)
	RedirectURI string

	// States is an etcd collection containing the state information associated
	// with every in-progress authentication flow. /authorization-code/callback
	// places users' ID tokens in here when they authenticate successfully, and
	// Authenticate() retrieves those ID tokens, converts them to Pachyderm
	// tokens, and returns users' Pachyderm tokens back to them—all scoped to the
	// state token identifying the login flow
	States col.Collection
}

// CryptoString returns a cryptographically random, URL safe string with length
// at least n
//
// TODO(msteffen): move away from UUIDv4 towards this (current implementation of
// UUIDv4 produces UUIDs via CSPRNG, but the UUIDv4 spec doesn't guarantee that
// behavior, and we shouldn't assume it going forward)
func CryptoString(n int) string {
	var numBytes int
	for n >= base64.RawURLEncoding.EncodedLen(numBytes) {
		numBytes++
	}
	b := make([]byte, numBytes)
	_, err := rand.Read(b)
	if err != nil {
		panic("could not generate cryptographically secure random string!")
	}

	return base64.RawURLEncoding.EncodeToString(b)
}

// NewOIDCSP creates a new InternalOIDCProvider object from the given parameters
func (a *apiServer) NewOIDCSP(issuer, clientID, clientSecret, redirectURI string) (*InternalOIDCProvider, error) {
	o := &InternalOIDCProvider{
		a:            a,
		Issuer:       issuer,
		ClientID:     clientID,
		ClientSecret: clientSecret,
		RedirectURI:  redirectURI,
		States: col.NewCollection(
			a.env.GetEtcdClient(),
			path.Join(oidcAuthnPrefix),
			nil,
			&auth.SessionInfo{},
			nil,
			nil,
		),
	}
	var err error
	o.Provider, err = oidc.NewProvider(
		// Due to the implementation of go-oidc, this context is used for RPCs
		// (fetching certificates) within all future OIDC authentication flows.
		// Thus, it must not have a timeout. We ideally should create a new
		// context.WithCancel() and cancel that new context if/when o.Provider is
		// updated, but we don't have a convenient place to put that cancel() call
		// and the effect of this omission is limited to in-flight authentication
		// flows at the moment that o.Provider updated, so we're ignoring it.
		context.Background(),
		issuer)
	if err != nil {
		return nil, err
	}
	return o, nil
}

// GetOIDCLoginURL uses the given state to generate a login URL for the OIDC provider object
func (o *InternalOIDCProvider) GetOIDCLoginURL(ctx context.Context) (string, string, error) {
	if o == nil {
		return "", "", notConfigured
	}
	// TODO(msteffen, adelelopez): We *think* this 'if' block can't run anymore:
	// (if o != nil, then o.Provider != nil)
	// remove if no one reports seeing this error in 1.11.0.
	if o.Provider == nil {
		o.Provider, err = oidc.NewProvider(context.Background(), o.Issuer)
		if err != nil {
			return "", "", fmt.Errorf("provider could not be found: %v", err)
		}
	}
	state := CryptoString(30)
	nonce := CryptoString(30)
	conf := oauth2.Config{
		ClientID:     o.ClientID,
		ClientSecret: o.ClientSecret,
		RedirectURL:  o.RedirectURI,
		Endpoint:     o.Provider.Endpoint(),
		// "openid" is a required scope for OpenID Connect flows.
		// "profile" and "email" are necessary for using the email as an identifier
		Scopes: []string{oidc.ScopeOpenID, "profile", "email"},
	}

	if _, err := col.NewSTM(ctx, o.a.env.GetEtcdClient(), func(stm col.STM) error {
		return o.States.ReadWrite(stm).PutTTL(state, &auth.SessionInfo{
			Nonce: nonce, // read & verified by /authorization-code/callback
		}, threeMinutes)
	}); err != nil {
		return "", "", errors.Wrap(err, "could not create OIDC login session")
	}

	url := conf.AuthCodeURL(state,
		oauth2.SetAuthURLParam("response_type", "code"),
		oauth2.SetAuthURLParam("nonce", nonce))
	return url, state, nil
}

// OIDCStateToEmail takes the state session created for the OIDC session
// and uses it discover the email of the user who obtained the
// code (or verify that the code belongs to them). This is how
// Pachyderm currently implements authorization in a production cluster
func (o *InternalOIDCProvider) OIDCStateToEmail(ctx context.Context, state string) (string, error) {
	// reestablish watch in a loop, in case there's a watch error
	var idToken string
	if err := backoff.RetryNotify(func() error {
		watcher, err := o.States.ReadOnly(ctx).WatchOne(state)
		if err != nil {
			return watchFailed
		}
		defer watcher.Close()

		// lookup the token from the given state
		for e := range watcher.Watch() {
			if e.Type == watch.EventError {
				// reestablish watch
				return errors.Wrapf(e.Err, "error watching OIDC state token during conversion")
			} else if e.Type == watch.EventDelete {
				return tokenDeleted
			}

			// see if there's an ID token attached to the OIDC state now
			var si auth.SessionInfo
			if err := si.Unmarshal(e.Value); err != nil {
				// retry watch (maybe a valid SessionInfo will appear later?)
				return errors.Wrapf(e.Err, "error unmarshalling OIDC SessionInfo during conversion")
			}
			if si.IDToken != "" {
				idToken = si.IDToken
				return nil // success
			}
			logrus.Errorf("ID token unset in OIDC conversion watch event")
		}
		return nil
	}, backoff.New60sBackOff(), func(err error, _ time.Duration) error {
		logrus.Errorf("error watching OIDC state token during authorization: %v", err)
		if err == watchFailed || err == tokenDeleted {
			return err // don't retry, just return the error
		}
		return nil
	}); err != nil {
		return "", err
	}

	// Use the ID token passed from the authorization callback as our token source
	ts := oauth2.StaticTokenSource(
		&oauth2.Token{
			AccessToken: idToken,
		},
	)
	userInfo, err := o.Provider.UserInfo(ctx, ts)
	if err != nil {
		return "", err
	}
	logrus.Infof("recovered user info with email: '%v'", userInfo.Email)
	return userInfo.Email, nil
}

func (a *apiServer) handleExchange(w http.ResponseWriter, req *http.Request) {
	ctx := context.Background()

	var err error
	sp := a.getOIDCSP()
	if sp == nil {
		http.Error(w, "OIDC has not been configured or was disabled", http.StatusConflict)
		return
	}
	conf := &oauth2.Config{
		ClientID:     sp.ClientID,
		ClientSecret: sp.ClientSecret,
		RedirectURL:  sp.RedirectURI,
		Scopes:       []string{"openid", "email", "profile"},
		Endpoint:     sp.Provider.Endpoint(),
	}

	code := req.URL.Query()["code"][0]
	state := req.URL.Query()["state"][0]

	logrus.Infof("session state and code are obtained")

	// Use the authorization code that is pushed to the redirect
	tok, err := conf.Exchange(ctx, code)
	if err != nil {
		logrus.Errorf("failed to exchange code: %v", err)
		http.Error(w, "Unauthorized", http.StatusUnauthorized)
		return
	}

	logrus.Info("exchanged OIDC code for token")

	var verifier = sp.Provider.Verifier(&oidc.Config{ClientID: conf.ClientID})
	// Extract the ID Token from OAuth2 token.
	rawIDToken, ok := tok.Extra("id_token").(string)
	if !ok {
		logrus.Errorf("missing id token")
		http.Error(w, "Unauthorized", http.StatusUnauthorized)
		return
	}

	logrus.Infof("raw ID Token: %v", rawIDToken)

	// Parse and verify ID Token payload.
	idToken, err := verifier.Verify(ctx, rawIDToken)
	if err != nil {
		logrus.Errorf("could not verify token: %v", err)
		http.Error(w, "Unauthorized", http.StatusUnauthorized)
		return
	}

	// Verify the ID token, and if it's valid, add it to this state's SessionInfo
	// in etcd, so that any concurrent Authorize() calls can discover it and give
	// the caller a Pachyderm token.
	_, err = col.NewSTM(ctx, a.env.GetEtcdClient(), func(stm col.STM) error {
		var si auth.SessionInfo
		return sp.States.ReadWrite(stm).Update(state, &si, func() error {
			if idToken.Nonce != si.Nonce {
				logrus.Errorf("IDP nonce %v did not match Pachyderm's session nonce %v",
					idToken.Nonce, si.Nonce)
				http.Error(w, "Unauthorized", http.StatusUnauthorized)
				return nil // no error in etcd read, even though the request is erroring
			}
			logrus.Infof("nonce is %v", idToken.Nonce)

			si.IDToken = rawIDToken
			logrus.Infof("saving state with access token")
			return nil
		})
	})

	fmt.Fprintf(w, "You are now logged in. Go back to the terminal to use Pachyderm!")
}

func (a *apiServer) serveOIDC() {
	// serve OIDC handler to exchange the auth code
	http.HandleFunc("/authorization-code/callback", a.handleExchange)
	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%v", OidcPort), nil))
}
<|endoftext|>


<commit_before>package s3

import (
	"errors"
	"fmt"
	"io"
	"net/http"
	"regexp"
	"strconv"
	"strings"

	"github.com/gogo/protobuf/types"
	"github.com/gorilla/mux"
	"github.com/pachyderm/pachyderm/src/client"
	pfsClient "github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/server/pkg/errutil"
	"github.com/pachyderm/pachyderm/src/server/pkg/uuid"
	"github.com/pachyderm/s2"
)

var multipartChunkPathMatcher = regexp.MustCompile(`([^/]+)/([^/]+)/(.+)/([^/]+)/(\d+)`)
var multipartKeepPathMatcher = regexp.MustCompile(`([^/]+)/([^/]+)/(.+)/([^/]+)/\.keep`)

func multipartChunkArgs(path string) (repo string, branch string, key string, uploadID string, partNumber int, err error) {
	match := multipartChunkPathMatcher.FindStringSubmatch(path)

	if len(match) == 0 {
		err = errors.New("invalid file path found in multipath bucket")
		return
	}

	repo = match[1]
	branch = match[2]
	key = match[3]
	uploadID = match[4]
	partNumber, err = strconv.Atoi(match[5])
	if err != nil {
		err = fmt.Errorf("invalid file path found in multipath bucket: %s", err)
		return
	}
	return
}

func multipartKeepArgs(path string) (repo string, branch string, key string, uploadID string, err error) {
	match := multipartKeepPathMatcher.FindStringSubmatch(path)

	if len(match) == 0 {
		err = errors.New("invalid file path found in multipath bucket")
		return
	}

	repo = match[1]
	branch = match[2]
	key = match[3]
	uploadID = match[4]
	return
}

func parentDirPath(repo, branch, key, uploadID string) string {
	return fmt.Sprintf("%s/%s/%s/%s", repo, branch, key, uploadID)
}

func chunkPath(repo, branch, key, uploadID string, partNumber int) string {
	return fmt.Sprintf("%s/%d", parentDirPath(repo, branch, key, uploadID), partNumber)
}

func keepPath(repo, branch, key, uploadID string) string {
	return fmt.Sprintf("%s/.keep", parentDirPath(repo, branch, key, uploadID))
}

func (c *controller) ensureRepo(pc *client.APIClient) error {
	_, err := pc.InspectBranch(c.repo, "master")
	if err != nil {
		err = pc.CreateRepo(c.repo)
		if err != nil && !strings.Contains(err.Error(), "as it already exists") {
			return err
		}

		err = pc.CreateBranch(c.repo, "master", "", nil)
		if err != nil && !strings.Contains(err.Error(), "as it already exists") {
			return err
		}
	}

	return nil
}

func (c *controller) ListMultipart(r *http.Request, bucket, keyMarker, uploadIDMarker string, maxUploads int) (*s2.ListMultipartResult, error) {
	vars := mux.Vars(r)
	pc, err := c.pachClient(vars["authAccessKey"])
	if err != nil {
		return nil, err
	}
	repo, branch, err := bucketArgs(r, bucket)
	if err != nil {
		return nil, err
	}
	_, err = pc.InspectBranch(repo, branch)
	if err != nil {
		return nil, maybeNotFoundError(r, err)
	}
	if err = c.ensureRepo(pc); err != nil {
		return nil, err
	}

	result := s2.ListMultipartResult{
		Uploads: []s2.Upload{},
	}

	err = pc.GlobFileF(c.repo, "master", fmt.Sprintf("%s/%s/*/*/.keep", repo, branch), func(fileInfo *pfsClient.FileInfo) error {
		_, _, key, uploadID, err := multipartKeepArgs(fileInfo.File.Path)
		if err != nil {
			return nil
		}

		if key <= keyMarker || uploadID <= uploadIDMarker {
			return nil
		}

		if len(result.Uploads) >= maxUploads {
			if maxUploads > 0 {
				result.IsTruncated = true
			}
			return errutil.ErrBreak
		}

		timestamp, err := types.TimestampFromProto(fileInfo.Committed)
		if err != nil {
			return err
		}

		result.Uploads = append(result.Uploads, s2.Upload{
			Key:          key,
			UploadID:     uploadID,
			Initiator:    defaultUser,
			StorageClass: globalStorageClass,
			Initiated:    timestamp,
		})

		return nil
	})

	return &result, err
}

func (c *controller) InitMultipart(r *http.Request, bucket, key string) (string, error) {
	vars := mux.Vars(r)
	pc, err := c.pachClient(vars["authAccessKey"])
	if err != nil {
		return "", err
	}
	repo, branch, err := bucketArgs(r, bucket)
	if err != nil {
		return "", err
	}
	_, err = pc.InspectBranch(repo, branch)
	if err != nil {
		return "", maybeNotFoundError(r, err)
	}
	if err = c.ensureRepo(pc); err != nil {
		return "", err
	}

	uploadID := uuid.NewWithoutDashes()

	path := fmt.Sprintf("%s/.keep", parentDirPath(repo, branch, key, uploadID))
	_, err = pc.PutFileOverwrite(c.repo, "master", path, strings.NewReader(""), 0)
	if err != nil {
		return "", err
	}

	return uploadID, nil
}

func (c *controller) AbortMultipart(r *http.Request, bucket, key, uploadID string) error {
	vars := mux.Vars(r)
	pc, err := c.pachClient(vars["authAccessKey"])
	if err != nil {
		return err
	}
	repo, branch, err := bucketArgs(r, bucket)
	if err != nil {
		return err
	}
	_, err = pc.InspectBranch(repo, branch)
	if err != nil {
		return maybeNotFoundError(r, err)
	}
	if err = c.ensureRepo(pc); err != nil {
		return err
	}

	_, err = pc.InspectFile(c.repo, "master", keepPath(repo, branch, key, uploadID))
	if err != nil {
		return s2.NoSuchUploadError(r)
	}

	err = pc.DeleteFile(c.repo, "master", parentDirPath(repo, branch, key, uploadID))
	if err != nil {
		return s2.InternalError(r, err)
	}

	return nil
}

func (c *controller) CompleteMultipart(r *http.Request, bucket, key, uploadID string, parts []s2.Part) (*s2.CompleteMultipartResult, error) {
	vars := mux.Vars(r)
	pc, err := c.pachClient(vars["authAccessKey"])
	if err != nil {
		return nil, err
	}
	repo, branch, err := bucketArgs(r, bucket)
	if err != nil {
		return nil, err
	}
	_, err = pc.InspectBranch(repo, branch)
	if err != nil {
		return nil, maybeNotFoundError(r, err)
	}
	if err = c.ensureRepo(pc); err != nil {
		return nil, err
	}

	_, err = pc.InspectFile(c.repo, "master", keepPath(repo, branch, key, uploadID))
	if err != nil {
		// TODO: differentiate between not found errors and other kinds
		return nil, s2.NoSuchUploadError(r)
	}

	for i, part := range parts {
		srcPath := chunkPath(repo, branch, key, uploadID, part.PartNumber)

		fileInfo, err := pc.InspectFile(c.repo, "master", srcPath)
		if err != nil {
			// TODO: differentiate between not found errors and other kinds
			return nil, s2.InvalidPartError(r)
		}

		// Only verify the ETag when it's of the same length as PFS file
		// hashes. This is because s3 clients will generally use md5 for
		// ETags, and would otherwise fail.
		expectedETag := fmt.Sprintf("%x", fileInfo.Hash)
		if len(part.ETag) == len(expectedETag) && part.ETag != expectedETag {
			return nil, s2.InvalidPartError(r)
		}

		if i < len(parts)-1 && fileInfo.SizeBytes < 5*1024*1024 {
			// each part, except for the last, is expected to be at least 5mb
			// in s3
			return nil, s2.EntityTooSmallError(r)
		}

		err = pc.CopyFile(c.repo, "master", srcPath, repo, branch, key, false)
		if err != nil {
			return nil, err
		}
	}

	// TODO: verify that this works
	err = pc.DeleteFile(c.repo, "master", parentDirPath(repo, branch, key, uploadID))
	if err != nil {
		return nil, err
	}

	fileInfo, err := pc.InspectFile(repo, branch, key)
	if err != nil {
		return nil, err
	}

	result := s2.CompleteMultipartResult{
		Location: globalLocation,
		ETag:     fmt.Sprintf("%x", fileInfo.Hash),
		Version:  fileInfo.File.Commit.ID,
	}
	return &result, nil
}

func (c *controller) ListMultipartChunks(r *http.Request, bucket, key, uploadID string, partNumberMarker, maxParts int) (*s2.ListMultipartChunksResult, error) {
	vars := mux.Vars(r)
	pc, err := c.pachClient(vars["authAccessKey"])
	if err != nil {
		return nil, err
	}
	repo, branch, err := bucketArgs(r, bucket)
	if err != nil {
		return nil, err
	}
	_, err = pc.InspectBranch(repo, branch)
	if err != nil {
		return nil, maybeNotFoundError(r, err)
	}
	if err = c.ensureRepo(pc); err != nil {
		return nil, err
	}

	result := s2.ListMultipartChunksResult{
		Initiator:    &defaultUser,
		Owner:        &defaultUser,
		StorageClass: globalStorageClass,
		Parts:        []s2.Part{},
	}

	err = pc.GlobFileF(c.repo, "master", fmt.Sprintf("%s/%s/%s/%s/*", repo, branch, key, uploadID), func(fileInfo *pfsClient.FileInfo) error {
		_, _, _, _, partNumber, err := multipartChunkArgs(fileInfo.File.Path)
		if err != nil {
			return nil
		}

		if partNumber <= partNumberMarker {
			return nil
		}

		if len(result.Parts) >= maxParts {
			if maxParts > 0 {
				result.IsTruncated = true
			}
			return errutil.ErrBreak
		}

		result.Parts = append(result.Parts, s2.Part{
			PartNumber: partNumber,
			ETag:       fmt.Sprintf("%x", fileInfo.Hash),
		})

		return nil
	})

	return &result, err
}

func (c *controller) UploadMultipartChunk(r *http.Request, bucket, key, uploadID string, partNumber int, reader io.Reader) (string, error) {
	vars := mux.Vars(r)
	pc, err := c.pachClient(vars["authAccessKey"])
	if err != nil {
		return "", err
	}
	repo, branch, err := bucketArgs(r, bucket)
	if err != nil {
		return "", err
	}
	_, err = pc.InspectBranch(repo, branch)
	if err != nil {
		err = maybeNotFoundError(r, err)
		return "", err
	}
	if err = c.ensureRepo(pc); err != nil {
		return "", err
	}

	_, err = pc.InspectFile(c.repo, "master", keepPath(repo, branch, key, uploadID))
	if err != nil {
		// TODO: differentiate between not found errors and other kinds
		return "", s2.NoSuchUploadError(r)
	}

	path := chunkPath(repo, branch, key, uploadID, partNumber)
	_, err = pc.PutFileOverwrite(c.repo, "master", path, reader, 0)
	if err != nil {
		return "", err
	}

	fileInfo, err := pc.InspectFile(c.repo, "master", path)
	if err != nil {
		return "", err
	}

	return fmt.Sprintf("%x", fileInfo.Hash), nil
}

func (c *controller) DeleteMultipartChunk(r *http.Request, bucket, key, uploadID string, partNumber int) error {
	vars := mux.Vars(r)
	pc, err := c.pachClient(vars["authAccessKey"])
	if err != nil {
		return err
	}
	repo, branch, err := bucketArgs(r, bucket)
	if err != nil {
		return err
	}
	_, err = pc.InspectBranch(repo, branch)
	if err != nil {
		return maybeNotFoundError(r, err)
	}
	if err = c.ensureRepo(pc); err != nil {
		return err
	}

	_, err = pc.InspectFile(c.repo, "master", keepPath(repo, branch, key, uploadID))
	if err != nil {
		return s2.NoSuchUploadError(r)
	}

	path := chunkPath(repo, branch, key, uploadID, partNumber)
	return pc.DeleteFile(c.repo, "master", path)
}
<commit_msg>Make sure to overwrite multipart files<commit_after>package s3

import (
	"errors"
	"fmt"
	"io"
	"net/http"
	"regexp"
	"strconv"
	"strings"

	"github.com/gogo/protobuf/types"
	"github.com/gorilla/mux"
	"github.com/pachyderm/pachyderm/src/client"
	pfsClient "github.com/pachyderm/pachyderm/src/client/pfs"
	pfsServer "github.com/pachyderm/pachyderm/src/server/pfs"
	"github.com/pachyderm/pachyderm/src/server/pkg/errutil"
	"github.com/pachyderm/pachyderm/src/server/pkg/uuid"
	"github.com/pachyderm/s2"
)

var multipartChunkPathMatcher = regexp.MustCompile(`([^/]+)/([^/]+)/(.+)/([^/]+)/(\d+)`)
var multipartKeepPathMatcher = regexp.MustCompile(`([^/]+)/([^/]+)/(.+)/([^/]+)/\.keep`)

func multipartChunkArgs(path string) (repo string, branch string, key string, uploadID string, partNumber int, err error) {
	match := multipartChunkPathMatcher.FindStringSubmatch(path)

	if len(match) == 0 {
		err = errors.New("invalid file path found in multipath bucket")
		return
	}

	repo = match[1]
	branch = match[2]
	key = match[3]
	uploadID = match[4]
	partNumber, err = strconv.Atoi(match[5])
	if err != nil {
		err = fmt.Errorf("invalid file path found in multipath bucket: %s", err)
		return
	}
	return
}

func multipartKeepArgs(path string) (repo string, branch string, key string, uploadID string, err error) {
	match := multipartKeepPathMatcher.FindStringSubmatch(path)

	if len(match) == 0 {
		err = errors.New("invalid file path found in multipath bucket")
		return
	}

	repo = match[1]
	branch = match[2]
	key = match[3]
	uploadID = match[4]
	return
}

func parentDirPath(repo, branch, key, uploadID string) string {
	return fmt.Sprintf("%s/%s/%s/%s", repo, branch, key, uploadID)
}

func chunkPath(repo, branch, key, uploadID string, partNumber int) string {
	return fmt.Sprintf("%s/%d", parentDirPath(repo, branch, key, uploadID), partNumber)
}

func keepPath(repo, branch, key, uploadID string) string {
	return fmt.Sprintf("%s/.keep", parentDirPath(repo, branch, key, uploadID))
}

func (c *controller) ensureRepo(pc *client.APIClient) error {
	_, err := pc.InspectBranch(c.repo, "master")
	if err != nil {
		err = pc.CreateRepo(c.repo)
		if err != nil && !strings.Contains(err.Error(), "as it already exists") {
			return err
		}

		err = pc.CreateBranch(c.repo, "master", "", nil)
		if err != nil && !strings.Contains(err.Error(), "as it already exists") {
			return err
		}
	}

	return nil
}

func (c *controller) ListMultipart(r *http.Request, bucket, keyMarker, uploadIDMarker string, maxUploads int) (*s2.ListMultipartResult, error) {
	vars := mux.Vars(r)
	pc, err := c.pachClient(vars["authAccessKey"])
	if err != nil {
		return nil, err
	}
	repo, branch, err := bucketArgs(r, bucket)
	if err != nil {
		return nil, err
	}
	_, err = pc.InspectBranch(repo, branch)
	if err != nil {
		return nil, maybeNotFoundError(r, err)
	}
	if err = c.ensureRepo(pc); err != nil {
		return nil, err
	}

	result := s2.ListMultipartResult{
		Uploads: []s2.Upload{},
	}

	err = pc.GlobFileF(c.repo, "master", fmt.Sprintf("%s/%s/*/*/.keep", repo, branch), func(fileInfo *pfsClient.FileInfo) error {
		_, _, key, uploadID, err := multipartKeepArgs(fileInfo.File.Path)
		if err != nil {
			return nil
		}

		if key <= keyMarker || uploadID <= uploadIDMarker {
			return nil
		}

		if len(result.Uploads) >= maxUploads {
			if maxUploads > 0 {
				result.IsTruncated = true
			}
			return errutil.ErrBreak
		}

		timestamp, err := types.TimestampFromProto(fileInfo.Committed)
		if err != nil {
			return err
		}

		result.Uploads = append(result.Uploads, s2.Upload{
			Key:          key,
			UploadID:     uploadID,
			Initiator:    defaultUser,
			StorageClass: globalStorageClass,
			Initiated:    timestamp,
		})

		return nil
	})

	return &result, err
}

func (c *controller) InitMultipart(r *http.Request, bucket, key string) (string, error) {
	vars := mux.Vars(r)
	pc, err := c.pachClient(vars["authAccessKey"])
	if err != nil {
		return "", err
	}
	repo, branch, err := bucketArgs(r, bucket)
	if err != nil {
		return "", err
	}
	_, err = pc.InspectBranch(repo, branch)
	if err != nil {
		return "", maybeNotFoundError(r, err)
	}
	if err = c.ensureRepo(pc); err != nil {
		return "", err
	}

	uploadID := uuid.NewWithoutDashes()

	path := fmt.Sprintf("%s/.keep", parentDirPath(repo, branch, key, uploadID))
	_, err = pc.PutFileOverwrite(c.repo, "master", path, strings.NewReader(""), 0)
	if err != nil {
		return "", err
	}

	return uploadID, nil
}

func (c *controller) AbortMultipart(r *http.Request, bucket, key, uploadID string) error {
	vars := mux.Vars(r)
	pc, err := c.pachClient(vars["authAccessKey"])
	if err != nil {
		return err
	}
	repo, branch, err := bucketArgs(r, bucket)
	if err != nil {
		return err
	}
	_, err = pc.InspectBranch(repo, branch)
	if err != nil {
		return maybeNotFoundError(r, err)
	}
	if err = c.ensureRepo(pc); err != nil {
		return err
	}

	_, err = pc.InspectFile(c.repo, "master", keepPath(repo, branch, key, uploadID))
	if err != nil {
		return s2.NoSuchUploadError(r)
	}

	err = pc.DeleteFile(c.repo, "master", parentDirPath(repo, branch, key, uploadID))
	if err != nil {
		return s2.InternalError(r, err)
	}

	return nil
}

func (c *controller) CompleteMultipart(r *http.Request, bucket, key, uploadID string, parts []s2.Part) (*s2.CompleteMultipartResult, error) {
	vars := mux.Vars(r)
	pc, err := c.pachClient(vars["authAccessKey"])
	if err != nil {
		return nil, err
	}
	repo, branch, err := bucketArgs(r, bucket)
	if err != nil {
		return nil, err
	}
	_, err = pc.InspectBranch(repo, branch)
	if err != nil {
		return nil, maybeNotFoundError(r, err)
	}
	if err = c.ensureRepo(pc); err != nil {
		return nil, err
	}

	_, err = pc.InspectFile(c.repo, "master", keepPath(repo, branch, key, uploadID))
	if err != nil {
		// TODO: differentiate between not found errors and other kinds
		return nil, s2.NoSuchUploadError(r)
	}

	// check if the destination file already exists, and if so, delete it
	_, err = pc.InspectFile(repo, branch, key)
	if err != nil && !pfsServer.IsFileNotFoundErr(err) {
		return nil, err
	}
	if !pfsServer.IsFileNotFoundErr(err) {
		err = pc.DeleteFile(repo, branch, key)
		if err != nil {
			return nil, err
		}
	}

	for i, part := range parts {
		srcPath := chunkPath(repo, branch, key, uploadID, part.PartNumber)

		fileInfo, err := pc.InspectFile(c.repo, "master", srcPath)
		if err != nil {
			// TODO: differentiate between not found errors and other kinds
			return nil, s2.InvalidPartError(r)
		}

		// Only verify the ETag when it's of the same length as PFS file
		// hashes. This is because s3 clients will generally use md5 for
		// ETags, and would otherwise fail.
		expectedETag := fmt.Sprintf("%x", fileInfo.Hash)
		if len(part.ETag) == len(expectedETag) && part.ETag != expectedETag {
			return nil, s2.InvalidPartError(r)
		}

		if i < len(parts)-1 && fileInfo.SizeBytes < 5*1024*1024 {
			// each part, except for the last, is expected to be at least 5mb
			// in s3
			return nil, s2.EntityTooSmallError(r)
		}

		err = pc.CopyFile(c.repo, "master", srcPath, repo, branch, key, false)
		if err != nil {
			return nil, err
		}
	}

	err = pc.DeleteFile(c.repo, "master", parentDirPath(repo, branch, key, uploadID))
	if err != nil {
		return nil, err
	}

	fileInfo, err := pc.InspectFile(repo, branch, key)
	if err != nil {
		return nil, err
	}

	result := s2.CompleteMultipartResult{
		Location: globalLocation,
		ETag:     fmt.Sprintf("%x", fileInfo.Hash),
		Version:  fileInfo.File.Commit.ID,
	}
	return &result, nil
}

func (c *controller) ListMultipartChunks(r *http.Request, bucket, key, uploadID string, partNumberMarker, maxParts int) (*s2.ListMultipartChunksResult, error) {
	vars := mux.Vars(r)
	pc, err := c.pachClient(vars["authAccessKey"])
	if err != nil {
		return nil, err
	}
	repo, branch, err := bucketArgs(r, bucket)
	if err != nil {
		return nil, err
	}
	_, err = pc.InspectBranch(repo, branch)
	if err != nil {
		return nil, maybeNotFoundError(r, err)
	}
	if err = c.ensureRepo(pc); err != nil {
		return nil, err
	}

	result := s2.ListMultipartChunksResult{
		Initiator:    &defaultUser,
		Owner:        &defaultUser,
		StorageClass: globalStorageClass,
		Parts:        []s2.Part{},
	}

	err = pc.GlobFileF(c.repo, "master", fmt.Sprintf("%s/%s/%s/%s/*", repo, branch, key, uploadID), func(fileInfo *pfsClient.FileInfo) error {
		_, _, _, _, partNumber, err := multipartChunkArgs(fileInfo.File.Path)
		if err != nil {
			return nil
		}

		if partNumber <= partNumberMarker {
			return nil
		}

		if len(result.Parts) >= maxParts {
			if maxParts > 0 {
				result.IsTruncated = true
			}
			return errutil.ErrBreak
		}

		result.Parts = append(result.Parts, s2.Part{
			PartNumber: partNumber,
			ETag:       fmt.Sprintf("%x", fileInfo.Hash),
		})

		return nil
	})

	return &result, err
}

func (c *controller) UploadMultipartChunk(r *http.Request, bucket, key, uploadID string, partNumber int, reader io.Reader) (string, error) {
	vars := mux.Vars(r)
	pc, err := c.pachClient(vars["authAccessKey"])
	if err != nil {
		return "", err
	}
	repo, branch, err := bucketArgs(r, bucket)
	if err != nil {
		return "", err
	}
	_, err = pc.InspectBranch(repo, branch)
	if err != nil {
		err = maybeNotFoundError(r, err)
		return "", err
	}
	if err = c.ensureRepo(pc); err != nil {
		return "", err
	}

	_, err = pc.InspectFile(c.repo, "master", keepPath(repo, branch, key, uploadID))
	if err != nil {
		// TODO: differentiate between not found errors and other kinds
		return "", s2.NoSuchUploadError(r)
	}

	path := chunkPath(repo, branch, key, uploadID, partNumber)
	_, err = pc.PutFileOverwrite(c.repo, "master", path, reader, 0)
	if err != nil {
		return "", err
	}

	fileInfo, err := pc.InspectFile(c.repo, "master", path)
	if err != nil {
		return "", err
	}

	return fmt.Sprintf("%x", fileInfo.Hash), nil
}

func (c *controller) DeleteMultipartChunk(r *http.Request, bucket, key, uploadID string, partNumber int) error {
	vars := mux.Vars(r)
	pc, err := c.pachClient(vars["authAccessKey"])
	if err != nil {
		return err
	}
	repo, branch, err := bucketArgs(r, bucket)
	if err != nil {
		return err
	}
	_, err = pc.InspectBranch(repo, branch)
	if err != nil {
		return maybeNotFoundError(r, err)
	}
	if err = c.ensureRepo(pc); err != nil {
		return err
	}

	_, err = pc.InspectFile(c.repo, "master", keepPath(repo, branch, key, uploadID))
	if err != nil {
		return s2.NoSuchUploadError(r)
	}

	path := chunkPath(repo, branch, key, uploadID, partNumber)
	return pc.DeleteFile(c.repo, "master", path)
}
<|endoftext|>


<commit_before>// Package ppsutil contains utilities for various PPS-related tasks, which are
// shared by both the PPS API and the worker binary. These utilities include:
// - Getting the RC name and querying k8s reguarding pipelines
// - Reading and writing pipeline resource requests and limits
// - Reading and writing EtcdPipelineInfos and PipelineInfos[1]
//
// [1] Note that PipelineInfo in particular is complicated because it contains
// fields that are not always set or are stored in multiple places
// ('job_state', for example, is not stored in PFS along with the rest of each
// PipelineInfo, because this field is volatile and we cannot commit to PFS
// every time it changes. 'job_counts' is the same, and 'reason' is in etcd
// because it is only updated alongside 'job_state').  As of 12/7/2017, these
// are the only fields not stored in PFS.
package ppsutil

import (
	"bytes"
	"fmt"
	"math"
	"path"
	"strings"
	"time"

	"github.com/gogo/protobuf/proto"
	"github.com/gogo/protobuf/types"
	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pps"
	col "github.com/pachyderm/pachyderm/src/server/pkg/collection"
	"github.com/pachyderm/pachyderm/src/server/pkg/ppsconsts"

	etcd "github.com/coreos/etcd/clientv3"
	log "github.com/sirupsen/logrus"
	"golang.org/x/net/context"
	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	kube "k8s.io/client-go/kubernetes"
)

// PipelineRepo creates a pfs repo for a given pipeline.
func PipelineRepo(pipeline *pps.Pipeline) *pfs.Repo {
	return &pfs.Repo{Name: pipeline.Name}
}

// PipelineRcName generates the name of the k8s replication controller that
// manages a pipeline's workers
func PipelineRcName(name string, version uint64) string {
	// k8s won't allow RC names that contain upper-case letters
	// or underscores
	// TODO: deal with name collision
	name = strings.Replace(name, "_", "-", -1)
	return fmt.Sprintf("pipeline-%s-v%d", strings.ToLower(name), version)
}

// GetRequestsResourceListFromPipeline returns a list of resources that the pipeline,
// minimally requires.
func GetRequestsResourceListFromPipeline(pipelineInfo *pps.PipelineInfo) (*v1.ResourceList, error) {
	return getResourceListFromSpec(pipelineInfo.ResourceRequests)
}

func getResourceListFromSpec(resources *pps.ResourceSpec) (*v1.ResourceList, error) {
	result := make(v1.ResourceList)

	if resources.Cpu != 0 {
		cpuStr := fmt.Sprintf("%f", resources.Cpu)
		cpuQuantity, err := resource.ParseQuantity(cpuStr)
		if err != nil {
			log.Warnf("error parsing cpu string: %s: %+v", cpuStr, err)
		} else {
			result[v1.ResourceCPU] = cpuQuantity
		}
	}

	if resources.Memory != "" {
		memQuantity, err := resource.ParseQuantity(resources.Memory)
		if err != nil {
			log.Warnf("error parsing memory string: %s: %+v", resources.Memory, err)
		} else {
			result[v1.ResourceMemory] = memQuantity
		}
	}

	if resources.Disk != "" { // needed because not all versions of k8s support disk resources
		diskQuantity, err := resource.ParseQuantity(resources.Disk)
		if err != nil {
			log.Warnf("error parsing disk string: %s: %+v", resources.Disk, err)
		} else {
			result[v1.ResourceEphemeralStorage] = diskQuantity
		}
	}

	if resources.Gpu != nil {
		gpuStr := fmt.Sprintf("%d", resources.Gpu.Number)
		gpuQuantity, err := resource.ParseQuantity(gpuStr)
		if err != nil {
			log.Warnf("error parsing gpu string: %s: %+v", gpuStr, err)
		} else {
			result[v1.ResourceName(resources.Gpu.Type)] = gpuQuantity
		}
	}

	return &result, nil
}

// GetLimitsResourceListFromPipeline returns a list of resources that the pipeline,
// maximally is limited to.
func GetLimitsResourceListFromPipeline(pipelineInfo *pps.PipelineInfo) (*v1.ResourceList, error) {
	return getResourceListFromSpec(pipelineInfo.ResourceLimits)
}

// getNumNodes attempts to retrieve the number of nodes in the current k8s
// cluster
func getNumNodes(kubeClient *kube.Clientset) (int, error) {
	nodeList, err := kubeClient.CoreV1().Nodes().List(metav1.ListOptions{})
	if err != nil {
		return 0, fmt.Errorf("unable to retrieve node list from k8s to determine parallelism: %v", err)
	}
	if len(nodeList.Items) == 0 {
		return 0, fmt.Errorf("pachyderm.pps.jobserver: no k8s nodes found")
	}
	return len(nodeList.Items), nil
}

// GetExpectedNumWorkers computes the expected number of workers that
// pachyderm will start given the ParallelismSpec 'spec'.
//
// This is only exported for testing
func GetExpectedNumWorkers(kubeClient *kube.Clientset, spec *pps.ParallelismSpec) (int, error) {
	if spec == nil || (spec.Constant == 0 && spec.Coefficient == 0) {
		return 1, nil
	} else if spec.Constant > 0 && spec.Coefficient == 0 {
		return int(spec.Constant), nil
	} else if spec.Constant == 0 && spec.Coefficient > 0 {
		// Start ('coefficient' * 'nodes') workers. Determine number of workers
		numNodes, err := getNumNodes(kubeClient)
		if err != nil {
			return 0, err
		}
		result := math.Floor(spec.Coefficient * float64(numNodes))
		return int(math.Max(result, 1)), nil
	}
	return 0, fmt.Errorf("unable to interpret ParallelismSpec %+v", spec)
}

// GetExpectedNumHashtrees computes the expected number of hashtrees that
// Pachyderm will create given the HashtreeSpec 'spec'.
func GetExpectedNumHashtrees(spec *pps.HashtreeSpec) (int64, error) {
	if spec == nil || spec.Constant == 0 {
		return 1, nil
	} else if spec.Constant > 0 {
		return int64(spec.Constant), nil
	}
	return 0, fmt.Errorf("unable to interpret HashtreeSpec %+v", spec)
}

// GetPipelineInfo retrieves and returns a valid PipelineInfo from PFS. It does
// the PFS read/unmarshalling of bytes as well as filling in missing fields
func GetPipelineInfo(pachClient *client.APIClient, ptr *pps.EtcdPipelineInfo) (*pps.PipelineInfo, error) {
	result := &pps.PipelineInfo{}
	buf := bytes.Buffer{}
	if err := pachClient.GetFile(ppsconsts.SpecRepo, ptr.SpecCommit.ID, ppsconsts.SpecFile, 0, 0, &buf); err != nil {
		return nil, fmt.Errorf("could not read existing PipelineInfo from PFS: %v", err)
	}
	if err := result.Unmarshal(buf.Bytes()); err != nil {
		return nil, fmt.Errorf("could not unmarshal PipelineInfo bytes from PFS: %v", err)
	}
	result.State = ptr.State
	result.Reason = ptr.Reason
	result.JobCounts = ptr.JobCounts
	result.LastJobState = ptr.LastJobState
	result.SpecCommit = ptr.SpecCommit
	return result, nil
}

// FailPipeline updates the pipeline's state to failed and sets the failure reason
func FailPipeline(ctx context.Context, etcdClient *etcd.Client, pipelinesCollection col.Collection, pipelineName string, reason string) error {
	_, err := col.NewSTM(ctx, etcdClient, func(stm col.STM) error {
		pipelines := pipelinesCollection.ReadWrite(stm)
		pipelinePtr := new(pps.EtcdPipelineInfo)
		if err := pipelines.Get(pipelineName, pipelinePtr); err != nil {
			return err
		}
		pipelinePtr.State = pps.PipelineState_PIPELINE_FAILURE
		pipelinePtr.Reason = reason
		pipelines.Put(pipelineName, pipelinePtr)
		return nil
	})
	return err
}

// JobInput fills in the commits for a JobInfo
func JobInput(pipelineInfo *pps.PipelineInfo, outputCommitInfo *pfs.CommitInfo) *pps.Input {
	// branchToCommit maps strings of the form "<repo>/<branch>" to PFS commits
	branchToCommit := make(map[string]*pfs.Commit)
	key := path.Join
	for _, prov := range outputCommitInfo.Provenance {
		branchToCommit[key(prov.Commit.Repo.Name, prov.Branch.Name)] = prov.Commit
	}
	jobInput := proto.Clone(pipelineInfo.Input).(*pps.Input)
	pps.VisitInput(jobInput, func(input *pps.Input) {
		if input.Pfs != nil {
			if commit, ok := branchToCommit[key(input.Pfs.Repo, input.Pfs.Branch)]; ok {
				input.Pfs.Commit = commit.ID
			}
		}
		if input.Cron != nil {
			if commit, ok := branchToCommit[key(input.Cron.Repo, "master")]; ok {
				input.Cron.Commit = commit.ID
			}
		}
		if input.Git != nil {
			if commit, ok := branchToCommit[key(input.Git.Name, input.Git.Branch)]; ok {
				input.Git.Commit = commit.ID
			}
		}
	})
	return jobInput
}

// PipelineReqFromInfo converts a PipelineInfo into a CreatePipelineRequest.
func PipelineReqFromInfo(pipelineInfo *pps.PipelineInfo) *pps.CreatePipelineRequest {
	return &pps.CreatePipelineRequest{
		Pipeline:         pipelineInfo.Pipeline,
		Transform:        pipelineInfo.Transform,
		ParallelismSpec:  pipelineInfo.ParallelismSpec,
		HashtreeSpec:     pipelineInfo.HashtreeSpec,
		Egress:           pipelineInfo.Egress,
		OutputBranch:     pipelineInfo.OutputBranch,
		ResourceRequests: pipelineInfo.ResourceRequests,
		ResourceLimits:   pipelineInfo.ResourceLimits,
		Input:            pipelineInfo.Input,
		Description:      pipelineInfo.Description,
		CacheSize:        pipelineInfo.CacheSize,
		EnableStats:      pipelineInfo.EnableStats,
		MaxQueueSize:     pipelineInfo.MaxQueueSize,
		Service:          pipelineInfo.Service,
		ChunkSpec:        pipelineInfo.ChunkSpec,
		DatumTimeout:     pipelineInfo.DatumTimeout,
		JobTimeout:       pipelineInfo.JobTimeout,
		Salt:             pipelineInfo.Salt,
		PodSpec:          pipelineInfo.PodSpec,
		PodPatch:         pipelineInfo.PodPatch,
		Spout:            pipelineInfo.Spout,
		SchedulingSpec:   pipelineInfo.SchedulingSpec,
		DatumTries:       pipelineInfo.DatumTries,
		Standby:          pipelineInfo.Standby,
		S3Out:            pipelineInfo.S3Out,
		Metadata:         pipelineInfo.Metadata,
	}
}

// IsTerminal returns 'true' if 'state' indicates that the job is done (i.e.
// the state will not change later: SUCCESS, FAILURE, KILLED) and 'false'
// otherwise.
func IsTerminal(state pps.JobState) bool {
	switch state {
	case pps.JobState_JOB_SUCCESS, pps.JobState_JOB_FAILURE, pps.JobState_JOB_KILLED:
		return true
	case pps.JobState_JOB_STARTING, pps.JobState_JOB_RUNNING, pps.JobState_JOB_MERGING:
		return false
	default:
		panic(fmt.Sprintf("unrecognized job state: %s", state))
	}
}

// UpdateJobState performs the operations involved with a job state transition.
func UpdateJobState(pipelines col.ReadWriteCollection, jobs col.ReadWriteCollection, jobPtr *pps.EtcdJobInfo, state pps.JobState, reason string) error {
	if jobPtr.State == pps.JobState_JOB_FAILURE {
		return fmt.Errorf("cannot put %q in state %s as it's already in state JOB_FAILURE", jobPtr.Job.ID, state.String())
	}

	// Update pipeline
	pipelinePtr := &pps.EtcdPipelineInfo{}
	if err := pipelines.Get(jobPtr.Pipeline.Name, pipelinePtr); err != nil {
		return err
	}
	if pipelinePtr.JobCounts == nil {
		pipelinePtr.JobCounts = make(map[int32]int32)
	}
	if pipelinePtr.JobCounts[int32(jobPtr.State)] != 0 {
		pipelinePtr.JobCounts[int32(jobPtr.State)]--
	}
	pipelinePtr.JobCounts[int32(state)]++
	pipelinePtr.LastJobState = state
	if err := pipelines.Put(jobPtr.Pipeline.Name, pipelinePtr); err != nil {
		return err
	}

	// Update job info
	var err error
	if state == pps.JobState_JOB_STARTING {
		jobPtr.Started, err = types.TimestampProto(time.Now())
	} else if IsTerminal(state) {
		jobPtr.Finished, err = types.TimestampProto(time.Now())
	}
	if err != nil {
		return err
	}
	jobPtr.State = state
	jobPtr.Reason = reason
	return jobs.Put(jobPtr.Job.ID, jobPtr)
}

// ContainsS3Inputs returns 'true' if 'in' is or contains any PFS inputs with
// 'S3' set to true. Any pipelines with s3 inputs lj
func ContainsS3Inputs(in *pps.Input) bool {
	var found bool
	pps.VisitInput(in, func(in *pps.Input) {
		if found {
			return
		}
		if in.Pfs != nil && in.Pfs.S3 {
			found = true
		}
	})
	return found
}
<commit_msg>ppsutil/util.go: Add SidecarS3GatewayService<commit_after>// Package ppsutil contains utilities for various PPS-related tasks, which are
// shared by both the PPS API and the worker binary. These utilities include:
// - Getting the RC name and querying k8s reguarding pipelines
// - Reading and writing pipeline resource requests and limits
// - Reading and writing EtcdPipelineInfos and PipelineInfos[1]
//
// [1] Note that PipelineInfo in particular is complicated because it contains
// fields that are not always set or are stored in multiple places
// ('job_state', for example, is not stored in PFS along with the rest of each
// PipelineInfo, because this field is volatile and we cannot commit to PFS
// every time it changes. 'job_counts' is the same, and 'reason' is in etcd
// because it is only updated alongside 'job_state').  As of 12/7/2017, these
// are the only fields not stored in PFS.
package ppsutil

import (
	"bytes"
	"fmt"
	"math"
	"path"
	"strings"
	"time"

	"github.com/gogo/protobuf/proto"
	"github.com/gogo/protobuf/types"
	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pps"
	col "github.com/pachyderm/pachyderm/src/server/pkg/collection"
	"github.com/pachyderm/pachyderm/src/server/pkg/ppsconsts"

	etcd "github.com/coreos/etcd/clientv3"
	log "github.com/sirupsen/logrus"
	"golang.org/x/net/context"
	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	kube "k8s.io/client-go/kubernetes"
)

// PipelineRepo creates a pfs repo for a given pipeline.
func PipelineRepo(pipeline *pps.Pipeline) *pfs.Repo {
	return &pfs.Repo{Name: pipeline.Name}
}

// PipelineRcName generates the name of the k8s replication controller that
// manages a pipeline's workers
func PipelineRcName(name string, version uint64) string {
	// k8s won't allow RC names that contain upper-case letters
	// or underscores
	// TODO: deal with name collision
	name = strings.Replace(name, "_", "-", -1)
	return fmt.Sprintf("pipeline-%s-v%d", strings.ToLower(name), version)
}

// GetRequestsResourceListFromPipeline returns a list of resources that the pipeline,
// minimally requires.
func GetRequestsResourceListFromPipeline(pipelineInfo *pps.PipelineInfo) (*v1.ResourceList, error) {
	return getResourceListFromSpec(pipelineInfo.ResourceRequests)
}

func getResourceListFromSpec(resources *pps.ResourceSpec) (*v1.ResourceList, error) {
	result := make(v1.ResourceList)

	if resources.Cpu != 0 {
		cpuStr := fmt.Sprintf("%f", resources.Cpu)
		cpuQuantity, err := resource.ParseQuantity(cpuStr)
		if err != nil {
			log.Warnf("error parsing cpu string: %s: %+v", cpuStr, err)
		} else {
			result[v1.ResourceCPU] = cpuQuantity
		}
	}

	if resources.Memory != "" {
		memQuantity, err := resource.ParseQuantity(resources.Memory)
		if err != nil {
			log.Warnf("error parsing memory string: %s: %+v", resources.Memory, err)
		} else {
			result[v1.ResourceMemory] = memQuantity
		}
	}

	if resources.Disk != "" { // needed because not all versions of k8s support disk resources
		diskQuantity, err := resource.ParseQuantity(resources.Disk)
		if err != nil {
			log.Warnf("error parsing disk string: %s: %+v", resources.Disk, err)
		} else {
			result[v1.ResourceEphemeralStorage] = diskQuantity
		}
	}

	if resources.Gpu != nil {
		gpuStr := fmt.Sprintf("%d", resources.Gpu.Number)
		gpuQuantity, err := resource.ParseQuantity(gpuStr)
		if err != nil {
			log.Warnf("error parsing gpu string: %s: %+v", gpuStr, err)
		} else {
			result[v1.ResourceName(resources.Gpu.Type)] = gpuQuantity
		}
	}

	return &result, nil
}

// GetLimitsResourceListFromPipeline returns a list of resources that the pipeline,
// maximally is limited to.
func GetLimitsResourceListFromPipeline(pipelineInfo *pps.PipelineInfo) (*v1.ResourceList, error) {
	return getResourceListFromSpec(pipelineInfo.ResourceLimits)
}

// getNumNodes attempts to retrieve the number of nodes in the current k8s
// cluster
func getNumNodes(kubeClient *kube.Clientset) (int, error) {
	nodeList, err := kubeClient.CoreV1().Nodes().List(metav1.ListOptions{})
	if err != nil {
		return 0, fmt.Errorf("unable to retrieve node list from k8s to determine parallelism: %v", err)
	}
	if len(nodeList.Items) == 0 {
		return 0, fmt.Errorf("pachyderm.pps.jobserver: no k8s nodes found")
	}
	return len(nodeList.Items), nil
}

// GetExpectedNumWorkers computes the expected number of workers that
// pachyderm will start given the ParallelismSpec 'spec'.
//
// This is only exported for testing
func GetExpectedNumWorkers(kubeClient *kube.Clientset, spec *pps.ParallelismSpec) (int, error) {
	if spec == nil || (spec.Constant == 0 && spec.Coefficient == 0) {
		return 1, nil
	} else if spec.Constant > 0 && spec.Coefficient == 0 {
		return int(spec.Constant), nil
	} else if spec.Constant == 0 && spec.Coefficient > 0 {
		// Start ('coefficient' * 'nodes') workers. Determine number of workers
		numNodes, err := getNumNodes(kubeClient)
		if err != nil {
			return 0, err
		}
		result := math.Floor(spec.Coefficient * float64(numNodes))
		return int(math.Max(result, 1)), nil
	}
	return 0, fmt.Errorf("unable to interpret ParallelismSpec %+v", spec)
}

// GetExpectedNumHashtrees computes the expected number of hashtrees that
// Pachyderm will create given the HashtreeSpec 'spec'.
func GetExpectedNumHashtrees(spec *pps.HashtreeSpec) (int64, error) {
	if spec == nil || spec.Constant == 0 {
		return 1, nil
	} else if spec.Constant > 0 {
		return int64(spec.Constant), nil
	}
	return 0, fmt.Errorf("unable to interpret HashtreeSpec %+v", spec)
}

// GetPipelineInfo retrieves and returns a valid PipelineInfo from PFS. It does
// the PFS read/unmarshalling of bytes as well as filling in missing fields
func GetPipelineInfo(pachClient *client.APIClient, ptr *pps.EtcdPipelineInfo) (*pps.PipelineInfo, error) {
	result := &pps.PipelineInfo{}
	buf := bytes.Buffer{}
	if err := pachClient.GetFile(ppsconsts.SpecRepo, ptr.SpecCommit.ID, ppsconsts.SpecFile, 0, 0, &buf); err != nil {
		return nil, fmt.Errorf("could not read existing PipelineInfo from PFS: %v", err)
	}
	if err := result.Unmarshal(buf.Bytes()); err != nil {
		return nil, fmt.Errorf("could not unmarshal PipelineInfo bytes from PFS: %v", err)
	}
	result.State = ptr.State
	result.Reason = ptr.Reason
	result.JobCounts = ptr.JobCounts
	result.LastJobState = ptr.LastJobState
	result.SpecCommit = ptr.SpecCommit
	return result, nil
}

// FailPipeline updates the pipeline's state to failed and sets the failure reason
func FailPipeline(ctx context.Context, etcdClient *etcd.Client, pipelinesCollection col.Collection, pipelineName string, reason string) error {
	_, err := col.NewSTM(ctx, etcdClient, func(stm col.STM) error {
		pipelines := pipelinesCollection.ReadWrite(stm)
		pipelinePtr := new(pps.EtcdPipelineInfo)
		if err := pipelines.Get(pipelineName, pipelinePtr); err != nil {
			return err
		}
		pipelinePtr.State = pps.PipelineState_PIPELINE_FAILURE
		pipelinePtr.Reason = reason
		pipelines.Put(pipelineName, pipelinePtr)
		return nil
	})
	return err
}

// JobInput fills in the commits for a JobInfo
func JobInput(pipelineInfo *pps.PipelineInfo, outputCommitInfo *pfs.CommitInfo) *pps.Input {
	// branchToCommit maps strings of the form "<repo>/<branch>" to PFS commits
	branchToCommit := make(map[string]*pfs.Commit)
	key := path.Join
	for _, prov := range outputCommitInfo.Provenance {
		branchToCommit[key(prov.Commit.Repo.Name, prov.Branch.Name)] = prov.Commit
	}
	jobInput := proto.Clone(pipelineInfo.Input).(*pps.Input)
	pps.VisitInput(jobInput, func(input *pps.Input) {
		if input.Pfs != nil {
			if commit, ok := branchToCommit[key(input.Pfs.Repo, input.Pfs.Branch)]; ok {
				input.Pfs.Commit = commit.ID
			}
		}
		if input.Cron != nil {
			if commit, ok := branchToCommit[key(input.Cron.Repo, "master")]; ok {
				input.Cron.Commit = commit.ID
			}
		}
		if input.Git != nil {
			if commit, ok := branchToCommit[key(input.Git.Name, input.Git.Branch)]; ok {
				input.Git.Commit = commit.ID
			}
		}
	})
	return jobInput
}

// PipelineReqFromInfo converts a PipelineInfo into a CreatePipelineRequest.
func PipelineReqFromInfo(pipelineInfo *pps.PipelineInfo) *pps.CreatePipelineRequest {
	return &pps.CreatePipelineRequest{
		Pipeline:         pipelineInfo.Pipeline,
		Transform:        pipelineInfo.Transform,
		ParallelismSpec:  pipelineInfo.ParallelismSpec,
		HashtreeSpec:     pipelineInfo.HashtreeSpec,
		Egress:           pipelineInfo.Egress,
		OutputBranch:     pipelineInfo.OutputBranch,
		ResourceRequests: pipelineInfo.ResourceRequests,
		ResourceLimits:   pipelineInfo.ResourceLimits,
		Input:            pipelineInfo.Input,
		Description:      pipelineInfo.Description,
		CacheSize:        pipelineInfo.CacheSize,
		EnableStats:      pipelineInfo.EnableStats,
		MaxQueueSize:     pipelineInfo.MaxQueueSize,
		Service:          pipelineInfo.Service,
		ChunkSpec:        pipelineInfo.ChunkSpec,
		DatumTimeout:     pipelineInfo.DatumTimeout,
		JobTimeout:       pipelineInfo.JobTimeout,
		Salt:             pipelineInfo.Salt,
		PodSpec:          pipelineInfo.PodSpec,
		PodPatch:         pipelineInfo.PodPatch,
		Spout:            pipelineInfo.Spout,
		SchedulingSpec:   pipelineInfo.SchedulingSpec,
		DatumTries:       pipelineInfo.DatumTries,
		Standby:          pipelineInfo.Standby,
		S3Out:            pipelineInfo.S3Out,
		Metadata:         pipelineInfo.Metadata,
	}
}

// IsTerminal returns 'true' if 'state' indicates that the job is done (i.e.
// the state will not change later: SUCCESS, FAILURE, KILLED) and 'false'
// otherwise.
func IsTerminal(state pps.JobState) bool {
	switch state {
	case pps.JobState_JOB_SUCCESS, pps.JobState_JOB_FAILURE, pps.JobState_JOB_KILLED:
		return true
	case pps.JobState_JOB_STARTING, pps.JobState_JOB_RUNNING, pps.JobState_JOB_MERGING:
		return false
	default:
		panic(fmt.Sprintf("unrecognized job state: %s", state))
	}
}

// UpdateJobState performs the operations involved with a job state transition.
func UpdateJobState(pipelines col.ReadWriteCollection, jobs col.ReadWriteCollection, jobPtr *pps.EtcdJobInfo, state pps.JobState, reason string) error {
	if jobPtr.State == pps.JobState_JOB_FAILURE {
		return fmt.Errorf("cannot put %q in state %s as it's already in state JOB_FAILURE", jobPtr.Job.ID, state.String())
	}

	// Update pipeline
	pipelinePtr := &pps.EtcdPipelineInfo{}
	if err := pipelines.Get(jobPtr.Pipeline.Name, pipelinePtr); err != nil {
		return err
	}
	if pipelinePtr.JobCounts == nil {
		pipelinePtr.JobCounts = make(map[int32]int32)
	}
	if pipelinePtr.JobCounts[int32(jobPtr.State)] != 0 {
		pipelinePtr.JobCounts[int32(jobPtr.State)]--
	}
	pipelinePtr.JobCounts[int32(state)]++
	pipelinePtr.LastJobState = state
	if err := pipelines.Put(jobPtr.Pipeline.Name, pipelinePtr); err != nil {
		return err
	}

	// Update job info
	var err error
	if state == pps.JobState_JOB_STARTING {
		jobPtr.Started, err = types.TimestampProto(time.Now())
	} else if IsTerminal(state) {
		jobPtr.Finished, err = types.TimestampProto(time.Now())
	}
	if err != nil {
		return err
	}
	jobPtr.State = state
	jobPtr.Reason = reason
	return jobs.Put(jobPtr.Job.ID, jobPtr)
}

// ContainsS3Inputs returns 'true' if 'in' is or contains any PFS inputs with
// 'S3' set to true. Any pipelines with s3 inputs lj
func ContainsS3Inputs(in *pps.Input) bool {
	var found bool
	pps.VisitInput(in, func(in *pps.Input) {
		if found {
			return
		}
		if in.Pfs != nil && in.Pfs.S3 {
			found = true
		}
	})
	return found
}

// SidecarS3GatewayService returns the name of the kubernetes service created
// for the job 'jobID' to hand sidecar s3 gateway requests. This helper is in
// ppsutil because both PPS (which creates the service, in the s3 gateway
// sidecar server) and the worker (which passes the endpoint to the user code)
// need to know it.
func SidecarS3GatewayService(jobID string) string {
	return "s3-" + jobID
}
<|endoftext|>