<commit_before>package s3

import (
	"bytes"
	"crypto/md5"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/dynamodb"
	"github.com/aws/aws-sdk-go/service/s3"
	multierror "github.com/hashicorp/go-multierror"
	uuid "github.com/hashicorp/go-uuid"
	"github.com/hashicorp/terraform/state"
	"github.com/hashicorp/terraform/state/remote"
)

// Store the last saved serial in dynamo with this suffix for consistency checks.
const (
	stateIDSuffix          = "-md5"
	s3ErrCodeInternalError = "InternalError"
)

type RemoteClient struct {
	s3Client             *s3.S3
	dynClient            *dynamodb.DynamoDB
	bucketName           string
	path                 string
	serverSideEncryption bool
	acl                  string
	kmsKeyID             string
	ddbTable             string
}

var (
	// The amount of time we will retry a state waiting for it to match the
	// expected checksum.
	consistencyRetryTimeout = 10 * time.Second

	// delay when polling the state
	consistencyRetryPollInterval = 2 * time.Second
)

// test hook called when checksums don't match
var testChecksumHook func()

func (c *RemoteClient) Get() (payload *remote.Payload, err error) {
	deadline := time.Now().Add(consistencyRetryTimeout)

	// If we have a checksum, and the returned payload doesn't match, we retry
	// up until deadline.
	for {
		payload, err = c.get()
		if err != nil {
			return nil, err
		}

		// If the remote state was manually removed the payload will be nil,
		// but if there's still a digest entry for that state we will still try
		// to compare the MD5 below.
		var digest []byte
		if payload != nil {
			digest = payload.MD5
		}

		// verify that this state is what we expect
		if expected, err := c.getMD5(); err != nil {
			log.Printf("[WARN] failed to fetch state md5: %s", err)
		} else if len(expected) > 0 && !bytes.Equal(expected, digest) {
			log.Printf("[WARN] state md5 mismatch: expected '%x', got '%x'", expected, digest)

			if testChecksumHook != nil {
				testChecksumHook()
			}

			if time.Now().Before(deadline) {
				time.Sleep(consistencyRetryPollInterval)
				log.Println("[INFO] retrying S3 RemoteClient.Get...")
				continue
			}

			return nil, fmt.Errorf(errBadChecksumFmt, digest)
		}

		break
	}

	return payload, err
}

func (c *RemoteClient) get() (*remote.Payload, error) {
	var output *s3.GetObjectOutput
	var err error

	output, err = c.s3Client.GetObject(&s3.GetObjectInput{
		Bucket: &c.bucketName,
		Key:    &c.path,
	})

	if err != nil {
		if awserr, ok := err.(awserr.Error); ok {
			switch awserr.Code() {
			case s3.ErrCodeNoSuchBucket:
				return nil, fmt.Errorf(errS3NoSuchBucket, err)
			case s3.ErrCodeNoSuchKey:
				return nil, nil
			}
		}
		return nil, err
	}

	defer output.Body.Close()

	buf := bytes.NewBuffer(nil)
	if _, err := io.Copy(buf, output.Body); err != nil {
		return nil, fmt.Errorf("Failed to read remote state: %s", err)
	}

	sum := md5.Sum(buf.Bytes())
	payload := &remote.Payload{
		Data: buf.Bytes(),
		MD5:  sum[:],
	}

	// If there was no data, then return nil
	if len(payload.Data) == 0 {
		return nil, nil
	}

	return payload, nil
}

func (c *RemoteClient) Put(data []byte) error {
	contentType := "application/json"
	contentLength := int64(len(data))

	i := &s3.PutObjectInput{
		ContentType:   &contentType,
		ContentLength: &contentLength,
		Body:          bytes.NewReader(data),
		Bucket:        &c.bucketName,
		Key:           &c.path,
	}

	if c.serverSideEncryption {
		if c.kmsKeyID != "" {
			i.SSEKMSKeyId = &c.kmsKeyID
			i.ServerSideEncryption = aws.String("aws:kms")
		} else {
			i.ServerSideEncryption = aws.String("AES256")
		}
	}

	if c.acl != "" {
		i.ACL = aws.String(c.acl)
	}

	log.Printf("[DEBUG] Uploading remote state to S3: %#v", i)

	_, err := c.s3Client.PutObject(i)
	if err != nil {
		return fmt.Errorf("failed to upload state: %s", err)
	}

	sum := md5.Sum(data)
	if err := c.putMD5(sum[:]); err != nil {
		// if this errors out, we unfortunately have to error out altogether,
		// since the next Get will inevitably fail.
		return fmt.Errorf("failed to store state MD5: %s", err)

	}

	return nil
}

func (c *RemoteClient) Delete() error {
	_, err := c.s3Client.DeleteObject(&s3.DeleteObjectInput{
		Bucket: &c.bucketName,
		Key:    &c.path,
	})

	if err != nil {
		return err
	}

	if err := c.deleteMD5(); err != nil {
		log.Printf("error deleting state md5: %s", err)
	}

	return nil
}

func (c *RemoteClient) Lock(info *state.LockInfo) (string, error) {
	if c.ddbTable == "" {
		return "", nil
	}

	info.Path = c.lockPath()

	if info.ID == "" {
		lockID, err := uuid.GenerateUUID()
		if err != nil {
			return "", err
		}

		info.ID = lockID
	}

	putParams := &dynamodb.PutItemInput{
		Item: map[string]*dynamodb.AttributeValue{
			"LockID": {S: aws.String(c.lockPath())},
			"Info":   {S: aws.String(string(info.Marshal()))},
		},
		TableName:           aws.String(c.ddbTable),
		ConditionExpression: aws.String("attribute_not_exists(LockID)"),
	}
	_, err := c.dynClient.PutItem(putParams)

	if err != nil {
		lockInfo, infoErr := c.getLockInfo()
		if infoErr != nil {
			err = multierror.Append(err, infoErr)
		}

		lockErr := &state.LockError{
			Err:  err,
			Info: lockInfo,
		}
		return "", lockErr
	}

	return info.ID, nil
}

func (c *RemoteClient) getMD5() ([]byte, error) {
	if c.ddbTable == "" {
		return nil, nil
	}

	getParams := &dynamodb.GetItemInput{
		Key: map[string]*dynamodb.AttributeValue{
			"LockID": {S: aws.String(c.lockPath() + stateIDSuffix)},
		},
		ProjectionExpression: aws.String("LockID, Digest"),
		TableName:            aws.String(c.ddbTable),
		ConsistentRead:       aws.Bool(true),
	}

	resp, err := c.dynClient.GetItem(getParams)
	if err != nil {
		return nil, err
	}

	var val string
	if v, ok := resp.Item["Digest"]; ok && v.S != nil {
		val = *v.S
	}

	sum, err := hex.DecodeString(val)
	if err != nil || len(sum) != md5.Size {
		return nil, errors.New("invalid md5")
	}

	return sum, nil
}

// store the hash of the state to that clients can check for stale state files.
func (c *RemoteClient) putMD5(sum []byte) error {
	if c.ddbTable == "" {
		return nil
	}

	if len(sum) != md5.Size {
		return errors.New("invalid payload md5")
	}

	putParams := &dynamodb.PutItemInput{
		Item: map[string]*dynamodb.AttributeValue{
			"LockID": {S: aws.String(c.lockPath() + stateIDSuffix)},
			"Digest": {S: aws.String(hex.EncodeToString(sum))},
		},
		TableName: aws.String(c.ddbTable),
	}
	_, err := c.dynClient.PutItem(putParams)
	if err != nil {
		log.Printf("[WARN] failed to record state serial in dynamodb: %s", err)
	}

	return nil
}

// remove the hash value for a deleted state
func (c *RemoteClient) deleteMD5() error {
	if c.ddbTable == "" {
		return nil
	}

	params := &dynamodb.DeleteItemInput{
		Key: map[string]*dynamodb.AttributeValue{
			"LockID": {S: aws.String(c.lockPath() + stateIDSuffix)},
		},
		TableName: aws.String(c.ddbTable),
	}
	if _, err := c.dynClient.DeleteItem(params); err != nil {
		return err
	}
	return nil
}

func (c *RemoteClient) getLockInfo() (*state.LockInfo, error) {
	getParams := &dynamodb.GetItemInput{
		Key: map[string]*dynamodb.AttributeValue{
			"LockID": {S: aws.String(c.lockPath())},
		},
		ProjectionExpression: aws.String("LockID, Info"),
		TableName:            aws.String(c.ddbTable),
		ConsistentRead:       aws.Bool(true),
	}

	resp, err := c.dynClient.GetItem(getParams)
	if err != nil {
		return nil, err
	}

	var infoData string
	if v, ok := resp.Item["Info"]; ok && v.S != nil {
		infoData = *v.S
	}

	lockInfo := &state.LockInfo{}
	err = json.Unmarshal([]byte(infoData), lockInfo)
	if err != nil {
		return nil, err
	}

	return lockInfo, nil
}

func (c *RemoteClient) Unlock(id string) error {
	if c.ddbTable == "" {
		return nil
	}

	lockErr := &state.LockError{}

	// TODO: store the path and lock ID in separate fields, and have proper
	// projection expression only delete the lock if both match, rather than
	// checking the ID from the info field first.
	lockInfo, err := c.getLockInfo()
	if err != nil {
		lockErr.Err = fmt.Errorf("failed to retrieve lock info: %s", err)
		return lockErr
	}
	lockErr.Info = lockInfo

	if lockInfo.ID != id {
		lockErr.Err = fmt.Errorf("lock id %q does not match existing lock", id)
		return lockErr
	}

	params := &dynamodb.DeleteItemInput{
		Key: map[string]*dynamodb.AttributeValue{
			"LockID": {S: aws.String(c.lockPath())},
		},
		TableName: aws.String(c.ddbTable),
	}
	_, err = c.dynClient.DeleteItem(params)

	if err != nil {
		lockErr.Err = err
		return lockErr
	}
	return nil
}

func (c *RemoteClient) lockPath() string {
	return fmt.Sprintf("%s/%s", c.bucketName, c.path)
}

const errBadChecksumFmt = `state data in S3 does not have the expected content.

This may be caused by unusually long delays in S3 processing a previous state
update.  Please wait for a minute or two and try again. If this problem
persists, and neither S3 nor DynamoDB are experiencing an outage, you may need
to manually verify the remote state and update the Digest value stored in the
DynamoDB table to the following value: %x
`

const errS3NoSuchBucket = `S3 bucket does not exist.

The referenced S3 bucket must have been previously created. If the S3 bucket
was created within the last minute, please wait for a minute or two and try
again.

Error: %s
`
<commit_msg>s3 backend: minor typo in putMD5 comment<commit_after>package s3

import (
	"bytes"
	"crypto/md5"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/dynamodb"
	"github.com/aws/aws-sdk-go/service/s3"
	multierror "github.com/hashicorp/go-multierror"
	uuid "github.com/hashicorp/go-uuid"
	"github.com/hashicorp/terraform/state"
	"github.com/hashicorp/terraform/state/remote"
)

// Store the last saved serial in dynamo with this suffix for consistency checks.
const (
	stateIDSuffix          = "-md5"
	s3ErrCodeInternalError = "InternalError"
)

type RemoteClient struct {
	s3Client             *s3.S3
	dynClient            *dynamodb.DynamoDB
	bucketName           string
	path                 string
	serverSideEncryption bool
	acl                  string
	kmsKeyID             string
	ddbTable             string
}

var (
	// The amount of time we will retry a state waiting for it to match the
	// expected checksum.
	consistencyRetryTimeout = 10 * time.Second

	// delay when polling the state
	consistencyRetryPollInterval = 2 * time.Second
)

// test hook called when checksums don't match
var testChecksumHook func()

func (c *RemoteClient) Get() (payload *remote.Payload, err error) {
	deadline := time.Now().Add(consistencyRetryTimeout)

	// If we have a checksum, and the returned payload doesn't match, we retry
	// up until deadline.
	for {
		payload, err = c.get()
		if err != nil {
			return nil, err
		}

		// If the remote state was manually removed the payload will be nil,
		// but if there's still a digest entry for that state we will still try
		// to compare the MD5 below.
		var digest []byte
		if payload != nil {
			digest = payload.MD5
		}

		// verify that this state is what we expect
		if expected, err := c.getMD5(); err != nil {
			log.Printf("[WARN] failed to fetch state md5: %s", err)
		} else if len(expected) > 0 && !bytes.Equal(expected, digest) {
			log.Printf("[WARN] state md5 mismatch: expected '%x', got '%x'", expected, digest)

			if testChecksumHook != nil {
				testChecksumHook()
			}

			if time.Now().Before(deadline) {
				time.Sleep(consistencyRetryPollInterval)
				log.Println("[INFO] retrying S3 RemoteClient.Get...")
				continue
			}

			return nil, fmt.Errorf(errBadChecksumFmt, digest)
		}

		break
	}

	return payload, err
}

func (c *RemoteClient) get() (*remote.Payload, error) {
	var output *s3.GetObjectOutput
	var err error

	output, err = c.s3Client.GetObject(&s3.GetObjectInput{
		Bucket: &c.bucketName,
		Key:    &c.path,
	})

	if err != nil {
		if awserr, ok := err.(awserr.Error); ok {
			switch awserr.Code() {
			case s3.ErrCodeNoSuchBucket:
				return nil, fmt.Errorf(errS3NoSuchBucket, err)
			case s3.ErrCodeNoSuchKey:
				return nil, nil
			}
		}
		return nil, err
	}

	defer output.Body.Close()

	buf := bytes.NewBuffer(nil)
	if _, err := io.Copy(buf, output.Body); err != nil {
		return nil, fmt.Errorf("Failed to read remote state: %s", err)
	}

	sum := md5.Sum(buf.Bytes())
	payload := &remote.Payload{
		Data: buf.Bytes(),
		MD5:  sum[:],
	}

	// If there was no data, then return nil
	if len(payload.Data) == 0 {
		return nil, nil
	}

	return payload, nil
}

func (c *RemoteClient) Put(data []byte) error {
	contentType := "application/json"
	contentLength := int64(len(data))

	i := &s3.PutObjectInput{
		ContentType:   &contentType,
		ContentLength: &contentLength,
		Body:          bytes.NewReader(data),
		Bucket:        &c.bucketName,
		Key:           &c.path,
	}

	if c.serverSideEncryption {
		if c.kmsKeyID != "" {
			i.SSEKMSKeyId = &c.kmsKeyID
			i.ServerSideEncryption = aws.String("aws:kms")
		} else {
			i.ServerSideEncryption = aws.String("AES256")
		}
	}

	if c.acl != "" {
		i.ACL = aws.String(c.acl)
	}

	log.Printf("[DEBUG] Uploading remote state to S3: %#v", i)

	_, err := c.s3Client.PutObject(i)
	if err != nil {
		return fmt.Errorf("failed to upload state: %s", err)
	}

	sum := md5.Sum(data)
	if err := c.putMD5(sum[:]); err != nil {
		// if this errors out, we unfortunately have to error out altogether,
		// since the next Get will inevitably fail.
		return fmt.Errorf("failed to store state MD5: %s", err)

	}

	return nil
}

func (c *RemoteClient) Delete() error {
	_, err := c.s3Client.DeleteObject(&s3.DeleteObjectInput{
		Bucket: &c.bucketName,
		Key:    &c.path,
	})

	if err != nil {
		return err
	}

	if err := c.deleteMD5(); err != nil {
		log.Printf("error deleting state md5: %s", err)
	}

	return nil
}

func (c *RemoteClient) Lock(info *state.LockInfo) (string, error) {
	if c.ddbTable == "" {
		return "", nil
	}

	info.Path = c.lockPath()

	if info.ID == "" {
		lockID, err := uuid.GenerateUUID()
		if err != nil {
			return "", err
		}

		info.ID = lockID
	}

	putParams := &dynamodb.PutItemInput{
		Item: map[string]*dynamodb.AttributeValue{
			"LockID": {S: aws.String(c.lockPath())},
			"Info":   {S: aws.String(string(info.Marshal()))},
		},
		TableName:           aws.String(c.ddbTable),
		ConditionExpression: aws.String("attribute_not_exists(LockID)"),
	}
	_, err := c.dynClient.PutItem(putParams)

	if err != nil {
		lockInfo, infoErr := c.getLockInfo()
		if infoErr != nil {
			err = multierror.Append(err, infoErr)
		}

		lockErr := &state.LockError{
			Err:  err,
			Info: lockInfo,
		}
		return "", lockErr
	}

	return info.ID, nil
}

func (c *RemoteClient) getMD5() ([]byte, error) {
	if c.ddbTable == "" {
		return nil, nil
	}

	getParams := &dynamodb.GetItemInput{
		Key: map[string]*dynamodb.AttributeValue{
			"LockID": {S: aws.String(c.lockPath() + stateIDSuffix)},
		},
		ProjectionExpression: aws.String("LockID, Digest"),
		TableName:            aws.String(c.ddbTable),
		ConsistentRead:       aws.Bool(true),
	}

	resp, err := c.dynClient.GetItem(getParams)
	if err != nil {
		return nil, err
	}

	var val string
	if v, ok := resp.Item["Digest"]; ok && v.S != nil {
		val = *v.S
	}

	sum, err := hex.DecodeString(val)
	if err != nil || len(sum) != md5.Size {
		return nil, errors.New("invalid md5")
	}

	return sum, nil
}

// store the hash of the state so that clients can check for stale state files.
func (c *RemoteClient) putMD5(sum []byte) error {
	if c.ddbTable == "" {
		return nil
	}

	if len(sum) != md5.Size {
		return errors.New("invalid payload md5")
	}

	putParams := &dynamodb.PutItemInput{
		Item: map[string]*dynamodb.AttributeValue{
			"LockID": {S: aws.String(c.lockPath() + stateIDSuffix)},
			"Digest": {S: aws.String(hex.EncodeToString(sum))},
		},
		TableName: aws.String(c.ddbTable),
	}
	_, err := c.dynClient.PutItem(putParams)
	if err != nil {
		log.Printf("[WARN] failed to record state serial in dynamodb: %s", err)
	}

	return nil
}

// remove the hash value for a deleted state
func (c *RemoteClient) deleteMD5() error {
	if c.ddbTable == "" {
		return nil
	}

	params := &dynamodb.DeleteItemInput{
		Key: map[string]*dynamodb.AttributeValue{
			"LockID": {S: aws.String(c.lockPath() + stateIDSuffix)},
		},
		TableName: aws.String(c.ddbTable),
	}
	if _, err := c.dynClient.DeleteItem(params); err != nil {
		return err
	}
	return nil
}

func (c *RemoteClient) getLockInfo() (*state.LockInfo, error) {
	getParams := &dynamodb.GetItemInput{
		Key: map[string]*dynamodb.AttributeValue{
			"LockID": {S: aws.String(c.lockPath())},
		},
		ProjectionExpression: aws.String("LockID, Info"),
		TableName:            aws.String(c.ddbTable),
		ConsistentRead:       aws.Bool(true),
	}

	resp, err := c.dynClient.GetItem(getParams)
	if err != nil {
		return nil, err
	}

	var infoData string
	if v, ok := resp.Item["Info"]; ok && v.S != nil {
		infoData = *v.S
	}

	lockInfo := &state.LockInfo{}
	err = json.Unmarshal([]byte(infoData), lockInfo)
	if err != nil {
		return nil, err
	}

	return lockInfo, nil
}

func (c *RemoteClient) Unlock(id string) error {
	if c.ddbTable == "" {
		return nil
	}

	lockErr := &state.LockError{}

	// TODO: store the path and lock ID in separate fields, and have proper
	// projection expression only delete the lock if both match, rather than
	// checking the ID from the info field first.
	lockInfo, err := c.getLockInfo()
	if err != nil {
		lockErr.Err = fmt.Errorf("failed to retrieve lock info: %s", err)
		return lockErr
	}
	lockErr.Info = lockInfo

	if lockInfo.ID != id {
		lockErr.Err = fmt.Errorf("lock id %q does not match existing lock", id)
		return lockErr
	}

	params := &dynamodb.DeleteItemInput{
		Key: map[string]*dynamodb.AttributeValue{
			"LockID": {S: aws.String(c.lockPath())},
		},
		TableName: aws.String(c.ddbTable),
	}
	_, err = c.dynClient.DeleteItem(params)

	if err != nil {
		lockErr.Err = err
		return lockErr
	}
	return nil
}

func (c *RemoteClient) lockPath() string {
	return fmt.Sprintf("%s/%s", c.bucketName, c.path)
}

const errBadChecksumFmt = `state data in S3 does not have the expected content.

This may be caused by unusually long delays in S3 processing a previous state
update.  Please wait for a minute or two and try again. If this problem
persists, and neither S3 nor DynamoDB are experiencing an outage, you may need
to manually verify the remote state and update the Digest value stored in the
DynamoDB table to the following value: %x
`

const errS3NoSuchBucket = `S3 bucket does not exist.

The referenced S3 bucket must have been previously created. If the S3 bucket
was created within the last minute, please wait for a minute or two and try
again.

Error: %s
`
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"net/url"
	"time"
)

const PollInterval = 100 * time.Millisecond

/*
FilePaginator paginates requests to a /file/read endpoint
sending the response as fileData on the data channel.
If max is a negative number paginate forever or until
a message is received at f.cancel. Otherwise find the
length of the file and paginate until we reach that position.
*/
type FilePaginator struct {
	data   chan *fileData // Results
	cancel chan bool      // Cancel the pagination
	offset int            // Current offset
	max    int            // Maximum offset
	tail   bool           // Begin at the end
	path   string         // Path to file
}

func (f *FilePaginator) init(c *Client) error {
	u := &url.URL{
		Path: "/files/read",
		RawQuery: url.Values{
			"path": []string{f.path},
		}.Encode(),
	}
	fd := &fileData{}
	if err := c.Get(u, fd); err != nil {
		return err
	}
	// If we are tailing output we start
	// at the end of the file. Since it
	// is impossible to tell how many
	// of the previous bytes equate to
	// a single line we just start at the
	// end and wait for more data.
	if f.tail {
		f.offset = fd.Offset
	} else {
		// Signal the end of this file once
		// we reach the total offset.
		f.max = fd.Offset
	}
	return nil
}

func (f *FilePaginator) Next(c *Client, _ ...Filter) error {
	select {
	case <-f.cancel:
		return ErrEndPagination
	default:
	}
	u := &url.URL{
		Path: "/files/read",
		RawQuery: url.Values{
			"offset": []string{fmt.Sprintf("%d", f.offset)},
			"path":   []string{f.path},
		}.Encode(),
	}
	fd := &fileData{}
	if err := c.Get(u, fd); err != nil {
		return err
	}
	f.offset += fd.Length()
	f.data <- fd
	// If max == 0 this is a tail operation
	if f.max > 0 {
		if f.max == f.offset {
			// We've reached the end of the file
			return ErrMaxExceeded
		}
	}
	time.Sleep(PollInterval)
	return nil
}

func (f *FilePaginator) Close() { close(f.data) }

// Pailer reads until n lines
// TODO: Add support for binary data
func Pailer(data <-chan *fileData, cancel chan bool, n int, w io.Writer) error {
	writer := bufio.NewWriter(w)
	var (
		buf   bytes.Buffer
		err   error
		lines int
	)
loop:
	for fd := range data {
		_, err = buf.WriteString(fd.Data)
		if err != nil {
			break loop
		}
		for {
			line, err := buf.ReadBytes('\n')
			if err != nil {
				if err == io.EOF {
					buf.Write(line)
					break
				} else {
					break loop
				}
			}
			_, err = writer.Write(line)
			if err != nil {
				break loop
			}
			writer.Flush()
			lines++
			if n > 0 {
				if lines >= n {
					cancel <- true
					break loop
				}
			}
		}
		if buf.Len() > 0 {
			writer.Write(buf.Bytes())
			writer.Flush()
		}
	}
	return err
}
<commit_msg>break pagination when no data returned and not tailing<commit_after>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"net/url"
	"time"
)

const PollInterval = 100 * time.Millisecond

/*
FilePaginator paginates requests to a /file/read endpoint
sending the response as fileData on the data channel.
If max is a negative number paginate forever or until
a message is received at f.cancel. Otherwise find the
length of the file and paginate until we reach that position.
*/
type FilePaginator struct {
	data   chan *fileData // Results
	cancel chan bool      // Cancel the pagination
	offset int            // Current offset
	max    int            // Maximum offset
	tail   bool           // Begin at the end
	path   string         // Path to file
}

func (f *FilePaginator) init(c *Client) error {
	u := &url.URL{
		Path: "/files/read",
		RawQuery: url.Values{
			"path": []string{f.path},
		}.Encode(),
	}
	fd := &fileData{}
	if err := c.Get(u, fd); err != nil {
		return err
	}
	// If we are tailing output we start
	// at the end of the file. Since it
	// is impossible to tell how many
	// of the previous bytes equate to
	// a single line we just start at the
	// end and wait for more data.
	if f.tail {
		f.offset = fd.Offset
	} else {
		// Signal the end of this file once
		// we reach the total offset.
		f.max = fd.Offset
	}
	return nil
}

func (f *FilePaginator) Next(c *Client, _ ...Filter) error {
	select {
	case <-f.cancel:
		return ErrEndPagination
	default:
	}
	u := &url.URL{
		Path: "/files/read",
		RawQuery: url.Values{
			"offset": []string{fmt.Sprintf("%d", f.offset)},
			"path":   []string{f.path},
		}.Encode(),
	}
	fd := &fileData{}
	if err := c.Get(u, fd); err != nil {
		return err
	}
	f.offset += fd.Length()
	f.data <- fd
	// If max == 0 this is a tail operation
	if f.max > 0 {
		if f.max == f.offset {
			// We've reached the end of the file
			return ErrMaxExceeded
		}
	}
	// If this is not a tail operation and no
	// data was returned break pagination.
	if !f.tail && fd.Length() == 0 {
		return ErrEndPagination
	}
	time.Sleep(PollInterval)
	return nil
}

func (f *FilePaginator) Close() { close(f.data) }

// Pailer reads until n lines
// TODO: Add support for binary data
func Pailer(data <-chan *fileData, cancel chan bool, n int, w io.Writer) error {
	writer := bufio.NewWriter(w)
	var (
		buf   bytes.Buffer
		err   error
		lines int
	)
loop:
	for fd := range data {
		_, err = buf.WriteString(fd.Data)
		if err != nil {
			break loop
		}
		for {
			line, err := buf.ReadBytes('\n')
			if err != nil {
				if err == io.EOF {
					buf.Write(line)
					break
				} else {
					break loop
				}
			}
			_, err = writer.Write(line)
			if err != nil {
				break loop
			}
			writer.Flush()
			lines++
			if n > 0 {
				if lines >= n {
					cancel <- true
					break loop
				}
			}
		}
		if buf.Len() > 0 {
			writer.Write(buf.Bytes())
			writer.Flush()
		}
	}
	return err
}
<|endoftext|>


<commit_before>package backend

import (
	"github.com/chrislusf/seaweedfs/weed/glog"
	"os"
	"time"
)

var (
	_ BackendStorageFile = &DiskFile{}
)

type DiskFile struct {
	File         *os.File
	fullFilePath string
	fileSize     int64
	modTime      time.Time
}

func NewDiskFile(f *os.File) *DiskFile {
	stat, err := f.Stat()
	if err != nil {
		glog.Fatalf("stat file %s: %v", f.Name(), err)
	}

	return &DiskFile{
		fullFilePath: f.Name(),
		File:         f,
		fileSize:     stat.Size(),
		modTime:      stat.ModTime(),
	}
}

func (df *DiskFile) ReadAt(p []byte, off int64) (n int, err error) {
	return df.File.ReadAt(p, off)
}

func (df *DiskFile) WriteAt(p []byte, off int64) (n int, err error) {
	n, err = df.File.WriteAt(p, off)
	if err == nil {
		waterMark := off + int64(n)
		if waterMark > df.fileSize {
			df.fileSize = waterMark
			df.modTime = time.Now()
		}
	}
	return
}

func (df *DiskFile) Truncate(off int64) error {
	err := df.File.Truncate(off)
	if err == nil {
		df.fileSize = off
		df.modTime = time.Now()
	}
	return err
}

func (df *DiskFile) Close() error {
	return df.File.Close()
}

func (df *DiskFile) GetStat() (datSize int64, modTime time.Time, err error) {
	return df.fileSize, df.modTime, nil
}

func (df *DiskFile) Name() string {
	return df.fullFilePath
}

func (df *DiskFile) Sync() error {
	return df.File.Sync()
}
<commit_msg>Revert "avoid file.Stat()"<commit_after>package backend

import (
	"os"
	"time"
)

var (
	_ BackendStorageFile = &DiskFile{}
)

type DiskFile struct {
	File         *os.File
	fullFilePath string
}

func NewDiskFile(f *os.File) *DiskFile {
	return &DiskFile{
		fullFilePath: f.Name(),
		File:         f,
	}
}

func (df *DiskFile) ReadAt(p []byte, off int64) (n int, err error) {
	return df.File.ReadAt(p, off)
}

func (df *DiskFile) WriteAt(p []byte, off int64) (n int, err error) {
	return df.File.WriteAt(p, off)
}

func (df *DiskFile) Truncate(off int64) error {
	return df.File.Truncate(off)
}

func (df *DiskFile) Close() error {
	return df.File.Close()
}

func (df *DiskFile) GetStat() (datSize int64, modTime time.Time, err error) {
	stat, e := df.File.Stat()
	if e == nil {
		return stat.Size(), stat.ModTime(), nil
	}
	return 0, time.Time{}, err
}

func (df *DiskFile) Name() string {
	return df.fullFilePath
}

func (df *DiskFile) Sync() error {
	return df.File.Sync()
}
<|endoftext|>


<commit_before>// Copyright 2014 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package graphics

import (
	"errors"
	"github.com/hajimehoshi/ebiten/internal"
	"github.com/hajimehoshi/ebiten/internal/opengl"
	"image"
	"image/draw"
)

func adjustImageForTexture(img image.Image) *image.RGBA {
	width, height := img.Bounds().Size().X, img.Bounds().Size().Y
	adjustedImageBounds := image.Rectangle{
		image.ZP,
		image.Point{
			internal.NextPowerOf2Int(width),
			internal.NextPowerOf2Int(height),
		},
	}
	if nrgba, ok := img.(*image.RGBA); ok && img.Bounds() == adjustedImageBounds {
		return nrgba
	}

	adjustedImage := image.NewRGBA(adjustedImageBounds)
	dstBounds := image.Rectangle{
		image.ZP,
		img.Bounds().Size(),
	}
	draw.Draw(adjustedImage, dstBounds, img, image.ZP, draw.Src)
	return adjustedImage
}

type Texture struct {
	native opengl.Texture
	width  int
	height int
}

func (t *Texture) Size() (width, height int) {
	return t.width, t.height
}

func NewTexture(c *opengl.Context, width, height int, filter opengl.Filter) (*Texture, error) {
	w := internal.NextPowerOf2Int(width)
	h := internal.NextPowerOf2Int(height)
	if w < 4 {
		return nil, errors.New("width must be equal or more than 4.")
	}
	if h < 4 {
		return nil, errors.New("height must be equal or more than 4.")
	}
	native, err := c.NewTexture(w, h, nil, filter)
	if err != nil {
		return nil, err
	}
	return &Texture{native, width, height}, nil
}

func NewTextureFromImage(c *opengl.Context, img image.Image, filter opengl.Filter) (*Texture, error) {
	origSize := img.Bounds().Size()
	if origSize.X < 4 {
		return nil, errors.New("width must be equal or more than 4.")
	}
	if origSize.Y < 4 {
		return nil, errors.New("height must be equal or more than 4.")
	}
	adjustedImage := adjustImageForTexture(img)
	size := adjustedImage.Bounds().Size()
	native, err := c.NewTexture(size.X, size.Y, adjustedImage.Pix, filter)
	if err != nil {
		return nil, err
	}
	return &Texture{native, origSize.X, origSize.Y}, nil
}

func (t *Texture) Dispose(c *opengl.Context) {
	c.DeleteTexture(t.native)
}

func (t *Texture) ReplacePixels(c *opengl.Context, p []uint8) error {
	c.BindTexture(t.native)
	c.TexSubImage2D(p, t.width, t.height)
	return nil
}
<commit_msg>Fix a variable name<commit_after>// Copyright 2014 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package graphics

import (
	"errors"
	"github.com/hajimehoshi/ebiten/internal"
	"github.com/hajimehoshi/ebiten/internal/opengl"
	"image"
	"image/draw"
)

func adjustImageForTexture(img image.Image) *image.RGBA {
	width, height := img.Bounds().Size().X, img.Bounds().Size().Y
	adjustedImageBounds := image.Rectangle{
		image.ZP,
		image.Point{
			internal.NextPowerOf2Int(width),
			internal.NextPowerOf2Int(height),
		},
	}
	if adjustedImage, ok := img.(*image.RGBA); ok && img.Bounds() == adjustedImageBounds {
		return adjustedImage
	}

	adjustedImage := image.NewRGBA(adjustedImageBounds)
	dstBounds := image.Rectangle{
		image.ZP,
		img.Bounds().Size(),
	}
	draw.Draw(adjustedImage, dstBounds, img, image.ZP, draw.Src)
	return adjustedImage
}

type Texture struct {
	native opengl.Texture
	width  int
	height int
}

func (t *Texture) Size() (width, height int) {
	return t.width, t.height
}

func NewTexture(c *opengl.Context, width, height int, filter opengl.Filter) (*Texture, error) {
	w := internal.NextPowerOf2Int(width)
	h := internal.NextPowerOf2Int(height)
	if w < 4 {
		return nil, errors.New("width must be equal or more than 4.")
	}
	if h < 4 {
		return nil, errors.New("height must be equal or more than 4.")
	}
	native, err := c.NewTexture(w, h, nil, filter)
	if err != nil {
		return nil, err
	}
	return &Texture{native, width, height}, nil
}

func NewTextureFromImage(c *opengl.Context, img image.Image, filter opengl.Filter) (*Texture, error) {
	origSize := img.Bounds().Size()
	if origSize.X < 4 {
		return nil, errors.New("width must be equal or more than 4.")
	}
	if origSize.Y < 4 {
		return nil, errors.New("height must be equal or more than 4.")
	}
	adjustedImage := adjustImageForTexture(img)
	size := adjustedImage.Bounds().Size()
	native, err := c.NewTexture(size.X, size.Y, adjustedImage.Pix, filter)
	if err != nil {
		return nil, err
	}
	return &Texture{native, origSize.X, origSize.Y}, nil
}

func (t *Texture) Dispose(c *opengl.Context) {
	c.DeleteTexture(t.native)
}

func (t *Texture) ReplacePixels(c *opengl.Context, p []uint8) error {
	c.BindTexture(t.native)
	c.TexSubImage2D(p, t.width, t.height)
	return nil
}
<|endoftext|>


<commit_before>// Package parg provides functionallity to emulate Python's argparse for
// setting-up and parsing a program's flags & arguments.
package parg

import (
	"fmt"
	"os"
	"strings"
)

// parser contains program-level settings and information, stores flags,
// and values collected upon parsing.
type parser struct {
	ProgramName string
	AllowAbbrev bool
	Flags       []*Flag
	UsageText   string
	Values      map[string]interface{}
}

// AddHelp adds a new flag to output usage information for the current parser
// and each of its flags.
func (p *parser) AddHelp() *parser {
	helpFlag := NewFlag("help", "Display usage information").Action(ShowHelp)
	shortHelpFlag := NewFlag("h", "Display usage information").Action(ShowHelp).Dest("help")

	p.Flags = append(p.Flags, helpFlag, shortHelpFlag)
	return p
}

// AddFlag appends the provided flag to the current parser.
func (p *parser) AddFlag(f *Flag) *parser {
	p.Flags = append(p.Flags, f)
	return p
}

// GetFlag retrieves the first flag with a public name matching the specified
// name, or will otherwise return an error.
func (p *parser) GetFlag(name string) (*Flag, error) {
	for _, flag := range p.Flags {
		if flag.PublicName == name {
			return flag, nil
		}
	}

	return nil, fmt.Errorf("No argument named: '%s'", name)
}

// GetHelp returns a string containing the parser's description text,
// and the usage information for each flag currently incorperated within
// the parser.
func (p *parser) GetHelp() string {
	// Get screen width to determine max line lengths later.
	screenWidth := getScreenWidth()

	var positional []*Flag
	var notPositional []*Flag
	var usage []string

	header := []string{"usage:", p.ProgramName}
	headerIndent := len(join(" ", header...))
	headerLen := headerIndent

	var notPosArgs []string
	var posArgs []string
	longest := 0

	for _, arg := range p.Flags {
		//if arg.IsPositional == false {
		notPositional = append(notPositional, arg)
		name := arg.GetUsage()
		if len(name) > longest {
			longest = len(name)
		}

		argUsg := arg.GetUsage()
		notPosArgs = append(notPosArgs, arg.GetUsage())
		headerLen = headerLen + len(argUsg)
		if headerLen+len(argUsg) > screenWidth {
			headerLen = headerIndent
			notPosArgs = append(notPosArgs, join("", "\n", spacer(headerIndent)))
		}
		/*} else {
			positional = append(positional, arg)
			name := arg.GetUsage()
			if len(name) > longest {
				longest = len(name)
			}

			argUsg := arg.GetUsage()
			posArgs = append(posArgs, arg.GetUsage())
			headerLen = headerLen + len(argUsg)
			if headerLen+len(argUsg) > screenWidth {
				headerLen = headerIndent
				posArgs = append(posArgs, join("", "\n", spacer(headerIndent)))
			}
		}*/
	}

	longest = longest + 4

	header = append(header, notPosArgs...)
	header = append(header, posArgs...)

	usage = append(usage, join(" ", header...), "\n")

	if len(p.UsageText) > 0 {
		usage = append(usage, "\n", p.UsageText, "\n")
	}

	if len(positional) > 0 {
		usage = append(usage, "\n", "positional arguments:", "\n")
		var names []string
		var help []string

		for _, arg := range positional {
			names = append(names, arg.GetUsage())
			help = append(help, arg.HelpText)
		}

		var lines []string
		for i, name := range names {
			lines = append(lines, "  ", name)
			lines = append(lines, spacer(longest-len(name)-2))
			if longest > screenWidth {
				lines = append(lines, "\n", spacer(longest))
			}
			lines = append(lines, help[i], "\n")
		}
		usage = append(usage, lines...)
	}

	if len(notPositional) > 0 {
		usage = append(usage, "\n", "optional arguments:", "\n")
		var names []string
		var help []string

		for _, arg := range notPositional {
			names = append(names, arg.DisplayName())
			help = append(help, arg.HelpText)
		}

		var lines []string
		for i, name := range names {
			lines = append(lines, "  ", name)
			lines = append(lines, spacer(longest-len(name)-2))
			if longest > screenWidth {
				lines = append(lines, "\n", spacer(longest))
			}

			helpLines := wordWrap(help[i], screenWidth-longest)
			lines = append(lines, helpLines[0], "\n")
			if len(helpLines) > 1 {
				for _, helpLine := range helpLines[1:len(helpLines)] {
					lines = append(lines, spacer(longest), helpLine, "\n")
				}
			}
		}
		usage = append(usage, lines...)
	}

	return join("", usage...)
}

// Parser accepts a slice of strings as flags and arguments to be parsed. The
// parser will call each encountered flag's action. Unexpected flags will
// cause an error. All errors are returned.
func (p *parser) Parse(allArgs ...string) error {
	p.Values = make(map[string]interface{})
	requiredFlags := make(map[string]*Flag)

	for _, flag := range p.Flags {
		if flag.IsRequired == true {
			requiredFlags[flag.PublicName] = flag
		}
		p.Values[flag.DestName] = flag.DefaultVal
	}

	flagNames, args := extractFlags(allArgs...)
	for _, flagName := range flagNames {
		var flag *Flag

		for _, f := range p.Flags {
			if flagName == f.PublicName {
				if _, ok := requiredFlags[flagName]; ok {
					delete(requiredFlags, flagName)
				}
				flag = f
				break
			}
		}

		if flag == nil {
			return fmt.Errorf("flag '%s' is not a valid flag.", flagName)
		}

		_, err := flag.DesiredAction(p, flag, args...)
		if err != nil {
			return err
		}
	}

	if len(requiredFlags) != 0 {
		for _, flag := range requiredFlags {
			return fmt.Errorf("flag '%s' is required but was not present.", flag.DisplayName())
		}
	}
	return nil
	/*
		for _, opt := range p.Arguments {
			if opt.IsPositional {
				positionals = append(positionals, opt)
			} else {
				flags = append(flags, opt)
			}
			p.Values[opt.Name] = opt.DefaultVal
		}

		count := 0
		max := len(args)
		for count < max {
			arg := args[count]
			if arg == "--" {
				count = count + 2
				continue
			}
			if isFlagFormat(arg) {
				flagName := getFlagName(arg)
				for _, flag := range flags {
					if flag.Name == flagName {
						if count+1 < max {
							_, err := flag.ActionType(p, flag, args[count+1:]...)
							if err != nil {
								fmt.Println(err.Error())
								break
							}
						} else {
							_, err := flag.ActionType(p, flag, "")
							if err != nil {
								fmt.Println(err.Error())
								break
							}
						}
					}
				}
			}
			count++
		}*/
}

// Path will set the parser's program name to the program name specified by the
// provided path.
func (p *parser) Path(progPath string) *parser {
	paths := strings.Split(progPath, string(os.PathSeparator))
	return p.Prog(paths[len(paths)-1])
}

// Prog sets the name of the parser directly.
func (p *parser) Prog(name string) *parser {
	p.ProgramName = name
	return p
}

// Usage sets the provide string as the usage/description text for the parser.
func (p *parser) Usage(usage string) *parser {
	p.UsageText = usage
	return p
}

// ShowHelp outputs to stdout the parser's generated help text.
func (p *parser) ShowHelp() *parser {
	fmt.Println(p.GetHelp())

	return p
}

// NewParser returns an instantiated pointer to a new parser instance, with
// a description matching the provided string.
func NewParser(desc string) *parser {
	p := parser{UsageText: desc}
	if len(os.Args) >= 1 {
		p.Path(os.Args[0])
	}
	return &p
}
<commit_msg>Removed legacy commented code<commit_after>// Package parg provides functionallity to emulate Python's argparse for
// setting-up and parsing a program's flags & arguments.
package parg

import (
	"fmt"
	"os"
	"strings"
)

// parser contains program-level settings and information, stores flags,
// and values collected upon parsing.
type parser struct {
	ProgramName string
	AllowAbbrev bool
	Flags       []*Flag
	UsageText   string
	Values      map[string]interface{}
}

// AddHelp adds a new flag to output usage information for the current parser
// and each of its flags.
func (p *parser) AddHelp() *parser {
	helpFlag := NewFlag("help", "Display usage information").Action(ShowHelp)
	shortHelpFlag := NewFlag("h", "Display usage information").Action(ShowHelp).Dest("help")

	p.Flags = append(p.Flags, helpFlag, shortHelpFlag)
	return p
}

// AddFlag appends the provided flag to the current parser.
func (p *parser) AddFlag(f *Flag) *parser {
	p.Flags = append(p.Flags, f)
	return p
}

// GetFlag retrieves the first flag with a public name matching the specified
// name, or will otherwise return an error.
func (p *parser) GetFlag(name string) (*Flag, error) {
	for _, flag := range p.Flags {
		if flag.PublicName == name {
			return flag, nil
		}
	}

	return nil, fmt.Errorf("No argument named: '%s'", name)
}

// GetHelp returns a string containing the parser's description text,
// and the usage information for each flag currently incorperated within
// the parser.
func (p *parser) GetHelp() string {
	// Get screen width to determine max line lengths later.
	screenWidth := getScreenWidth()

	var positional []*Flag
	var notPositional []*Flag
	var usage []string

	header := []string{"usage:", p.ProgramName}
	headerIndent := len(join(" ", header...))
	headerLen := headerIndent

	var notPosArgs []string
	var posArgs []string
	longest := 0

	for _, arg := range p.Flags {
		//if arg.IsPositional == false {
		notPositional = append(notPositional, arg)
		name := arg.GetUsage()
		if len(name) > longest {
			longest = len(name)
		}

		argUsg := arg.GetUsage()
		notPosArgs = append(notPosArgs, arg.GetUsage())
		headerLen = headerLen + len(argUsg)
		if headerLen+len(argUsg) > screenWidth {
			headerLen = headerIndent
			notPosArgs = append(notPosArgs, join("", "\n", spacer(headerIndent)))
		}
		/*} else {
			positional = append(positional, arg)
			name := arg.GetUsage()
			if len(name) > longest {
				longest = len(name)
			}

			argUsg := arg.GetUsage()
			posArgs = append(posArgs, arg.GetUsage())
			headerLen = headerLen + len(argUsg)
			if headerLen+len(argUsg) > screenWidth {
				headerLen = headerIndent
				posArgs = append(posArgs, join("", "\n", spacer(headerIndent)))
			}
		}*/
	}

	longest = longest + 4

	header = append(header, notPosArgs...)
	header = append(header, posArgs...)

	usage = append(usage, join(" ", header...), "\n")

	if len(p.UsageText) > 0 {
		usage = append(usage, "\n", p.UsageText, "\n")
	}

	if len(positional) > 0 {
		usage = append(usage, "\n", "positional arguments:", "\n")
		var names []string
		var help []string

		for _, arg := range positional {
			names = append(names, arg.GetUsage())
			help = append(help, arg.HelpText)
		}

		var lines []string
		for i, name := range names {
			lines = append(lines, "  ", name)
			lines = append(lines, spacer(longest-len(name)-2))
			if longest > screenWidth {
				lines = append(lines, "\n", spacer(longest))
			}
			lines = append(lines, help[i], "\n")
		}
		usage = append(usage, lines...)
	}

	if len(notPositional) > 0 {
		usage = append(usage, "\n", "optional arguments:", "\n")
		var names []string
		var help []string

		for _, arg := range notPositional {
			names = append(names, arg.DisplayName())
			help = append(help, arg.HelpText)
		}

		var lines []string
		for i, name := range names {
			lines = append(lines, "  ", name)
			lines = append(lines, spacer(longest-len(name)-2))
			if longest > screenWidth {
				lines = append(lines, "\n", spacer(longest))
			}

			helpLines := wordWrap(help[i], screenWidth-longest)
			lines = append(lines, helpLines[0], "\n")
			if len(helpLines) > 1 {
				for _, helpLine := range helpLines[1:len(helpLines)] {
					lines = append(lines, spacer(longest), helpLine, "\n")
				}
			}
		}
		usage = append(usage, lines...)
	}

	return join("", usage...)
}

// Parser accepts a slice of strings as flags and arguments to be parsed. The
// parser will call each encountered flag's action. Unexpected flags will
// cause an error. All errors are returned.
func (p *parser) Parse(allArgs ...string) error {
	p.Values = make(map[string]interface{})
	requiredFlags := make(map[string]*Flag)

	for _, flag := range p.Flags {
		if flag.IsRequired == true {
			requiredFlags[flag.PublicName] = flag
		}
		p.Values[flag.DestName] = flag.DefaultVal
	}

	flagNames, args := extractFlags(allArgs...)
	for _, flagName := range flagNames {
		var flag *Flag

		for _, f := range p.Flags {
			if flagName == f.PublicName {
				if _, ok := requiredFlags[flagName]; ok {
					delete(requiredFlags, flagName)
				}
				flag = f
				break
			}
		}

		if flag == nil {
			return fmt.Errorf("flag '%s' is not a valid flag.", flagName)
		}

		_, err := flag.DesiredAction(p, flag, args...)
		if err != nil {
			return err
		}
	}

	if len(requiredFlags) != 0 {
		for _, flag := range requiredFlags {
			return fmt.Errorf("flag '%s' is required but was not present.", flag.DisplayName())
		}
	}
	return nil
}

// Path will set the parser's program name to the program name specified by the
// provided path.
func (p *parser) Path(progPath string) *parser {
	paths := strings.Split(progPath, string(os.PathSeparator))
	return p.Prog(paths[len(paths)-1])
}

// Prog sets the name of the parser directly.
func (p *parser) Prog(name string) *parser {
	p.ProgramName = name
	return p
}

// Usage sets the provide string as the usage/description text for the parser.
func (p *parser) Usage(usage string) *parser {
	p.UsageText = usage
	return p
}

// ShowHelp outputs to stdout the parser's generated help text.
func (p *parser) ShowHelp() *parser {
	fmt.Println(p.GetHelp())

	return p
}

// NewParser returns an instantiated pointer to a new parser instance, with
// a description matching the provided string.
func NewParser(desc string) *parser {
	p := parser{UsageText: desc}
	if len(os.Args) >= 1 {
		p.Path(os.Args[0])
	}
	return &p
}
<|endoftext|>


<commit_before>package sprite_sass

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"path/filepath"
	"strings"

	// "github.com/kr/pretty"
)

func init() {
	log.SetFlags(log.Lshortfile | log.Ldate | log.Ltime)
}

type Replace struct {
	Start, End int
	Value      []byte
}

type Parser struct {
	shift                int
	Chop                 []Replace
	Pwd, Input, ImageDir string
	Includes             []string
	Items                []Item
	Output               []byte
	Sprites              map[string]ImageList
	Vars                 map[string]string
}

// Parser reads the tokens from the lexer and performs
// conversions and/or substitutions for sprite*() calls.
//
// Parser creates a map of all variables and sprites
// (created via sprite-map calls).
func (p *Parser) Start(f string) []byte {
	if p.ImageDir == "" {
		p.ImageDir = "."
	}
	p.Vars = make(map[string]string)
	p.Sprites = make(map[string]ImageList)
	fvar, err := ioutil.ReadFile(f)
	if err != nil {
		panic(err)
	}

	p.Items, p.Input, err = p.parser(filepath.Dir(f), string(fvar))
	tokens := p.Items
	if err != nil {
		panic(err)
	}
	var (
		def, cmd string
	)
	for i := 0; i < len(tokens); i = i + 1 {
		token := tokens[i]
		last := i
		// Generate list of vars
		if token.Type == VAR {
			def = fmt.Sprintf("%s", token)
			val := ""
			nested := false
			for {
				i++
				token = tokens[i]
				switch token.Type {
				case LPAREN:
					nested = true
				case RPAREN:
					nested = false
				case CMD:
					cmd = fmt.Sprintf("%s", token)
					val += cmd
				case FILE:
					i = p.File(cmd, last, i)
					def = ""
					cmd = ""
				case SUB:
					// Can this ever happen, do we care?
					fmt.Println("SUB")
					// fmt.Println(p.Input[token.Pos-100 : token.Pos+100])
					// fmt.Println(token)
				default:
					//fmt.Printf("Default: %s\n", token)
					val += fmt.Sprintf("%s", token)
				}

				if !nested && tokens[i].Type != CMD {
					break
				}
			}
			if def != "" {
				p.Vars[def] = val
			}
			//Replace subsitution tokens
		} else if token.Type == SUB {
			if cmd == "sprite" {
				//Capture sprite
				sprite := p.Sprites[fmt.Sprintf("%s", token)]
				//Capture filename
				i++
				name := fmt.Sprintf("%s", tokens[i])
				repl := sprite.CSS(name)
				p.Mark(tokens[i-3].Pos, tokens[i+2].Pos, repl)
				tokens = append(tokens[:i-3], tokens[i:]...)
				i = i - 3
				def = ""
				cmd = ""
			} else if cmd == "sprite-dimensions" {
				sprite := p.Sprites[fmt.Sprintf("%s", token)]
				// Walk forward to file name
				i++
				repl := fmt.Sprintf("width: %dpx;\n"+
					"height: %dpx;\n",
					sprite.Width(), sprite.Height())
				p.Mark(tokens[i-4].Pos, tokens[i+3].Pos, repl)
				tokens = append(tokens[:i-4], tokens[i:]...)
				i = i - 4
				def = ""
				cmd = ""
			} else {
				tokens[i].Value = p.Vars[token.Value]
			}
		} else if token.Type == CMD {
			cmd = fmt.Sprintf("%s", token)
		}
	}
	p.Output = process(p.Input, p.Items, 0)
	p.Replace()
	// DEBUG
	// for _, item := range p.Items {
	// 	fmt.Printf("%s %s\n", item.Type, item)
	// }
	return p.Output
}

// Replace iterates through the list of substrings to
// cut or replace, adjusting for shift of the output
// buffer as a result of these ops.
func (p *Parser) Replace() {
	for _, c := range p.Chop {
		begin := c.Start - p.shift
		end := c.End - p.shift
		// fmt.Println("Cut: ", string(p.Output[begin:end]))
		// Adjust shift for number of bytes deleted and inserted
		p.shift += end - begin
		p.shift -= len(c.Value)
		suf := append(c.Value, p.Output[end:]...)
		p.Output = append(p.Output[:begin], suf...)
	}
}

// Mark segments of the input string for future deletion.
func (p *Parser) Mark(start, end int, val string) {
	// fmt.Println("Mark:", string(p.Input[start:end]), "~")
	p.Chop = append(p.Chop, Replace{start, end, []byte(val)})
}

// Processes file which usually mean cutting some of the input
// text.
func (p *Parser) File(cmd string, start, end int) int {
	first := p.Items[start]
	item := p.Items[end]
	// Find the next newline, failing that find the semicolon
	i := end
	if cmd == "sprite-map" {
		for ; p.Items[i].Type != RPAREN; i++ {
		}
		i = i - 1
		// Verify that the statement ends with semicolon
		interest := p.Items[i+3]
		// Mark this area for deletion, since doing so now would
		// invalidate all subsequent tokens positions.
		p.Mark(first.Pos, interest.Pos, "")
		if cmd == "sprite-map" {
			imgs := ImageList{}
			glob := fmt.Sprintf("%s", item)
			name := fmt.Sprintf("%s", p.Items[start])
			imgs.Decode(p.ImageDir + "/" + glob)
			imgs.Vertical = true
			imgs.Combine()
			p.Sprites[name] = imgs
		}
		//TODO: Generate filename
		//imgs.Export("generated.png")
	}
	return i
}

func process(in string, items []Item, pos int) []byte {

	var out []byte
	l := len(items)

	if pos >= len(in) {
		return []byte("")
	}

	// TODO: There's an error where items[1] has an invalid
	// position.
	if l > 1 && items[1].Pos > items[0].Pos {
		out = append(out, in[items[0].Pos:items[1].Pos]...)
		out = append(out, process(in, items[1:], pos)...)
	} else {
		out = append(out, in[items[0].Pos:]...)
	}

	return out
}

// parser recrusively resolves all imports and tokenizes the
// input string
func (p *Parser) parser(pwd, input string) ([]Item, string, error) {

	var (
		status    []Item
		importing bool
		output    []byte
		pos       int
		last      *Item
	)

	lex := New(func(lex *Lexer) StateFn {
		return lex.Action()
	}, input)

	for {
		item := lex.Next()
		err := item.Error()
		//fmt.Println(item.Type, item.Value)
		if err != nil {
			return nil, string(output),
				fmt.Errorf("Error: %v (pos %d)", err, item.Pos)
		}
		switch item.Type {
		case ItemEOF:
			output = append(output, input[pos:]...)
			return status, string(output), nil
		case IMPORT:
			output = append(output, input[pos:item.Pos]...)
			last = item
			importing = true
		case INCLUDE, CMT:
			output = append(output, input[pos:item.Pos]...)
			pos = item.Pos
			status = append(status, *item)
		default:
			if importing {

				pwd, contents, err := p.ImportPath(pwd, fmt.Sprintf("%s", *item))

				if err != nil {
					log.Println(err)
				}
				//Eat the semicolon
				item := lex.Next()
				pos = item.Pos + len(item.Value)
				if item.Type != SEMIC {
					panic("@import must be followed by ;")
				}

				moreTokens, moreOutput, err := p.parser(
					pwd,
					contents)
				// Lexer needs to be adjusted for current
				// position of end of @import
				for i, _ := range moreTokens {
					moreTokens[i].Pos += last.Pos
				}

				if err != nil {
					panic(err)
				}
				output = append(output, moreOutput...)
				status = append(status, moreTokens...)
				importing = false
			} else {
				output = append(output, input[pos:item.Pos]...)
				pos = item.Pos
				status = append(status, *item)
			}
		}
	}
}

func (p *Parser) ImportPath(dir, file string) (string, string, error) {
	// fmt.Println("Importing: " + file)
	baseerr := ""
	//Load and retrieve all tokens from imported file
	path, err := filepath.Abs(fmt.Sprintf("%s/%s.scss", dir, file))
	if err != nil {
		panic(err)
	}
	pwd := filepath.Dir(path)
	// Sass put _ in front of imported files
	fpath := pwd + "/_" + filepath.Base(path)
	contents, err := ioutil.ReadFile(fpath)
	if err == nil {
		return pwd, string(contents), nil
	}
	baseerr += fpath + "\n"
	if strings.HasSuffix(err.Error(), "no such file or directory") {
		// Look through the import path for the file
		for _, lib := range p.Includes {
			path, err := filepath.Abs(lib + "/" + file)
			if err != nil {
				panic(err)
			}
			pwd := filepath.Dir(path)
			fpath = pwd + "/_" + filepath.Base(path) + ".scss"
			contents, err := ioutil.ReadFile(fpath)
			baseerr += fpath + "\n"
			if err == nil {
				return pwd, string(contents), nil
			}
		}
	}

	return pwd, string(contents), errors.New("Could not import: " +
		file + "\nTried:\n" + baseerr)
}
<commit_msg>Use dimensions generator<commit_after>package sprite_sass

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"path/filepath"
	"strings"

	// "github.com/kr/pretty"
)

func init() {
	log.SetFlags(log.Lshortfile | log.Ldate | log.Ltime)
}

type Replace struct {
	Start, End int
	Value      []byte
}

type Parser struct {
	shift                int
	Chop                 []Replace
	Pwd, Input, ImageDir string
	Includes             []string
	Items                []Item
	Output               []byte
	Sprites              map[string]ImageList
	Vars                 map[string]string
}

// Parser reads the tokens from the lexer and performs
// conversions and/or substitutions for sprite*() calls.
//
// Parser creates a map of all variables and sprites
// (created via sprite-map calls).
func (p *Parser) Start(f string) []byte {
	if p.ImageDir == "" {
		p.ImageDir = "."
	}
	p.Vars = make(map[string]string)
	p.Sprites = make(map[string]ImageList)
	fvar, err := ioutil.ReadFile(f)
	if err != nil {
		panic(err)
	}

	p.Items, p.Input, err = p.parser(filepath.Dir(f), string(fvar))
	tokens := p.Items
	if err != nil {
		panic(err)
	}
	var (
		def, cmd string
	)
	for i := 0; i < len(tokens); i = i + 1 {
		token := tokens[i]
		last := i
		// Generate list of vars
		if token.Type == VAR {
			def = fmt.Sprintf("%s", token)
			val := ""
			nested := false
			for {
				i++
				token = tokens[i]
				switch token.Type {
				case LPAREN:
					nested = true
				case RPAREN:
					nested = false
				case CMD:
					cmd = fmt.Sprintf("%s", token)
					val += cmd
				case FILE:
					i = p.File(cmd, last, i)
					def = ""
					cmd = ""
				case SUB:
					// Can this ever happen, do we care?
					fmt.Println("SUB")
					// fmt.Println(p.Input[token.Pos-100 : token.Pos+100])
					// fmt.Println(token)
				default:
					//fmt.Printf("Default: %s\n", token)
					val += fmt.Sprintf("%s", token)
				}

				if !nested && tokens[i].Type != CMD {
					break
				}
			}
			if def != "" {
				p.Vars[def] = val
			}
			//Replace subsitution tokens
		} else if token.Type == SUB {
			if cmd == "sprite" {
				//Capture sprite
				sprite := p.Sprites[fmt.Sprintf("%s", token)]
				//Capture filename
				i++
				name := fmt.Sprintf("%s", tokens[i])
				repl := sprite.CSS(name)
				p.Mark(tokens[i-3].Pos, tokens[i+2].Pos, repl)
				tokens = append(tokens[:i-3], tokens[i:]...)
				i = i - 3
				def = ""
				cmd = ""
			} else if cmd == "sprite-dimensions" {
				sprite := p.Sprites[fmt.Sprintf("%s", token)]
				// Walk forward to file name
				i++
				repl := sprite.Dimensions(tokens[i].String())
				p.Mark(tokens[i-4].Pos, tokens[i+3].Pos, repl)
				tokens = append(tokens[:i-4], tokens[i:]...)
				i = i - 4
				def = ""
				cmd = ""
			} else {
				tokens[i].Value = p.Vars[token.Value]
			}
		} else if token.Type == CMD {
			cmd = fmt.Sprintf("%s", token)
		}
	}
	p.Output = process(p.Input, p.Items, 0)
	p.Replace()
	// DEBUG
	// for _, item := range p.Items {
	// 	fmt.Printf("%s %s\n", item.Type, item)
	// }
	return p.Output
}

// Replace iterates through the list of substrings to
// cut or replace, adjusting for shift of the output
// buffer as a result of these ops.
func (p *Parser) Replace() {
	for _, c := range p.Chop {
		begin := c.Start - p.shift
		end := c.End - p.shift
		// fmt.Println("Cut: ", string(p.Output[begin:end]))
		// Adjust shift for number of bytes deleted and inserted
		p.shift += end - begin
		p.shift -= len(c.Value)
		suf := append(c.Value, p.Output[end:]...)
		p.Output = append(p.Output[:begin], suf...)
	}
}

// Mark segments of the input string for future deletion.
func (p *Parser) Mark(start, end int, val string) {
	// fmt.Println("Mark:", string(p.Input[start:end]), "~")
	p.Chop = append(p.Chop, Replace{start, end, []byte(val)})
}

// Processes file which usually mean cutting some of the input
// text.
func (p *Parser) File(cmd string, start, end int) int {
	first := p.Items[start]
	item := p.Items[end]
	// Find the next newline, failing that find the semicolon
	i := end
	if cmd == "sprite-map" {
		for ; p.Items[i].Type != RPAREN; i++ {
		}
		i = i - 1
		// Verify that the statement ends with semicolon
		interest := p.Items[i+3]
		// Mark this area for deletion, since doing so now would
		// invalidate all subsequent tokens positions.
		p.Mark(first.Pos, interest.Pos, "")
		if cmd == "sprite-map" {
			imgs := ImageList{}
			glob := fmt.Sprintf("%s", item)
			name := fmt.Sprintf("%s", p.Items[start])
			imgs.Decode(p.ImageDir + "/" + glob)
			imgs.Vertical = true
			imgs.Combine()
			p.Sprites[name] = imgs
		}
		//TODO: Generate filename
		//imgs.Export("generated.png")
	}
	return i
}

func process(in string, items []Item, pos int) []byte {

	var out []byte
	l := len(items)

	if pos >= len(in) {
		return []byte("")
	}

	// TODO: There's an error where items[1] has an invalid
	// position.
	if l > 1 && items[1].Pos > items[0].Pos {
		out = append(out, in[items[0].Pos:items[1].Pos]...)
		out = append(out, process(in, items[1:], pos)...)
	} else {
		out = append(out, in[items[0].Pos:]...)
	}

	return out
}

// parser recrusively resolves all imports and tokenizes the
// input string
func (p *Parser) parser(pwd, input string) ([]Item, string, error) {

	var (
		status    []Item
		importing bool
		output    []byte
		pos       int
		last      *Item
	)

	lex := New(func(lex *Lexer) StateFn {
		return lex.Action()
	}, input)

	for {
		item := lex.Next()
		err := item.Error()
		//fmt.Println(item.Type, item.Value)
		if err != nil {
			return nil, string(output),
				fmt.Errorf("Error: %v (pos %d)", err, item.Pos)
		}
		switch item.Type {
		case ItemEOF:
			output = append(output, input[pos:]...)
			return status, string(output), nil
		case IMPORT:
			output = append(output, input[pos:item.Pos]...)
			last = item
			importing = true
		case INCLUDE, CMT:
			output = append(output, input[pos:item.Pos]...)
			pos = item.Pos
			status = append(status, *item)
		default:
			if importing {

				pwd, contents, err := p.ImportPath(pwd, fmt.Sprintf("%s", *item))

				if err != nil {
					log.Println(err)
				}
				//Eat the semicolon
				item := lex.Next()
				pos = item.Pos + len(item.Value)
				if item.Type != SEMIC {
					panic("@import must be followed by ;")
				}

				moreTokens, moreOutput, err := p.parser(
					pwd,
					contents)
				// Lexer needs to be adjusted for current
				// position of end of @import
				for i, _ := range moreTokens {
					moreTokens[i].Pos += last.Pos
				}

				if err != nil {
					panic(err)
				}
				output = append(output, moreOutput...)
				status = append(status, moreTokens...)
				importing = false
			} else {
				output = append(output, input[pos:item.Pos]...)
				pos = item.Pos
				status = append(status, *item)
			}
		}
	}
}

func (p *Parser) ImportPath(dir, file string) (string, string, error) {
	// fmt.Println("Importing: " + file)
	baseerr := ""
	//Load and retrieve all tokens from imported file
	path, err := filepath.Abs(fmt.Sprintf("%s/%s.scss", dir, file))
	if err != nil {
		panic(err)
	}
	pwd := filepath.Dir(path)
	// Sass put _ in front of imported files
	fpath := pwd + "/_" + filepath.Base(path)
	contents, err := ioutil.ReadFile(fpath)
	if err == nil {
		return pwd, string(contents), nil
	}
	baseerr += fpath + "\n"
	if strings.HasSuffix(err.Error(), "no such file or directory") {
		// Look through the import path for the file
		for _, lib := range p.Includes {
			path, err := filepath.Abs(lib + "/" + file)
			if err != nil {
				panic(err)
			}
			pwd := filepath.Dir(path)
			fpath = pwd + "/_" + filepath.Base(path) + ".scss"
			contents, err := ioutil.ReadFile(fpath)
			baseerr += fpath + "\n"
			if err == nil {
				return pwd, string(contents), nil
			}
		}
	}

	return pwd, string(contents), errors.New("Could not import: " +
		file + "\nTried:\n" + baseerr)
}
<|endoftext|>


<commit_before>package main
import (
    "fmt"
)
// func getGrammar() map[string] (map[string] float64) {
//     grammar := map[string] map[string] float64 {}
//     temp := make(map[string] float64)
//     temp["S"] = 1.0
//     grammar["NP VP"] = make(map[string] float64)
//     grammar["NP VP"] = temp
//
//     temp = make(map[string] float64,0)
//     temp["PP"] = 1.0
//     grammar["P NP"] = temp
//
//     temp = make(map[string] float64,0)
//     temp["VP"] = 0.7
//     grammar["V NP"] = temp
//
//     temp = make(map[string] float64,0)
//     temp["VP"] = 0.3
//     grammar["VP PP"] = temp
//
//     return grammar
// }
func removeDuplicates(elements []string) []string {
    // Use map to record duplicates as we find them.
    encountered := map[string]bool{}
    result := []string{}

    for v := range elements {
    	if encountered[elements[v]] == true {
    	    // Do not add duplicate.
    	} else {
    	    // Record this element as an encountered element.
    	    encountered[elements[v]] = true
    	    // Append to result slice.
    	    result = append(result, elements[v])
    	}
    }
    // Return the new slice.
    return result
}

func get_grammar() map[string] (map[string] float64) {
    grammar := map[string] map[string] float64 {}
    temp := make(map[string] float64)
    temp["S"] = 1.0
    grammar["B C"] = make(map[string] float64)
    // grammar["A B"] = make(map[string] float64)
    // grammar["A B"] = temp
    grammar["B C"] = temp

    temp = make(map[string] float64,0)
    temp["A"] = 0.5
    grammar["B A"] = temp

    temp = make(map[string] float64,0)
    temp["A"] = 0.5
    temp["C"] = 0.5
    grammar["a"] = temp

    temp = make(map[string] float64,0)
    temp["B"] = 0.7
    grammar["C C"] = temp
    grammar["b"] = temp

    temp = make(map[string] float64,0)
    temp["C"] = 0.3
    temp["S"] = 1.0
    grammar["A B"] = temp

    return grammar
}

func get_combinations(grammar map[string] map[string] float64,left []string,right []string) []string{
    temp := make([]string,0)
    for _,i := range left {
        for _,j := range right {
            for k := range grammar[i +" "+j] {
                temp = append(temp,k)
            }
        }
    }
    //fmt.Printf("%#v============\n",temp)
    return temp
}

func parser(tokens []string) {
    grammar := get_grammar()
    cyk_grid := make([][][]string,len(tokens))
    for i,_ := range tokens {
        temp := make([]string,0)
        for j:= range grammar[tokens[i]] {
            temp = append(temp,j)
        }
        //fmt.Printf("%v\n",temp)
        cyk_grid[0] = append(cyk_grid[0],temp)
    }

    //fmt.Printf("%#v\n%#v\n",cyk_grid,grammar)
    for i:= 1; i<len(tokens);i++ {
        cyk_grid[i] = make([][]string,0)
        for j:=0;j< len(tokens) - i;j++ {
            //cyk_grid[i] = append(cyk_grid[i],tokens[j])
            val := make([]string,0)
            // fmt.Printf("(%d %d) (%d %d) (%d %d) (%d %d)\n",i-1,j,0,i+j,0,j,i-1,j+1)
            temp := get_combinations(grammar,cyk_grid[i-1][j],cyk_grid[0][i+j])
            for _,k := range temp {
                val = append(val,k)
            }
            // if(len(val) > 0) {
            //     cyk_grid[i] = append(cyk_grid[i],val)
            // }
            // fmt.Printf("%#v\n",val)
            if(i > 1) {
                // fmt.Printf("%#v %#v %#v %#v\n",cyk_grid[i-1][j],cyk_grid[0][i+j],cyk_grid[0][j],cyk_grid[i-1][j+1])
                temp2 := get_combinations(grammar,cyk_grid[0][j],cyk_grid[i-1][j+1])
                //cyk_grid[i] = append(cyk_grid[i],val)
                for _,k := range temp2 {
                    val = append(val,k)
                }
                val = removeDuplicates(val)
                // fmt.Printf("%#v*****************\n",val)
            }
            cyk_grid[i] = append(cyk_grid[i],val)
            // fmt.Printf("(((%d %d),",i-1,j)
            // fmt.Printf("(%d %d)),",0, i + j)
            // fmt.Printf("((%d %d),",0,j)
            // fmt.Printf("(%d %d))),",i-1,j+1)
        }
        //fmt.Printf("\n")
    }
    // for _,i := range cyk_grid {
    //     fmt.Printf("%#v\n",i)
    // }
}
<commit_msg>Removing Comments<commit_after>package main
import (
    "fmt"
)
// func getGrammar() map[string] (map[string] float64) {
//     grammar := map[string] map[string] float64 {}
//     temp := make(map[string] float64)
//     temp["S"] = 1.0
//     grammar["NP VP"] = make(map[string] float64)
//     grammar["NP VP"] = temp
//
//     temp = make(map[string] float64,0)
//     temp["PP"] = 1.0
//     grammar["P NP"] = temp
//
//     temp = make(map[string] float64,0)
//     temp["VP"] = 0.7
//     grammar["V NP"] = temp
//
//     temp = make(map[string] float64,0)
//     temp["VP"] = 0.3
//     grammar["VP PP"] = temp
//
//     return grammar
// }
func removeDuplicates(elements []string) []string {
    encountered := map[string]bool{}
    result := []string{}

    for v := range elements {
    	if encountered[elements[v]] == true {
    	    // Do not add duplicate.
    	} else {
    	    encountered[elements[v]] = true
    	    result = append(result, elements[v])
    	}
    }
    return result
}

func get_grammar() map[string] (map[string] float64) {
    grammar := map[string] map[string] float64 {}
    temp := make(map[string] float64)
    temp["S"] = 1.0
    grammar["B C"] = make(map[string] float64)
    grammar["B C"] = temp

    temp = make(map[string] float64,0)
    temp["A"] = 0.5
    grammar["B A"] = temp

    temp = make(map[string] float64,0)
    temp["A"] = 0.5
    temp["C"] = 0.5
    grammar["a"] = temp

    temp = make(map[string] float64,0)
    temp["B"] = 0.7
    grammar["C C"] = temp
    grammar["b"] = temp

    temp = make(map[string] float64,0)
    temp["C"] = 0.3
    temp["S"] = 1.0
    grammar["A B"] = temp

    return grammar
}

func get_combinations(grammar map[string] map[string] float64,left []string,right []string) []string{
    temp := make([]string,0)
    for _,i := range left {
        for _,j := range right {
            for k := range grammar[i +" "+j] {
                temp = append(temp,k)
            }
        }
    }
    return temp
}

func parser(tokens []string) {
    grammar := get_grammar()
    cyk_grid := make([][][]string,len(tokens))
    for i,_ := range tokens {
        temp := make([]string,0)
        for j:= range grammar[tokens[i]] {
            temp = append(temp,j)
        }
        cyk_grid[0] = append(cyk_grid[0],temp)
    }

    for i:= 1; i<len(tokens);i++ {
        cyk_grid[i] = make([][]string,0)
        for j:=0;j< len(tokens) - i;j++ {
            val := make([]string,0)
            temp := get_combinations(grammar,cyk_grid[i-1][j],cyk_grid[0][i+j])
            for _,k := range temp {
                val = append(val,k)
            }
            if(i > 1) {
                temp2 := get_combinations(grammar,cyk_grid[0][j],cyk_grid[i-1][j+1])
                for _,k := range temp2 {
                    val = append(val,k)
                }
                val = removeDuplicates(val)
            }
            cyk_grid[i] = append(cyk_grid[i],val)
        }
    }
}
<|endoftext|>


<commit_before>// Released under an MIT-style license. See LICENSE.

package main

import "github.com/peterh/liner"

type ReadStringer interface {
	ReadString(delim byte) (line string, err error)
}

type scanner struct {
	deref func(string, uintptr) Cell
	process func(Cell)
	task    *Task

	input ReadStringer
	line  []rune

	state  int
	indent int

	cursor int
	start  int

	previous rune
	token    rune

	finished bool
}

const (
	ssStart = iota
	ssAmpersand
	ssBang
	ssBangGreater
	ssColon
	ssComment
	ssDoubleQuoted
	ssGreater
	ssLess
	ssPipe
	ssSingleQuoted
	ssSymbol
)

func (s *scanner) Lex(lval *yySymType) (token int) {
	var operator = map[string]string{
		"!>":  "redirect-stderr",
		"!>>": "append-stderr",
		"!|":  "pipe-stderr",
		"!|+": "channel-stderr",
		"&":   "spawn",
		"&&":  "and",
		"<":   "redirect-stdin",
		"<(":  "substitute-stdout",
		">":   "redirect-stdout",
		">(":  "substitute-stdin",
		">>":  "append-stdout",
		"|":   "pipe-stdout",
		"|+":  "channel-stdout",
		"||":  "or",
	}

	defer func() {
		exists := false

		switch s.token {
		case BACKGROUND, ORF, ANDF, PIPE, REDIRECT, SUBSTITUTE:
			lval.s, exists = operator[string(s.line[s.start:s.cursor])]
			if exists {
				break
			}
			fallthrough
		default:
			lval.s = string(s.line[s.start:s.cursor])
		}

		s.state = ssStart
		s.previous = s.token
		s.token = 0
	}()

main:
	for s.token == 0 {
		if s.cursor >= len(s.line) {
			if s.finished {
				return 0
			}

			line, error := s.input.ReadString('\n')
			if error == liner.ErrPromptAborted {
				s.start = 0
				s.token = ERROR
				break
			}

			runes := []rune(line)
			last := len(runes) - 2
			if last >= 0 && runes[last] == '\r' {
				runes = append(runes[0:last], '\n')
				last--
			}

			if last >= 0 && runes[last] == '\\' {
				runes = runes[0:last]
			}

			if error != nil {
				runes = append(runes, '\n')
				s.finished = true
			}

			if s.start < s.cursor-1 {
				s.line = append(s.line[s.start:s.cursor], runes...)
				s.cursor -= s.start
			} else {
				s.cursor = 0
				s.line = runes
			}
			s.start = 0
			s.token = 0
		}

		switch s.state {
		case ssStart:
			s.start = s.cursor

			switch s.line[s.cursor] {
			default:
				s.state = ssSymbol
				continue main
			case '\n', '%', '(', ')', ';', '@', '^', '`', '{', '}':
				s.token = s.line[s.cursor]
			case '&':
				s.state = ssAmpersand
			case '<':
				s.state = ssLess
			case '|':
				s.state = ssPipe
			case '\t', ' ':
				s.state = ssStart
			case '!':
				s.state = ssBang
			case '"':
				s.state = ssDoubleQuoted
			case '#':
				s.state = ssComment
			case '\'':
				s.state = ssSingleQuoted
			case ':':
				s.state = ssColon
			case '>':
				s.state = ssGreater
			}

		case ssAmpersand:
			switch s.line[s.cursor] {
			case '&':
				s.token = ANDF
			default:
				s.token = BACKGROUND
				continue main
			}

		case ssBang:
			switch s.line[s.cursor] {
			case '>':
				s.state = ssBangGreater
			case '|':
				s.state = ssPipe
			default:
				s.state = ssSymbol
				continue main
			}

		case ssBangGreater:
			s.token = REDIRECT
			if s.line[s.cursor] != '>' {
				continue main
			}

		case ssColon:
			switch s.line[s.cursor] {
			case ':':
				s.token = CONS
			default:
				s.token = ':'
				continue main
			}

		case ssComment:
			for s.line[s.cursor] != '\n' ||
				s.line[s.cursor-1] == '\\' {
				s.cursor++

				if s.cursor >= len(s.line) {
					continue main
				}
			}
			s.cursor--
			s.state = ssStart

		case ssDoubleQuoted:
			for s.cursor < len(s.line) && s.line[s.cursor] != '"' ||
				s.cursor > 0 && s.line[s.cursor-1] == '\\' {
				s.cursor++
			}
			if s.cursor >= len(s.line) {
				if s.line[s.cursor-1] == '\n' {
					s.line = append(s.line[0:s.cursor-1], []rune("\\n")...)
				}
				continue main
			}
			s.token = DOUBLE_QUOTED

		case ssGreater:
			s.token = REDIRECT
			if s.line[s.cursor] == '(' {
				s.token = SUBSTITUTE
			} else if s.line[s.cursor] != '>' {
				continue main
			}

		case ssLess:
			s.token = REDIRECT
			if s.line[s.cursor] == '(' {
				s.token = SUBSTITUTE
			} else {
				continue main
			}

		case ssPipe:
			switch s.line[s.cursor] {
			case '+':
				s.token = PIPE
			case '|':
				s.token = ORF
			default:
				s.token = PIPE
				continue main
			}

		case ssSingleQuoted:
			for s.cursor < len(s.line) && s.line[s.cursor] != '\'' {
				s.cursor++
			}
			if s.cursor >= len(s.line) {
				if s.line[s.cursor-1] == '\n' {
					s.line = append(s.line[0:s.cursor-1], []rune("\\n")...)
				}
				continue main
			}
			s.token = SINGLE_QUOTED

		case ssSymbol:
			switch s.line[s.cursor] {
			case '\n', '%', '&', '\'', '(', ')', ';',
				'<', '@', '^', '`', '{', '|', '}',
				'\t', ' ', '"', '#', ':', '>':
				s.token = SYMBOL
				continue main
			}

		}
		s.cursor++

		if s.token == '\n' {
			switch s.previous {
			case ORF, ANDF, PIPE, REDIRECT:
				s.token = 0
			}
		}
	}

	return int(s.token)
}

func (s *scanner) Error(msg string) {
	println(msg)
}

func Parse(t *Task, r ReadStringer, d func(string, uintptr) Cell, p func(Cell)) {
	s := new(scanner)

	s.deref = d
	s.process = p
	s.task = t

	s.input = r
	s.line = []rune("")

	s.state = ssStart
	s.indent = 0

	s.cursor = 0
	s.start = 0

	s.previous = 0
	s.token = 0

	yyParse(s)
}

//go:generate go tool yacc -o grammar.go grammar.y
//go:generate sed -i.save -f scripts/grammar.sed grammar.go
//go:generate go fmt grammar.go
//go:generate rm -f y.output grammar.go.save
<commit_msg>Formatting.<commit_after>// Released under an MIT-style license. See LICENSE.

package main

import "github.com/peterh/liner"

type ReadStringer interface {
	ReadString(delim byte) (line string, err error)
}

type scanner struct {
	deref   func(string, uintptr) Cell
	process func(Cell)
	task    *Task

	input ReadStringer
	line  []rune

	state  int
	indent int

	cursor int
	start  int

	previous rune
	token    rune

	finished bool
}

const (
	ssStart = iota
	ssAmpersand
	ssBang
	ssBangGreater
	ssColon
	ssComment
	ssDoubleQuoted
	ssGreater
	ssLess
	ssPipe
	ssSingleQuoted
	ssSymbol
)

func (s *scanner) Lex(lval *yySymType) (token int) {
	var operator = map[string]string{
		"!>":  "redirect-stderr",
		"!>>": "append-stderr",
		"!|":  "pipe-stderr",
		"!|+": "channel-stderr",
		"&":   "spawn",
		"&&":  "and",
		"<":   "redirect-stdin",
		"<(":  "substitute-stdout",
		">":   "redirect-stdout",
		">(":  "substitute-stdin",
		">>":  "append-stdout",
		"|":   "pipe-stdout",
		"|+":  "channel-stdout",
		"||":  "or",
	}

	defer func() {
		exists := false

		switch s.token {
		case BACKGROUND, ORF, ANDF, PIPE, REDIRECT, SUBSTITUTE:
			lval.s, exists = operator[string(s.line[s.start:s.cursor])]
			if exists {
				break
			}
			fallthrough
		default:
			lval.s = string(s.line[s.start:s.cursor])
		}

		s.state = ssStart
		s.previous = s.token
		s.token = 0
	}()

main:
	for s.token == 0 {
		if s.cursor >= len(s.line) {
			if s.finished {
				return 0
			}

			line, error := s.input.ReadString('\n')
			if error == liner.ErrPromptAborted {
				s.start = 0
				s.token = ERROR
				break
			}

			runes := []rune(line)
			last := len(runes) - 2
			if last >= 0 && runes[last] == '\r' {
				runes = append(runes[0:last], '\n')
				last--
			}

			if last >= 0 && runes[last] == '\\' {
				runes = runes[0:last]
			}

			if error != nil {
				runes = append(runes, '\n')
				s.finished = true
			}

			if s.start < s.cursor-1 {
				s.line = append(s.line[s.start:s.cursor], runes...)
				s.cursor -= s.start
			} else {
				s.cursor = 0
				s.line = runes
			}
			s.start = 0
			s.token = 0
		}

		switch s.state {
		case ssStart:
			s.start = s.cursor

			switch s.line[s.cursor] {
			default:
				s.state = ssSymbol
				continue main
			case '\n', '%', '(', ')', ';', '@', '^', '`', '{', '}':
				s.token = s.line[s.cursor]
			case '&':
				s.state = ssAmpersand
			case '<':
				s.state = ssLess
			case '|':
				s.state = ssPipe
			case '\t', ' ':
				s.state = ssStart
			case '!':
				s.state = ssBang
			case '"':
				s.state = ssDoubleQuoted
			case '#':
				s.state = ssComment
			case '\'':
				s.state = ssSingleQuoted
			case ':':
				s.state = ssColon
			case '>':
				s.state = ssGreater
			}

		case ssAmpersand:
			switch s.line[s.cursor] {
			case '&':
				s.token = ANDF
			default:
				s.token = BACKGROUND
				continue main
			}

		case ssBang:
			switch s.line[s.cursor] {
			case '>':
				s.state = ssBangGreater
			case '|':
				s.state = ssPipe
			default:
				s.state = ssSymbol
				continue main
			}

		case ssBangGreater:
			s.token = REDIRECT
			if s.line[s.cursor] != '>' {
				continue main
			}

		case ssColon:
			switch s.line[s.cursor] {
			case ':':
				s.token = CONS
			default:
				s.token = ':'
				continue main
			}

		case ssComment:
			for s.line[s.cursor] != '\n' ||
				s.line[s.cursor-1] == '\\' {
				s.cursor++

				if s.cursor >= len(s.line) {
					continue main
				}
			}
			s.cursor--
			s.state = ssStart

		case ssDoubleQuoted:
			for s.cursor < len(s.line) && s.line[s.cursor] != '"' ||
				s.cursor > 0 && s.line[s.cursor-1] == '\\' {
				s.cursor++
			}
			if s.cursor >= len(s.line) {
				if s.line[s.cursor-1] == '\n' {
					s.line = append(s.line[0:s.cursor-1], []rune("\\n")...)
				}
				continue main
			}
			s.token = DOUBLE_QUOTED

		case ssGreater:
			s.token = REDIRECT
			if s.line[s.cursor] == '(' {
				s.token = SUBSTITUTE
			} else if s.line[s.cursor] != '>' {
				continue main
			}

		case ssLess:
			s.token = REDIRECT
			if s.line[s.cursor] == '(' {
				s.token = SUBSTITUTE
			} else {
				continue main
			}

		case ssPipe:
			switch s.line[s.cursor] {
			case '+':
				s.token = PIPE
			case '|':
				s.token = ORF
			default:
				s.token = PIPE
				continue main
			}

		case ssSingleQuoted:
			for s.cursor < len(s.line) && s.line[s.cursor] != '\'' {
				s.cursor++
			}
			if s.cursor >= len(s.line) {
				if s.line[s.cursor-1] == '\n' {
					s.line = append(s.line[0:s.cursor-1], []rune("\\n")...)
				}
				continue main
			}
			s.token = SINGLE_QUOTED

		case ssSymbol:
			switch s.line[s.cursor] {
			case '\n', '%', '&', '\'', '(', ')', ';',
				'<', '@', '^', '`', '{', '|', '}',
				'\t', ' ', '"', '#', ':', '>':
				s.token = SYMBOL
				continue main
			}

		}
		s.cursor++

		if s.token == '\n' {
			switch s.previous {
			case ORF, ANDF, PIPE, REDIRECT:
				s.token = 0
			}
		}
	}

	return int(s.token)
}

func (s *scanner) Error(msg string) {
	println(msg)
}

func Parse(t *Task, r ReadStringer, d func(string, uintptr) Cell, p func(Cell)) {
	s := new(scanner)

	s.deref = d
	s.process = p
	s.task = t

	s.input = r
	s.line = []rune("")

	s.state = ssStart
	s.indent = 0

	s.cursor = 0
	s.start = 0

	s.previous = 0
	s.token = 0

	yyParse(s)
}

//go:generate go tool yacc -o grammar.go grammar.y
//go:generate sed -i.save -f scripts/grammar.sed grammar.go
//go:generate go fmt grammar.go
//go:generate rm -f y.output grammar.go.save
<|endoftext|>


<commit_before>package parser

import (
	"fmt"
	"io"
	"regexp"
	"strings"
)

type Match interface{}

type MatchString string

type MatchTree []Match

type TaggedMatch struct {
	Match Match
	Tag string
}

type Grammer struct {
	parse func(rs io.ReadSeeker) (Match, error)
}

func (g *Grammer) Set(ng *Grammer) {
	g.parse = ng.parse
}

func (g *Grammer) Node(node func(Match) (Match, error)) {
	oldp := g.parse
	g.parse = func(rs io.ReadSeeker) (Match, error) {
		m, err := oldp(rs)
		if err == nil {
			m, err = node(m)
			if err == nil {
				return m, nil
			}
		}
		return nil, err
	}
}

func (g *Grammer) Parse(rs io.ReadSeeker) (Match, error) {

}

func Set(set string) *Grammer {
	regset, _ := regexp.Compile(fmt.Sprintf("[%s]", set))
	return &Grammer{parse: func(rs io.ReadSeeker) (Match, error) {
		pos, _ := rs.Seek(0, 1)
		b := make([]byte, 1)
		c, _ := rs.Read(b)
		if c < 1 {
			rs.Seek(pos, 0)
			return nil, fmt.Errorf("Unexpected EOF")
		}
		if regset.Match(b) {
			m := MatchString(b)
			return m, nil
		}
		rs.Seek(pos, 0)
		return nil, fmt.Errorf("Expected %s, got %s", set, string(b))
	}}
}

func Lit(text string) *Grammer {
	return &Grammer{parse: func(rs io.ReadSeeker) (Match, error) {
		pos, _ := rs.Seek(0, 1)
		b := make([]byte, len(text))
		c, _ := rs.Read(b)
		if c < len(text) {
			rs.Seek(pos, 0)
			return nil, fmt.Errorf("Unexpected EOF")
		}
		if string(b) == text {
			m := MatchString(text)
			return m, nil
		}
		rs.Seek(pos, 0)
		return nil, fmt.Errorf("Expected %s, got %s", text, string(b))
	}}
}

func And(ps ...*Grammer) *Grammer {
	return &Grammer{parse: func(rs io.ReadSeeker) (Match, error) {
		pos, _ := rs.Seek(0, 1)
		matches := []Match{}
		for _, p := range ps {
			m, err := p.parse(rs)
			if err != nil {
				rs.Seek(pos, 0)
				return nil, err
			}
			matches = append(matches, m)
		}
		mt := MatchTree(matches)
		return mt, nil
	}}
}

func Or(ps ...*Grammer) *Grammer {
	return &Grammer{parse: func(rs io.ReadSeeker) (Match, error) {
		pos, _ := rs.Seek(0, 1)
		errs := []error{}
		for _, p := range ps {
			m, err := p.parse(rs)
			if err == nil {
				return m, nil
			} else {
				errs = append(errs, err)
			}
			rs.Seek(pos, 0)
		}
		return nil, fmt.Errorf("Or error, expected: (%v)", errs)
	}}
}

func Mult(n, m int, p *Grammer) *Grammer {
	if m == 0 {
		m = int(^uint(0) >> 1)
	}
	return &Grammer{parse: func(rs io.ReadSeeker) (Match, error) {
		pos, _ := rs.Seek(0, 1)
		ms := make(MatchTree, 0)
		for i := 0; i < m; i++ {
			match, err1 := p.parse(rs)
			if err1 != nil {
				if i < n {
					rs.Seek(pos, 0)
					return nil, fmt.Errorf("Error: not enough")
				}
				return ms, nil
			}
			ms = append(ms, match)
		}
		return ms, nil
	}}
}

func Tag(tag string, g *Grammer) *Grammer {
	return &Grammer{parse: func(rs io.ReadSeeker) (Match, error) {
		m, err := g.parse(rs)
		if err != nil {
			return nil, err
		}
		tm := TaggedMatch{
			Match: m,
			Tag: tag,
		}
		return tm, nil
	}}
}

func GetTag(m Match, tag string) *TaggedMatch {
	switch m := m.(type) {
	case MatchTree:
		for _, mi := range m {
			tm := GetTag(mi, tag)
			if tm != nil {
				return tm
			}
		}
		return nil
	case MatchString:
		return nil
	case TaggedMatch:
		if tag == m.Tag {
			return &m
		}
		return GetTag(m.Match, tag)
	}
	return nil
}

func String(m Match) string {
	switch m := m.(type) {
	case MatchTree:
		ss := make([]string, len(m))
		for i, mi := range m {
			ss[i] = String(mi)
		}
		return strings.Join(ss, "")
	case MatchString:
		return string(m)
	}
	return ""
}<commit_msg>Add .Parse and .ParseString<commit_after>package parser

import (
	"fmt"
	"io"
	"regexp"
	"strings"
)

type Match interface{}

type MatchString string

type MatchTree []Match

type TaggedMatch struct {
	Match Match
	Tag string
}

type Grammer struct {
	parse func(rs io.ReadSeeker) (Match, error)
}

func (g *Grammer) Set(ng *Grammer) {
	g.parse = ng.parse
}

func (g *Grammer) Node(node func(Match) (Match, error)) {
	oldp := g.parse
	g.parse = func(rs io.ReadSeeker) (Match, error) {
		m, err := oldp(rs)
		if err == nil {
			m, err = node(m)
			if err == nil {
				return m, nil
			}
		}
		return nil, err
	}
}

func (g *Grammer) Parse(rs io.ReadSeeker) (Match, error) {
	m, err := g.parse(rs)
	if err != nil {
		return nil, err
	}
	return m, nil
}

func (g *Grammer) ParseString(s string) (Match, error) {
	m, err := g.parse(strings.NewReader(s)
	if err != nil {
		return nil, err
	}
	return m, nil
}

func Set(set string) *Grammer {
	regset, _ := regexp.Compile(fmt.Sprintf("[%s]", set))
	return &Grammer{parse: func(rs io.ReadSeeker) (Match, error) {
		pos, _ := rs.Seek(0, 1)
		b := make([]byte, 1)
		c, _ := rs.Read(b)
		if c < 1 {
			rs.Seek(pos, 0)
			return nil, fmt.Errorf("Unexpected EOF")
		}
		if regset.Match(b) {
			m := MatchString(b)
			return m, nil
		}
		rs.Seek(pos, 0)
		return nil, fmt.Errorf("Expected %s, got %s", set, string(b))
	}}
}

func Lit(text string) *Grammer {
	return &Grammer{parse: func(rs io.ReadSeeker) (Match, error) {
		pos, _ := rs.Seek(0, 1)
		b := make([]byte, len(text))
		c, _ := rs.Read(b)
		if c < len(text) {
			rs.Seek(pos, 0)
			return nil, fmt.Errorf("Unexpected EOF")
		}
		if string(b) == text {
			m := MatchString(text)
			return m, nil
		}
		rs.Seek(pos, 0)
		return nil, fmt.Errorf("Expected %s, got %s", text, string(b))
	}}
}

func And(ps ...*Grammer) *Grammer {
	return &Grammer{parse: func(rs io.ReadSeeker) (Match, error) {
		pos, _ := rs.Seek(0, 1)
		matches := []Match{}
		for _, p := range ps {
			m, err := p.parse(rs)
			if err != nil {
				rs.Seek(pos, 0)
				return nil, err
			}
			matches = append(matches, m)
		}
		mt := MatchTree(matches)
		return mt, nil
	}}
}

func Or(ps ...*Grammer) *Grammer {
	return &Grammer{parse: func(rs io.ReadSeeker) (Match, error) {
		pos, _ := rs.Seek(0, 1)
		errs := []error{}
		for _, p := range ps {
			m, err := p.parse(rs)
			if err == nil {
				return m, nil
			} else {
				errs = append(errs, err)
			}
			rs.Seek(pos, 0)
		}
		return nil, fmt.Errorf("Or error, expected: (%v)", errs)
	}}
}

func Mult(n, m int, p *Grammer) *Grammer {
	if m == 0 {
		m = int(^uint(0) >> 1)
	}
	return &Grammer{parse: func(rs io.ReadSeeker) (Match, error) {
		pos, _ := rs.Seek(0, 1)
		ms := make(MatchTree, 0)
		for i := 0; i < m; i++ {
			match, err1 := p.parse(rs)
			if err1 != nil {
				if i < n {
					rs.Seek(pos, 0)
					return nil, fmt.Errorf("Error: not enough")
				}
				return ms, nil
			}
			ms = append(ms, match)
		}
		return ms, nil
	}}
}

func Tag(tag string, g *Grammer) *Grammer {
	return &Grammer{parse: func(rs io.ReadSeeker) (Match, error) {
		m, err := g.parse(rs)
		if err != nil {
			return nil, err
		}
		tm := TaggedMatch{
			Match: m,
			Tag: tag,
		}
		return tm, nil
	}}
}

func GetTag(m Match, tag string) *TaggedMatch {
	switch m := m.(type) {
	case MatchTree:
		for _, mi := range m {
			tm := GetTag(mi, tag)
			if tm != nil {
				return tm
			}
		}
		return nil
	case MatchString:
		return nil
	case TaggedMatch:
		if tag == m.Tag {
			return &m
		}
		return GetTag(m.Match, tag)
	}
	return nil
}

func String(m Match) string {
	switch m := m.(type) {
	case MatchTree:
		ss := make([]string, len(m))
		for i, mi := range m {
			ss[i] = String(mi)
		}
		return strings.Join(ss, "")
	case MatchString:
		return string(m)
	}
	return ""
}<|endoftext|>