<commit_before>package mgostasher

import (
  "context"
  "time"
  "google.golang.org/api/blogger/v3"
  "github.com/dghubble/go-twitter/twitter"
  "gopkg.in/mgo.v2"
  "gopkg.in/mgo.v2/bson"
)

const (
  DefaultBlogPageCollection = "blog_page"
  DefaultBlogPostCollection = "blog_post"
  DefaultBlogCollection = "blog"
  DefaultTweetCollection = "tweet"
  DefaultTwitterUserCollection = "twitterer"
)

type BlogStasher struct {
  BlogPageCollection *mgo.Collection
  BlogPostCollection *mgo.Collection
  BlogCollection *mgo.Collection
}

type TwitterStasher struct {
  TweetCollection *mgo.Collection
  TwitterUserCollection *mgo.Collection
}

type BlogPost struct {
  Id *bson.ObjectId `bson:"_id,omitempty" json:",omitempty"`
  BlogPost *blogger.Post `bson:"blogPost"`
  Updated *time.Time `bson:"updated"`
}

type BlogPage struct {
  Id *bson.ObjectId `bson:"_id,omitempty" json:",omitempty"`
  BlogPage *blogger.Page `bson:"blogPage"`
  Updated *time.Time `bson:"updated"`
}

type Blog struct {
  Id *bson.ObjectId `bson:"_id,omitempty" json:",omitempty"`
  Blog *blogger.Blog `bson:"blog,omitempty"`
  Updated *time.Time `bson:"updated,omitempty"`
  PostListEtag *string `bson:"postListEtag,omitempty"`
  PostListUpdated *time.Time `bson:"postListUpdated,omitempty"`
  PageListEtag *string `bson:"pageListEtag,omitempty"`
  PageListUpdated *time.Time `bson:"pageListUpdated,omitempty"`
}

type Tweet struct {
  Id *bson.ObjectId `bson:"_id,omitempty" json:",omitempty"`
  Tweet *twitter.Tweet `bson:"tweet"`
  OEmbed *twitter.OEmbedTweet `bson:"oEmbed"`
}

type TwitterUser struct {
  Id *bson.ObjectId `bson:"_id,omitempty" json:",omitempty"`
  TwitterUser *twitter.User `bson:"twitterUser"`
}

func DefaultBlogStasher(db *mgo.Database) (m *BlogStasher) {
  m = new(BlogStasher)
  m.BlogPageCollection = db.C(DefaultBlogPageCollection)
  m.BlogPostCollection = db.C(DefaultBlogPostCollection)
  m.BlogCollection = db.C(DefaultBlogCollection)
  return
}

func DefaultTwitterStasher(db *mgo.Database) (m *TwitterStasher) {
  m = new(TwitterStasher)
  m.TweetCollection = db.C(DefaultTweetCollection)
  m.TwitterUserCollection = db.C(DefaultTwitterUserCollection)
  return
}

func(m *BlogStasher) GetPageListEtag(ctx context.Context, blogId string) (etag *string) {
  dbBlog := new(Blog)
  err := m.BlogCollection.Find(bson.M{"blog.id": blogId}).Select(bson.M{"pageListEtag": 1}).One(dbBlog)
  if err != nil {
    if err == mgo.ErrNotFound {
      return nil
    }
    panic(err)
  }
  return dbBlog.PageListEtag
}
func(m *BlogStasher) HasPage(ctx context.Context, pageId string, etag string) bool {
  n, err := m.BlogPageCollection.Find(bson.M{"blogPage.id": pageId, "blogPage.etag": etag}).Count()
  if err != nil { panic(err) }
  if n > 0 { return true }
  return false
}
func WrapBlogPage(page *blogger.Page) *BlogPage {
  dbPage := &BlogPage{
    BlogPage: page,
  }
  pageUpdated, err := time.Parse(time.RFC3339, page.Updated)
  if err != nil { panic(err) }
  dbPage.Updated = &pageUpdated
  return dbPage
}
func(m *BlogStasher) StashPage(ctx context.Context, page *blogger.Page) {
  dbPage := WrapBlogPage(page)
  _, err := m.BlogPageCollection.Upsert(bson.M{"blogPage.id": page.Id}, &dbPage);
  if err != nil { panic(err) }
}
func(m *BlogStasher) StashPageEtags(ctx context.Context, blogId string, listEtag string, pageEtags map[string]string, newUpdated *time.Time) {
  iter := m.BlogPageCollection.
      Find(nil).
      Select(bson.M{"blogPage.id": 1, "blogPage.etag": 1}).
      Iter()
  defer iter.Close()
  dbPage := new(BlogPage)
  for iter.Next(dbPage) {
    if pageEtags[dbPage.BlogPage.Id] != dbPage.BlogPage.Etag {
      if err := m.BlogPageCollection.RemoveId(dbPage.Id); err != nil {
        panic(err)
      }
    }
  }
  if err := iter.Err(); err != nil {
    panic(err)
  }

  _, err := m.BlogCollection.Upsert(bson.M{"blog.id": blogId}, bson.M{"$set": &Blog{PageListEtag: &listEtag, PageListUpdated: newUpdated}})
  if (err != nil) { panic(err) }
}

func(m *BlogStasher) GetPostListEtag(ctx context.Context, blogId string) (etag *string) {
  dbBlog := new(Blog)
  err := m.BlogCollection.Find(bson.M{"blog.id": blogId}).Select(bson.M{"postListEtag": 1}).One(dbBlog)
  if err != nil {
    if err == mgo.ErrNotFound {
      return nil
    }
    panic(err)
  }
  return dbBlog.PostListEtag
}
func(m *BlogStasher) HasPost(ctx context.Context, postId string, etag string) bool {
  n, err := m.BlogPostCollection.Find(bson.M{"blogPost.id": postId, "blogPost.etag": etag}).Count()
  if err != nil { panic(err) }
  if n > 0 { return true }
  return false
}

func WrapBlogPost(post *blogger.Post) *BlogPost {
  dbPost := &BlogPost{
    BlogPost: post,
  }
  postUpdated, err := time.Parse(time.RFC3339, post.Updated)
  if err != nil { panic(err) }
  dbPost.Updated = &postUpdated
  return dbPost
}
func(m *BlogStasher) StashPost(ctx context.Context, post *blogger.Post) {
  dbPost := WrapBlogPost(post)
  _, err := m.BlogPostCollection.Upsert(bson.M{"blogPost.id": post.Id}, dbPost);
  if err != nil { panic(err) }
}
func(m *BlogStasher) StashPostEtags(ctx context.Context, blogId string, listEtag string, postEtags map[string]string, newUpdated *time.Time) {
  iter := m.BlogPostCollection.
      Find(nil).
      Select(bson.M{"blogPost.id": 1, "blogPost.etag": 1}).
      Iter()
  defer iter.Close()
  dbPost := new(BlogPost)
  for iter.Next(dbPost) {
    if postEtags[dbPost.BlogPost.Id] != dbPost.BlogPost.Etag {
      if err := m.BlogPostCollection.RemoveId(dbPost.Id); err != nil {
        panic(err)
      }
    }
  }
  if err := iter.Err(); err != nil {
    panic(err)
  }

  _, err := m.BlogCollection.Upsert(bson.M{"blog.id": blogId}, bson.M{"$set": &Blog{PostListEtag: &listEtag, PostListUpdated: newUpdated}})
  if (err != nil) { panic(err) }
}

func(m *BlogStasher) StashBlog(ctx context.Context, blog *blogger.Blog) {
  dbBlog := Blog{
    Blog: blog,
  }
  updated, err := time.Parse(time.RFC3339, blog.Updated)
  if err != nil { panic(err) }
  dbBlog.Updated = &updated
  _, err = m.BlogCollection.Upsert(bson.M{"blog.id": blog.Id}, bson.M{"$set": &dbBlog})
  if (err != nil) { panic(err) }
}

func(m *TwitterStasher) GetLastTweetId(ctx context.Context, userId int64) int64 {
  tweet := new(Tweet)
  err := m.TweetCollection.Find(bson.M{"tweet.user.id": userId}).Select(bson.M{"tweet.id": 1}).One(tweet)
  if err != nil {
    if err == mgo.ErrNotFound {
      return 0
    }
    panic(err)
  }
  return tweet.Tweet.ID
}

func(m *TwitterStasher) StashUser(ctx context.Context, user *twitter.User) {
  dbUser := &TwitterUser{TwitterUser: user}
  _, err := m.TwitterUserCollection.Upsert(bson.M{"user.id": user.ID}, &dbUser);
  if err != nil { panic(err) }
}

func(m *TwitterStasher) StashTweet(ctx context.Context, tweet *twitter.Tweet, oembed *twitter.OEmbedTweet) {
  dbTweet := &Tweet{Tweet: tweet, OEmbed: oembed}
  _, err := m.TweetCollection.Upsert(bson.M{"tweet.id": tweet.ID}, &dbTweet);
  if err != nil { panic(err) }
}
<commit_msg>More json formatting<commit_after>package mgostasher

import (
  "context"
  "time"
  "google.golang.org/api/blogger/v3"
  "github.com/dghubble/go-twitter/twitter"
  "gopkg.in/mgo.v2"
  "gopkg.in/mgo.v2/bson"
)

const (
  DefaultBlogPageCollection = "blog_page"
  DefaultBlogPostCollection = "blog_post"
  DefaultBlogCollection = "blog"
  DefaultTweetCollection = "tweet"
  DefaultTwitterUserCollection = "twitterer"
)

type BlogStasher struct {
  BlogPageCollection *mgo.Collection
  BlogPostCollection *mgo.Collection
  BlogCollection *mgo.Collection
}

type TwitterStasher struct {
  TweetCollection *mgo.Collection
  TwitterUserCollection *mgo.Collection
}

type BlogPost struct {
  Id *bson.ObjectId `bson:"_id,omitempty" json:"id,omitempty"`
  BlogPost *blogger.Post `bson:"blogPost" json:"blogPost"`
  Updated *time.Time `bson:"updated" json:"updated"`
}

type BlogPage struct {
  Id *bson.ObjectId `bson:"_id,omitempty" json:"id,omitempty"`
  BlogPage *blogger.Page `bson:"blogPage" json:"blogPage"`
  Updated *time.Time `bson:"updated" json:"updated"`
}

type Blog struct {
  Id *bson.ObjectId `bson:"_id,omitempty" json:"id,omitempty"`
  Blog *blogger.Blog `bson:"blog,omitempty" json:"blog"`
  Updated *time.Time `bson:"updated,omitempty" json:"updated"`
  PostListEtag *string `bson:"postListEtag,omitempty"`
  PostListUpdated *time.Time `bson:"postListUpdated,omitempty"`
  PageListEtag *string `bson:"pageListEtag,omitempty"`
  PageListUpdated *time.Time `bson:"pageListUpdated,omitempty"`
}

type Tweet struct {
  Id *bson.ObjectId `bson:"_id,omitempty" json:",omitempty"`
  Tweet *twitter.Tweet `bson:"tweet"`
  OEmbed *twitter.OEmbedTweet `bson:"oEmbed"`
}

type TwitterUser struct {
  Id *bson.ObjectId `bson:"_id,omitempty" json:",omitempty"`
  TwitterUser *twitter.User `bson:"twitterUser"`
}

func DefaultBlogStasher(db *mgo.Database) (m *BlogStasher) {
  m = new(BlogStasher)
  m.BlogPageCollection = db.C(DefaultBlogPageCollection)
  m.BlogPostCollection = db.C(DefaultBlogPostCollection)
  m.BlogCollection = db.C(DefaultBlogCollection)
  return
}

func DefaultTwitterStasher(db *mgo.Database) (m *TwitterStasher) {
  m = new(TwitterStasher)
  m.TweetCollection = db.C(DefaultTweetCollection)
  m.TwitterUserCollection = db.C(DefaultTwitterUserCollection)
  return
}

func(m *BlogStasher) GetPageListEtag(ctx context.Context, blogId string) (etag *string) {
  dbBlog := new(Blog)
  err := m.BlogCollection.Find(bson.M{"blog.id": blogId}).Select(bson.M{"pageListEtag": 1}).One(dbBlog)
  if err != nil {
    if err == mgo.ErrNotFound {
      return nil
    }
    panic(err)
  }
  return dbBlog.PageListEtag
}
func(m *BlogStasher) HasPage(ctx context.Context, pageId string, etag string) bool {
  n, err := m.BlogPageCollection.Find(bson.M{"blogPage.id": pageId, "blogPage.etag": etag}).Count()
  if err != nil { panic(err) }
  if n > 0 { return true }
  return false
}
func WrapBlogPage(page *blogger.Page) *BlogPage {
  dbPage := &BlogPage{
    BlogPage: page,
  }
  pageUpdated, err := time.Parse(time.RFC3339, page.Updated)
  if err != nil { panic(err) }
  dbPage.Updated = &pageUpdated
  return dbPage
}
func(m *BlogStasher) StashPage(ctx context.Context, page *blogger.Page) {
  dbPage := WrapBlogPage(page)
  _, err := m.BlogPageCollection.Upsert(bson.M{"blogPage.id": page.Id}, &dbPage);
  if err != nil { panic(err) }
}
func(m *BlogStasher) StashPageEtags(ctx context.Context, blogId string, listEtag string, pageEtags map[string]string, newUpdated *time.Time) {
  iter := m.BlogPageCollection.
      Find(nil).
      Select(bson.M{"blogPage.id": 1, "blogPage.etag": 1}).
      Iter()
  defer iter.Close()
  dbPage := new(BlogPage)
  for iter.Next(dbPage) {
    if pageEtags[dbPage.BlogPage.Id] != dbPage.BlogPage.Etag {
      if err := m.BlogPageCollection.RemoveId(dbPage.Id); err != nil {
        panic(err)
      }
    }
  }
  if err := iter.Err(); err != nil {
    panic(err)
  }

  _, err := m.BlogCollection.Upsert(bson.M{"blog.id": blogId}, bson.M{"$set": &Blog{PageListEtag: &listEtag, PageListUpdated: newUpdated}})
  if (err != nil) { panic(err) }
}

func(m *BlogStasher) GetPostListEtag(ctx context.Context, blogId string) (etag *string) {
  dbBlog := new(Blog)
  err := m.BlogCollection.Find(bson.M{"blog.id": blogId}).Select(bson.M{"postListEtag": 1}).One(dbBlog)
  if err != nil {
    if err == mgo.ErrNotFound {
      return nil
    }
    panic(err)
  }
  return dbBlog.PostListEtag
}
func(m *BlogStasher) HasPost(ctx context.Context, postId string, etag string) bool {
  n, err := m.BlogPostCollection.Find(bson.M{"blogPost.id": postId, "blogPost.etag": etag}).Count()
  if err != nil { panic(err) }
  if n > 0 { return true }
  return false
}

func WrapBlogPost(post *blogger.Post) *BlogPost {
  dbPost := &BlogPost{
    BlogPost: post,
  }
  postUpdated, err := time.Parse(time.RFC3339, post.Updated)
  if err != nil { panic(err) }
  dbPost.Updated = &postUpdated
  return dbPost
}
func(m *BlogStasher) StashPost(ctx context.Context, post *blogger.Post) {
  dbPost := WrapBlogPost(post)
  _, err := m.BlogPostCollection.Upsert(bson.M{"blogPost.id": post.Id}, dbPost);
  if err != nil { panic(err) }
}
func(m *BlogStasher) StashPostEtags(ctx context.Context, blogId string, listEtag string, postEtags map[string]string, newUpdated *time.Time) {
  iter := m.BlogPostCollection.
      Find(nil).
      Select(bson.M{"blogPost.id": 1, "blogPost.etag": 1}).
      Iter()
  defer iter.Close()
  dbPost := new(BlogPost)
  for iter.Next(dbPost) {
    if postEtags[dbPost.BlogPost.Id] != dbPost.BlogPost.Etag {
      if err := m.BlogPostCollection.RemoveId(dbPost.Id); err != nil {
        panic(err)
      }
    }
  }
  if err := iter.Err(); err != nil {
    panic(err)
  }

  _, err := m.BlogCollection.Upsert(bson.M{"blog.id": blogId}, bson.M{"$set": &Blog{PostListEtag: &listEtag, PostListUpdated: newUpdated}})
  if (err != nil) { panic(err) }
}

func(m *BlogStasher) StashBlog(ctx context.Context, blog *blogger.Blog) {
  dbBlog := Blog{
    Blog: blog,
  }
  updated, err := time.Parse(time.RFC3339, blog.Updated)
  if err != nil { panic(err) }
  dbBlog.Updated = &updated
  _, err = m.BlogCollection.Upsert(bson.M{"blog.id": blog.Id}, bson.M{"$set": &dbBlog})
  if (err != nil) { panic(err) }
}

func(m *TwitterStasher) GetLastTweetId(ctx context.Context, userId int64) int64 {
  tweet := new(Tweet)
  err := m.TweetCollection.Find(bson.M{"tweet.user.id": userId}).Select(bson.M{"tweet.id": 1}).One(tweet)
  if err != nil {
    if err == mgo.ErrNotFound {
      return 0
    }
    panic(err)
  }
  return tweet.Tweet.ID
}

func(m *TwitterStasher) StashUser(ctx context.Context, user *twitter.User) {
  dbUser := &TwitterUser{TwitterUser: user}
  _, err := m.TwitterUserCollection.Upsert(bson.M{"user.id": user.ID}, &dbUser);
  if err != nil { panic(err) }
}

func(m *TwitterStasher) StashTweet(ctx context.Context, tweet *twitter.Tweet, oembed *twitter.OEmbedTweet) {
  dbTweet := &Tweet{Tweet: tweet, OEmbed: oembed}
  _, err := m.TweetCollection.Upsert(bson.M{"tweet.id": tweet.ID}, &dbTweet);
  if err != nil { panic(err) }
}
<|endoftext|>


<commit_before>package async

import (
  "reflect"
)

/*

  Manipulate data in a slice in Waterfall mode.

  Each Routine will be called with the value and index of the current position
  in the slice. When calling the Done function, an error will cause the
  mapping to immediately exit. All other arguments are sent back as the
  replacement for the current value.

  For example, take a look at one of the tests for this function:
    func TestMapInt(t *testing.T) {
      ints := []int{1, 2, 3, 4, 5}

      expects := []int{2, 4, 6, 8, 10}

      mapper := func(done Done, args ...interface{}) {
        Status("Hit int")
        Status("Args: %+v\n", args)
        // We know we want an int. So we can cast this to an integer, multiply by
        // 2, and send it back as the result.
        done(nil, args[0].(int)*2)
      }

      final := func(err error, results ...interface{}) {
        Status("Hit int end")
        Status("Results: %+v\n", results)
        for i := 0; i < len(results); i++ {
          if results[i] != expects[i] {
            t.Errorf("Did not map correctly.")
            break
          }
        }
      }

      Map(ints, mapper, final)
    }

*/
func Map(data interface{}, routine Routine, callbacks ...Done) {
  var (
    routines []Routine
    results  []interface{}
  )

  d := reflect.ValueOf(data)

  for i := 0; i < d.Len(); i++ {
    v := d.Index(i).Interface()
    routines = append(routines, func(id int) Routine {
      return func(done Done, args ...interface{}) {
        done = func(original Done) Done {
          return func(err error, args ...interface{}) {
            results = append(results, args...)
            if id == (d.Len() - 1) {
              original(err, results...)
              return
            }
            original(err, args...)
          }
        }(done)

        routine(done, v, id)
      }
    }(i))
  }

  Waterfall(routines, callbacks...)
}

/*

  Manipulate data in a slice in Parallel mode.

  Each Routine will be called with the value and index of the current position
  in the slice. When calling the Done function, an error will cause the
  mapping to immediately exit. All other arguments are sent back as the
  replacement for the current value.

  For example, take a look at one of the tests for this function:
    func TestMapStringParallel(t *testing.T) {
      str := []string{
        "test",
        "test2",
        "test3",
        "test4",
        "test5",
      }

      expects := []string{
        "test1",
        "test2",
        "test3",
        "test4",
        "test5",
      }

      mapper := func(done Done, args ...interface{}) {
        Status("Hit string")
        Status("Args: %+v\n", args)
        if args[1] == 0 {
          done(nil, "test1")
          return
        }
        done(nil, args[0])
      }

      final := func(err error, results ...interface{}) {
        Status("Hit string end")
        Status("Results: %+v\n", results)
        for i := 0; i < len(results); i++ {
          if results[i] != expects[i] {
            t.Errorf("Did not map correctly.")
            break
          }
        }
      }

      MapParallel(str, mapper, final)
    }

  The output of mapping in Parallel mode cannot be guaranteed to stay in the
  same order, due to the fact that it may take longer to process some things
  in your map routine. If you need the data to stay in the order it is in, use
  Map instead to ensure it stays in order.

*/
func MapParallel(data interface{}, routine Routine, callbacks ...Done) {
  var routines []Routine

  d := reflect.ValueOf(data)

  for i := 0; i < d.Len(); i++ {
    v := d.Index(i).Interface()
    routines = append(routines, func(id int) Routine {
      return func(done Done, args ...interface{}) {
        done = func(original Done) Done {
          return func(err error, args ...interface{}) {
            original(err, args...)
          }
        }(done)

        routine(done, v, id)
      }
    }(i))
  }

  Parallel(routines, callbacks...)
}
<commit_msg>Remove redundancy in MapParallel<commit_after>package async

import (
  "reflect"
)

/*

  Manipulate data in a slice in Waterfall mode.

  Each Routine will be called with the value and index of the current position
  in the slice. When calling the Done function, an error will cause the
  mapping to immediately exit. All other arguments are sent back as the
  replacement for the current value.

  For example, take a look at one of the tests for this function:
    func TestMapInt(t *testing.T) {
      ints := []int{1, 2, 3, 4, 5}

      expects := []int{2, 4, 6, 8, 10}

      mapper := func(done Done, args ...interface{}) {
        Status("Hit int")
        Status("Args: %+v\n", args)
        // We know we want an int. So we can cast this to an integer, multiply by
        // 2, and send it back as the result.
        done(nil, args[0].(int)*2)
      }

      final := func(err error, results ...interface{}) {
        Status("Hit int end")
        Status("Results: %+v\n", results)
        for i := 0; i < len(results); i++ {
          if results[i] != expects[i] {
            t.Errorf("Did not map correctly.")
            break
          }
        }
      }

      Map(ints, mapper, final)
    }

*/
func Map(data interface{}, routine Routine, callbacks ...Done) {
  var (
    routines []Routine
    results  []interface{}
  )

  d := reflect.ValueOf(data)

  for i := 0; i < d.Len(); i++ {
    v := d.Index(i).Interface()
    routines = append(routines, func(id int) Routine {
      return func(done Done, args ...interface{}) {
        done = func(original Done) Done {
          return func(err error, args ...interface{}) {
            results = append(results, args...)
            if id == (d.Len() - 1) {
              original(err, results...)
              return
            }
            original(err, args...)
          }
        }(done)

        routine(done, v, id)
      }
    }(i))
  }

  Waterfall(routines, callbacks...)
}

/*

  Manipulate data in a slice in Parallel mode.

  Each Routine will be called with the value and index of the current position
  in the slice. When calling the Done function, an error will cause the
  mapping to immediately exit. All other arguments are sent back as the
  replacement for the current value.

  For example, take a look at one of the tests for this function:
    func TestMapStringParallel(t *testing.T) {
      str := []string{
        "test",
        "test2",
        "test3",
        "test4",
        "test5",
      }

      expects := []string{
        "test1",
        "test2",
        "test3",
        "test4",
        "test5",
      }

      mapper := func(done Done, args ...interface{}) {
        Status("Hit string")
        Status("Args: %+v\n", args)
        if args[1] == 0 {
          done(nil, "test1")
          return
        }
        done(nil, args[0])
      }

      final := func(err error, results ...interface{}) {
        Status("Hit string end")
        Status("Results: %+v\n", results)
        for i := 0; i < len(results); i++ {
          if results[i] != expects[i] {
            t.Errorf("Did not map correctly.")
            break
          }
        }
      }

      MapParallel(str, mapper, final)
    }

  The output of mapping in Parallel mode cannot be guaranteed to stay in the
  same order, due to the fact that it may take longer to process some things
  in your map routine. If you need the data to stay in the order it is in, use
  Map instead to ensure it stays in order.

*/
func MapParallel(data interface{}, routine Routine, callbacks ...Done) {
  var routines []Routine

  d := reflect.ValueOf(data)

  for i := 0; i < d.Len(); i++ {
    v := d.Index(i).Interface()
    routines = append(routines, func(id int) Routine {
      return func(done Done, args ...interface{}) {
        routine(done, v, id)
      }
    }(i))
  }

  Parallel(routines, callbacks...)
}
<|endoftext|>


<commit_before>/* pi-init2
 *
 * A shim to drop onto a Raspberry Pi to write some files to its root
 * filesystem before giving way to the real /sbin/init.  Its goal is simply
 * to allow you to customise a RPi by dropping files into that FAT32 /boot
 * partition, as opposed to either 1) booting it and manually setting it up, or
 * 2) having to mount the root partition, which Windows & Mac users can't easily
 * do.
 *
 * Cross-compile on Mac/Linux:
 *   GOOS=linux GOARCH=arm go build packages projects.bytemark.co.uk/pi-init2
 *
 * Cross-compile on Windows:
 *   set GOOS=linux
 *   set GOARCH=arm
 *   go build packages projects.bytemark.co.uk/pi-init2
 */

package main

import "os"
import "fmt"
import "golang.org/x/sys/unix"
import "syscall" // for Exec only

func checkFatalAllowed(desc string, err error, allowedErrnos []syscall.Errno) {
	if (err != nil) {
		errno, ok := err.(syscall.Errno)
		if ok {
			for _, b := range allowedErrnos {
				if b == errno {
					return
				}
			}
		}
		fmt.Println("error " + desc + ":" +err.Error())
		unix.Exit(1);
	}
}

func checkFatal(desc string, err error) {
	checkFatalAllowed(desc, err, []syscall.Errno{})
}

func main() {

	exists := []syscall.Errno{syscall.EEXIST};
	noent  := []syscall.Errno{syscall.ENOENT};

	checkFatal("changing directory",
		unix.Chdir("/setup"))
	checkFatal("remount rw",
		unix.Mount("/","/","vfat", syscall.MS_REMOUNT, ""), )
	checkFatalAllowed(
		"making tmp",
		unix.Mkdir("tmp", 0770),
		exists)
	checkFatalAllowed(
		"making new_root", unix.Mkdir("new_root", 0770), exists)
	checkFatal("mounting tmp",
		unix.Mount("", "tmp", "tmpfs", 0, ""))
	checkFatal("create device node",
		unix.Mknod("tmp/mmcblk0p2", 0660 | syscall.S_IFBLK, 179<<8 | 2))
	checkFatal("mounting real root",
		unix.Mount("tmp/mmcblk0p2", "new_root", "ext4", 0, ""))
	checkFatal("pivoting",
		unix.PivotRoot("new_root", "new_root/boot"))
	checkFatal("changing to /boot",
		unix.Chdir("/boot"))
	checkFatal("unmounting tmp",
		unix.Unmount("setup/tmp", 0))
	checkFatalAllowed(
		"remove wpa_supplicant.conf",
		unix.Unlink("/etc/wpa_supplicant/wpa_supplicant.conf"), noent)
	checkFatalAllowed(
		"remove rc.local",
		unix.Unlink("/etc/rc.local"), noent)
	checkFatal("symlink wpa_supplicant.conf",
		unix.Symlink(
			"/boot/setup/wpa_supplicant.conf",
			"/etc/wpa_supplicant/wpa_supplicant.conf"))
	checkFatal("symlink rc.local",
		unix.Symlink("/boot/setup/rc.local", "/etc/rc.local"))

	// use deprecated API because Exec has been removed from rebuild syscall
	// stuff :-O  Hopefully we will get a hook in Raspbian before this becomes
	// useless.
	checkFatal("exec real init",
		syscall.Exec("/sbin/init", os.Args, nil))
}
<commit_msg>Added code to create symlinks into the 'appliance' folder, removed the hardwired code that only used rc.local.<commit_after>/* pi-init2
 *
 * A shim to drop onto a Raspberry Pi to write some files to its root
 * filesystem before giving way to the real /sbin/init.  Its goal is simply
 * to allow you to customise a RPi by dropping files into that FAT32 /boot
 * partition, as opposed to either 1) booting it and manually setting it up, or
 * 2) having to mount the root partition, which Windows & Mac users can't easily
 * do.
 *
 * Cross-compile on Mac/Linux:
 *   GOOS=linux GOARCH=arm go build packages projects.bytemark.co.uk/pi-init2
 *
 * Cross-compile on Windows:
 *   set GOOS=linux
 *   set GOARCH=arm
 *   go build packages projects.bytemark.co.uk/pi-init2
 */

package main

import "os"
import "fmt"
import "path/filepath"
import "golang.org/x/sys/unix"
import "syscall" // for Exec only

func checkFatalAllowed(desc string, err error, allowedErrnos []syscall.Errno) {
	if (err != nil) {
		errno, ok := err.(syscall.Errno)
		if ok {
			for _, b := range allowedErrnos {
				if b == errno {
					return
				}
			}
		}
		fmt.Println("error " + desc + ":" +err.Error())
		unix.Exit(1);
	}
}

func checkFatal(desc string, err error) {
	checkFatalAllowed(desc, err, []syscall.Errno{})
}

func copyAppliance(path string, info os.FileInfo, err error) error {
	info, err = os.Stat(path)
	if err != nil {
		// should only be called with real directories
		return err
	}

	// for now we don't care about permissions

	if (info.IsDir()) {
		if (os.Mkdir("/" + path, os.FileMode(int(0755))) != nil) {
			return err
		}
	} else {
		// remove any existing file in place, ignore error, but let's
		// not use RemoveAll to delete directories, not sure anything
		// useful can come of that
		os.Remove("/" + path)

		if (os.Symlink("/boot/appliance/"+path, "/"+path) != nil) {
			return err
		}

		fmt.Println("Symlinked "+path+" to /boot/appliance")
	}

	return nil
}

func main() {

	exists := []syscall.Errno{syscall.EEXIST};

	checkFatal("changing directory",
		unix.Chdir("/"))
	checkFatal("remount rw",
		unix.Mount("/","/","vfat", syscall.MS_REMOUNT, ""), )
	checkFatalAllowed(
		"making tmp",
		unix.Mkdir("tmp", 0770),
		exists)
	checkFatalAllowed(
		"making new_root", unix.Mkdir("new_root", 0770), exists)
	checkFatal("mounting tmp",
		unix.Mount("", "tmp", "tmpfs", 0, ""))
	checkFatal("create device node",
		unix.Mknod("tmp/mmcblk0p2", 0660 | syscall.S_IFBLK, 179<<8 | 2))
	checkFatal("mounting real root",
		unix.Mount("tmp/mmcblk0p2", "new_root", "ext4", 0, ""))
	checkFatal("pivoting",
		unix.PivotRoot("new_root", "new_root/boot"))
	checkFatal("unmounting /boot/tmp",
		unix.Unmount("/boot/tmp", 0))
	checkFatal("Removing /boot/tmp",
		os.Remove("/boot/new_root"))
	checkFatal("Removing /boot/new_root",
		os.Remove("/boot/tmp"))
	checkFatal("changing into appliance directory",
		unix.Chdir("/boot/appliance"))
	checkFatal("copying appliance to root",
		filepath.Walk(".", copyAppliance))
	/*checkFatalAllowed(
		"remove wpa_supplicant.conf",
		unix.Unlink("/etc/wpa_supplicant/wpa_supplicant.conf"), noent)
	checkFatalAllowed(
		"remove rc.local",
		unix.Unlink("/etc/rc.local"), noent)
	checkFatal("symlink wpa_supplicant.conf",
		unix.Symlink(
			"/boot/setup/wpa_supplicant.conf",
			"/etc/wpa_supplicant/wpa_supplicant.conf"))
	checkFatal("symlink rc.local",
		unix.Symlink("/boot/setup/rc.local", "/etc/rc.local"))*/

	// use deprecated API because Exec has been removed from rebuild syscall
	// stuff :-O  Hopefully we will get a hook in Raspbian before this becomes
	// useless.
	checkFatal("exec real init",
		syscall.Exec("/sbin/init", os.Args, nil))
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fuse

import (
	"fmt"
	"syscall"
	"unsafe"

	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/fuse/internal/buffer"
	"github.com/jacobsa/fuse/internal/fusekernel"
)

// Given an op, return the response that should be sent to the kernel. If the
// op requires no response, return a nil response. If the op is unknown, return
// an error that should be sent to the kernel.
func kernelResponse(
	untyped fuseops.Op,
	protocol fusekernel.Protocol) (b buffer.OutMessage, err error) {
	switch o := untyped.(type) {
	case *fuseops.LookUpInodeOp:
		size := fusekernel.EntryOutSize(protocol)
		b = buffer.NewOutMessage(size)
		out := (*fusekernel.EntryOut)(b.Grow(size))
		convertChildInodeEntry(&o.Entry, out)

	case *fuseops.GetInodeAttributesOp:
		size := fusekernel.AttrOutSize(protocol)
		b = buffer.NewOutMessage(size)
		out := (*fusekernel.AttrOut)(b.Grow(size))
		out.AttrValid, out.AttrValidNsec = convertExpirationTime(o.AttributesExpiration)
		convertAttributes(o.Inode, &o.Attributes, &out.Attr)

	case *fuseops.SetInodeAttributesOp:
		size := fusekernel.AttrOutSize(protocol)
		b = buffer.NewOutMessage(size)
		out := (*fusekernel.AttrOut)(b.Grow(size))
		out.AttrValid, out.AttrValidNsec = convertExpirationTime(o.AttributesExpiration)
		convertAttributes(o.Inode, &o.Attributes, &out.Attr)

	case *fuseops.ForgetInodeOp:
		// No response.

	case *fuseops.MkDirOp:
		size := fusekernel.EntryOutSize(protocol)
		b = buffer.NewOutMessage(size)
		out := (*fusekernel.EntryOut)(b.Grow(size))
		convertChildInodeEntry(&o.Entry, out)

	case *fuseops.CreateFileOp:
		eSize := fusekernel.EntryOutSize(protocol)
		b = buffer.NewOutMessage(eSize + unsafe.Sizeof(fusekernel.OpenOut{}))

		e := (*fusekernel.EntryOut)(b.Grow(eSize))
		convertChildInodeEntry(&o.Entry, e)

		oo := (*fusekernel.OpenOut)(b.Grow(unsafe.Sizeof(fusekernel.OpenOut{})))
		oo.Fh = uint64(o.Handle)

	case *fuseops.CreateSymlinkOp:
		size := fusekernel.EntryOutSize(protocol)
		b = buffer.NewOutMessage(size)
		out := (*fusekernel.EntryOut)(b.Grow(size))
		convertChildInodeEntry(&o.Entry, out)

	case *fuseops.RenameOp:
		b = buffer.NewOutMessage(0)

	case *fuseops.RmDirOp:
		b = buffer.NewOutMessage(0)

	case *fuseops.UnlinkOp:
		b = buffer.NewOutMessage(0)

	case *fuseops.OpenDirOp:
		b = buffer.NewOutMessage(unsafe.Sizeof(fusekernel.OpenOut{}))
		out := (*fusekernel.OpenOut)(b.Grow(unsafe.Sizeof(fusekernel.OpenOut{})))
		out.Fh = uint64(o.Handle)

	case *fuseops.ReadDirOp:
		b = buffer.NewOutMessage(uintptr(len(o.Data)))
		b.Append(o.Data)

	case *fuseops.ReleaseDirHandleOp:
		b = buffer.NewOutMessage(0)

	case *fuseops.OpenFileOp:
		b = buffer.NewOutMessage(unsafe.Sizeof(fusekernel.OpenOut{}))
		out := (*fusekernel.OpenOut)(b.Grow(unsafe.Sizeof(fusekernel.OpenOut{})))
		out.Fh = uint64(o.Handle)

	case *fuseops.ReadFileOp:
		b = buffer.NewOutMessage(uintptr(len(o.Data)))
		b.Append(o.Data)

	case *fuseops.WriteFileOp:
		b = buffer.NewOutMessage(unsafe.Sizeof(fusekernel.WriteOut{}))
		out := (*fusekernel.WriteOut)(b.Grow(unsafe.Sizeof(fusekernel.WriteOut{})))
		out.Size = uint32(len(o.Data))

	case *fuseops.SyncFileOp:
		b = buffer.NewOutMessage(0)

	case *fuseops.FlushFileOp:
		b = buffer.NewOutMessage(0)

	case *fuseops.ReleaseFileHandleOp:
		b = buffer.NewOutMessage(0)

	case *fuseops.ReadSymlinkOp:
		b = buffer.NewOutMessage(uintptr(len(o.Target)))
		b.AppendString(o.Target)

	case *internalStatFSOp:
		b = buffer.NewOutMessage(unsafe.Sizeof(fusekernel.StatfsOut{}))
		b.Grow(unsafe.Sizeof(fusekernel.StatfsOut{}))

	case *internalInterruptOp:
		// No response.

	case *internalInitOp:
		b = buffer.NewOutMessage(unsafe.Sizeof(fusekernel.InitOut{}))
		out := (*fusekernel.InitOut)(b.Grow(unsafe.Sizeof(fusekernel.InitOut{})))

		out.Major = o.Library.Major
		out.Minor = o.Library.Minor
		out.MaxReadahead = o.MaxReadahead
		out.Flags = uint32(o.Flags)
		out.MaxWrite = o.MaxWrite

	default:
		err = syscall.ENOSYS
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Internal
////////////////////////////////////////////////////////////////////////

// A sentinel used for unknown ops. The user is expected to respond with a
// non-nil error.
type unknownOp struct {
	opCode uint32
	inode  fuseops.InodeID
}

func (o *unknownOp) ShortDesc() (desc string) {
	desc = fmt.Sprintf("<opcode %d>(inode=%v)", o.opCode, o.inode)
	return
}

func (o *unknownOp) DebugString() string {
	return o.ShortDesc()
}

// Common implementation for our "internal" ops that don't need to be pretty.
type internalOp struct {
}

func (o *internalOp) ShortDesc() string   { return "<internalOp>" }
func (o *internalOp) DebugString() string { return "<internalOp>" }

type internalStatFSOp struct {
	internalOp
}

func (o *internalStatFSOp) kernelResponse() (b buffer.OutMessage) {
	b = buffer.NewOutMessage(unsafe.Sizeof(fusekernel.StatfsOut{}))
	b.Grow(unsafe.Sizeof(fusekernel.StatfsOut{}))

	return
}

type internalInterruptOp struct {
	internalOp
	FuseID uint64
}

func (o *internalInterruptOp) kernelResponse() (b buffer.OutMessage) {
	panic("Shouldn't get here.")
}

type internalInitOp struct {
	internalOp

	// In
	Kernel fusekernel.Protocol

	// Out
	Library      fusekernel.Protocol
	MaxReadahead uint32
	Flags        fusekernel.InitFlags
	MaxWrite     uint32
}

func (o *internalInitOp) kernelResponse() (b buffer.OutMessage) {
	b = buffer.NewOutMessage(unsafe.Sizeof(fusekernel.InitOut{}))
	out := (*fusekernel.InitOut)(b.Grow(unsafe.Sizeof(fusekernel.InitOut{})))

	out.Major = o.Library.Major
	out.Minor = o.Library.Minor
	out.MaxReadahead = o.MaxReadahead
	out.Flags = uint32(o.Flags)
	out.MaxWrite = o.MaxWrite

	return
}
<commit_msg>Retooled kernelResponse.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fuse

import (
	"fmt"
	"syscall"
	"unsafe"

	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/fuse/internal/buffer"
	"github.com/jacobsa/fuse/internal/fusekernel"
)

// Return the response that should be sent to the kernel. If the op requires no
// response, return a nil response.
func kernelResponse(
	fuseID uint64,
	op fuseops.Op,
	opErr error,
	protocol fusekernel.Protocol) (msg []byte) {
	// If the user replied with an error, create room enough just for the result
	// header and fill it in with an error. Otherwise create an appropriate
	// response.
	var b *buffer.OutMessage
	if opErr != nil {
		b = buffer.NewOutMessage(0)
		if errno, ok := opErr.(syscall.Errno); ok {
			b.OutHeader().Error = -int32(errno)
		} else {
			b.OutHeader().Error = -int32(syscall.EIO)
		}
	} else {
		b = kernelResponseForOp(op, protocol)
	}

	msg = b.Bytes()

	// Fill in the rest of the header.
	h := b.OutHeader()
	h.Unique = fuseID
	h.Len = uint32(len(msg))

	return
}

// Like kernelResponse, but assumes the user replied with a nil error to the
// op.
func kernelResponseForOp(
	op fuseops.Op,
	protocol fusekernel.Protocol) (b *buffer.OutMessage) {
	// Create the appropriate output message
	switch o := op.(type) {
	case *fuseops.LookUpInodeOp:
		size := fusekernel.EntryOutSize(protocol)
		b = buffer.NewOutMessage(size)
		out := (*fusekernel.EntryOut)(b.Grow(size))
		convertChildInodeEntry(&o.Entry, out)

	case *fuseops.GetInodeAttributesOp:
		size := fusekernel.AttrOutSize(protocol)
		b = buffer.NewOutMessage(size)
		out := (*fusekernel.AttrOut)(b.Grow(size))
		out.AttrValid, out.AttrValidNsec = convertExpirationTime(o.AttributesExpiration)
		convertAttributes(o.Inode, &o.Attributes, &out.Attr)

	case *fuseops.SetInodeAttributesOp:
		size := fusekernel.AttrOutSize(protocol)
		b = buffer.NewOutMessage(size)
		out := (*fusekernel.AttrOut)(b.Grow(size))
		out.AttrValid, out.AttrValidNsec = convertExpirationTime(o.AttributesExpiration)
		convertAttributes(o.Inode, &o.Attributes, &out.Attr)

	case *fuseops.ForgetInodeOp:
		// No response.

	case *fuseops.MkDirOp:
		size := fusekernel.EntryOutSize(protocol)
		b = buffer.NewOutMessage(size)
		out := (*fusekernel.EntryOut)(b.Grow(size))
		convertChildInodeEntry(&o.Entry, out)

	case *fuseops.CreateFileOp:
		eSize := fusekernel.EntryOutSize(protocol)
		b = buffer.NewOutMessage(eSize + unsafe.Sizeof(fusekernel.OpenOut{}))

		e := (*fusekernel.EntryOut)(b.Grow(eSize))
		convertChildInodeEntry(&o.Entry, e)

		oo := (*fusekernel.OpenOut)(b.Grow(unsafe.Sizeof(fusekernel.OpenOut{})))
		oo.Fh = uint64(o.Handle)

	case *fuseops.CreateSymlinkOp:
		size := fusekernel.EntryOutSize(protocol)
		b = buffer.NewOutMessage(size)
		out := (*fusekernel.EntryOut)(b.Grow(size))
		convertChildInodeEntry(&o.Entry, out)

	case *fuseops.RenameOp:
		b = buffer.NewOutMessage(0)

	case *fuseops.RmDirOp:
		b = buffer.NewOutMessage(0)

	case *fuseops.UnlinkOp:
		b = buffer.NewOutMessage(0)

	case *fuseops.OpenDirOp:
		b = buffer.NewOutMessage(unsafe.Sizeof(fusekernel.OpenOut{}))
		out := (*fusekernel.OpenOut)(b.Grow(unsafe.Sizeof(fusekernel.OpenOut{})))
		out.Fh = uint64(o.Handle)

	case *fuseops.ReadDirOp:
		b = buffer.NewOutMessage(uintptr(len(o.Data)))
		b.Append(o.Data)

	case *fuseops.ReleaseDirHandleOp:
		b = buffer.NewOutMessage(0)

	case *fuseops.OpenFileOp:
		b = buffer.NewOutMessage(unsafe.Sizeof(fusekernel.OpenOut{}))
		out := (*fusekernel.OpenOut)(b.Grow(unsafe.Sizeof(fusekernel.OpenOut{})))
		out.Fh = uint64(o.Handle)

	case *fuseops.ReadFileOp:
		b = buffer.NewOutMessage(uintptr(len(o.Data)))
		b.Append(o.Data)

	case *fuseops.WriteFileOp:
		b = buffer.NewOutMessage(unsafe.Sizeof(fusekernel.WriteOut{}))
		out := (*fusekernel.WriteOut)(b.Grow(unsafe.Sizeof(fusekernel.WriteOut{})))
		out.Size = uint32(len(o.Data))

	case *fuseops.SyncFileOp:
		b = buffer.NewOutMessage(0)

	case *fuseops.FlushFileOp:
		b = buffer.NewOutMessage(0)

	case *fuseops.ReleaseFileHandleOp:
		b = buffer.NewOutMessage(0)

	case *fuseops.ReadSymlinkOp:
		b = buffer.NewOutMessage(uintptr(len(o.Target)))
		b.AppendString(o.Target)

	case *internalStatFSOp:
		b = buffer.NewOutMessage(unsafe.Sizeof(fusekernel.StatfsOut{}))
		b.Grow(unsafe.Sizeof(fusekernel.StatfsOut{}))

	case *internalInterruptOp:
		// No response.

	case *internalInitOp:
		b = buffer.NewOutMessage(unsafe.Sizeof(fusekernel.InitOut{}))
		out := (*fusekernel.InitOut)(b.Grow(unsafe.Sizeof(fusekernel.InitOut{})))

		out.Major = o.Library.Major
		out.Minor = o.Library.Minor
		out.MaxReadahead = o.MaxReadahead
		out.Flags = uint32(o.Flags)
		out.MaxWrite = o.MaxWrite

	default:
		panic(fmt.Sprintf("Unknown op: %#v", op))
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Internal
////////////////////////////////////////////////////////////////////////

// A sentinel used for unknown ops. The user is expected to respond with a
// non-nil error.
type unknownOp struct {
	opCode uint32
	inode  fuseops.InodeID
}

func (o *unknownOp) ShortDesc() (desc string) {
	desc = fmt.Sprintf("<opcode %d>(inode=%v)", o.opCode, o.inode)
	return
}

func (o *unknownOp) DebugString() string {
	return o.ShortDesc()
}

// Common implementation for our "internal" ops that don't need to be pretty.
type internalOp struct {
}

func (o *internalOp) ShortDesc() string   { return "<internalOp>" }
func (o *internalOp) DebugString() string { return "<internalOp>" }

type internalStatFSOp struct {
	internalOp
}

func (o *internalStatFSOp) kernelResponse() (b buffer.OutMessage) {
	b = buffer.NewOutMessage(unsafe.Sizeof(fusekernel.StatfsOut{}))
	b.Grow(unsafe.Sizeof(fusekernel.StatfsOut{}))

	return
}

type internalInterruptOp struct {
	internalOp
	FuseID uint64
}

func (o *internalInterruptOp) kernelResponse() (b buffer.OutMessage) {
	panic("Shouldn't get here.")
}

type internalInitOp struct {
	internalOp

	// In
	Kernel fusekernel.Protocol

	// Out
	Library      fusekernel.Protocol
	MaxReadahead uint32
	Flags        fusekernel.InitFlags
	MaxWrite     uint32
}

func (o *internalInitOp) kernelResponse() (b buffer.OutMessage) {
	b = buffer.NewOutMessage(unsafe.Sizeof(fusekernel.InitOut{}))
	out := (*fusekernel.InitOut)(b.Grow(unsafe.Sizeof(fusekernel.InitOut{})))

	out.Major = o.Library.Major
	out.Minor = o.Library.Minor
	out.MaxReadahead = o.MaxReadahead
	out.Flags = uint32(o.Flags)
	out.MaxWrite = o.MaxWrite

	return
}
<|endoftext|>


<commit_before>package sdk

/*
   Copyright 2016-2017 Alexander I.Grafov <grafov@gmail.com>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

   ॐ तारे तुत्तारे तुरे स्व
*/

type Org struct {
	ID      uint    `json:"id"`
	Name    string  `json:"name"`
	Address Address `json:"address"`
}

type OrgUser struct {
	ID    uint   `json:"id"`
	OrgId uint   `json:"orgId"`
	Email string `json:"email"`
	Login string `json:"login"`
	Role  string `json:"role"`
}
<commit_msg>userId json tag instead of id (#134)<commit_after>package sdk

/*
   Copyright 2016-2017 Alexander I.Grafov <grafov@gmail.com>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

   ॐ तारे तुत्तारे तुरे स्व
*/

type Org struct {
	ID      uint    `json:"id"`
	Name    string  `json:"name"`
	Address Address `json:"address"`
}

type OrgUser struct {
	ID    uint   `json:"userId"`
	OrgId uint   `json:"orgId"`
	Email string `json:"email"`
	Login string `json:"login"`
	Role  string `json:"role"`
}
<|endoftext|>


<commit_before>package orm

import (
	"database/sql"
	"fmt"
	"reflect"
	"strings"
)

var dbHive map[string]*sql.DB = make(map[string]*sql.DB)

//create new database
func NewDatabase(dbname, dbtype, url string) {
	db, err := sql.Open(dbtype, url)
	if err != nil {
		panic(err.Error())
	}
	err = db.Ping()
	if err != nil {
		panic(err.Error())
	}
	dbHive[dbname] = db
}

//module
type Module struct {
	columnstr string //select field
	tableName string //table
	filters   string //condition
	orderby   string //orderby
	limit     string //limit
	join      string //join
	pk        string //pk
	dbname    string //dbname
}

//create new Module
func NewModule(tableName string) *Module {
	m := &Module{tableName: tableName, columnstr: "*", dbname: "default", pk: "id"}
	return m
}

func (m *Module) GetDB() *sql.DB {
	return dbHive[m.dbname]
}

//change db
func (m *Module) User(dbname string) *Module {
	m.dbname = dbname
	return m
}

//select fields
func (m *Module) Select(fields ...string) *Module {
	m.columnstr = ""
	for _, f := range fields {
		m.columnstr = m.columnstr + f + ","
	}
	return m
}

//Filter
func (m *Module) Filter(param ...interface{}) *Module {
	for _, p := range param {
		m.filters += fmt.Sprintf("%v", p)
	}
	return m
}

//orderBy
func (m *Module) OrderBy(param string) *Module {
	m.orderby = fmt.Sprintf("ORDER By %v", param)
	return m
}

//limit
func (m *Module) Limit(size ...int) *Module {
	if len(size) > 1 {
		m.limit = fmt.Sprintf("Limit %d,%d", size[0], size[1])
		return m
	} else {
		m.limit = fmt.Sprintf("Limit %d", size[0])
		return m
	}
}

//leftJoin
func (m *Module) LeftJoin(table, condition string) *Module {
	m.join = fmt.Sprintf("LEFT JOIN %v ON %v", table, condition)
	return m
}

//rightJoin
func (m *Module) RightJoin(table, condition string) *Module {
	m.join = fmt.Sprintf("RIGHT JOIN %v ON %v", table, condition)
	return m
}

//join
func (m *Module) Join(table, condition string) *Module {
	m.join = fmt.Sprintf("INNER JOIN %v ON %v", table, condition)
	return m
}

//fulljoin
func (m *Module) FullJoin(table, condition string) *Module {
	m.join = fmt.Sprintf("FULL JOIN %v ON %v", table, condition)
	return m
}

func (m *Module) getSqlString() string {
	columnstr := m.columnstr
	if l := len(columnstr); l > 1 {
		columnstr = columnstr[:l-1]
	}
	where := m.filters
	where = strings.TrimSpace(where)
	if len(where) > 0 {
		where = "where " + where
	}
	query := fmt.Sprintf("select %v from %v %v %v %v %v", columnstr, m.tableName, m.join, where, m.orderby, m.limit)
	return query
}

func (m *Module) Query(callBackFunc func(*sql.Rows)) error {
	db := dbHive[m.dbname]
	rows, err := db.Query(m.getSqlString())
	if err != nil {
		return err
	}
	defer rows.Close()
	callBackFunc(rows)
	return nil
}
func (m *Module) QueryOne(callBackFunc func(*sql.Row)) {
	db := dbHive[m.dbname]
	row := db.QueryRow(m.getSqlString())
	callBackFunc(row)
}

func (m *Module) OneRecord() (record Record, err error) {
	rs, err := m.Limit(1).AllRecords()
	if err != nil {
		return record, err
	}
	return rs[0], nil
}

func (m *Module) AllRecords() ([]Record, error) {
	db := dbHive[m.dbname]
	rows, err := db.Query(m.getSqlString())
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	records := make([]Record, 0)
	columns, _ := rows.Columns()
	values := make([]sql.RawBytes, len(columns))
	scanargs := make([]interface{}, len(values))
	for i := range values {
		scanargs[i] = &values[i]
	}
	for rows.Next() {
		err := rows.Scan(scanargs...)
		if err != nil {
			fmt.Println(err)
		}
		record := NewRecord()
		for i, v := range values {
			record.result[columns[i]] = v
		}
		records = append(records, record)
	}
	return records, nil
}
func (m *Module) SetPK(pk string) *Module {
	m.pk = pk
	return m
}

func (m *Module) FindRecordById(id int) *Module {
	m.Filter(m.pk, "=", id)
	return m
}

func (m *Module) Insert(record Record) error {
	columns := ""
	values := ""
	for c, v := range record.param {
		columns = columns + c + ","
		rv := reflect.ValueOf(v)
		switch rv.Kind() {
		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Bool:
			values = values + fmt.Sprintf("%v", v) + ","
		default:
			values = values + fmt.Sprintf("'%v'", v) + ","
		}
	}
	if l := len(columns); l > 0 {
		columns = columns[:l-1]
	}
	if l := len(values); l > 0 {
		values = values[:l-1]
	}
	insertSql := fmt.Sprintf("insert into %v(%v) values(%v)", m.tableName, columns, values)
	fmt.Println(insertSql)
	db := dbHive[m.dbname]
	_, err := db.Exec(insertSql)
	return err
}

func (m *Module) DeleteById(id int) error {
	m.Filter(m.pk, "=", id)
	return m.Delete()
}

func (m *Module) Delete() error {
	where := m.filters
	where = strings.TrimSpace(where)
	if len(where) > 0 {
		where = "where " + where
	}
	delSql := fmt.Sprintf("delete %v %v", m.tableName, where)
	fmt.Println(delSql)
	_, err := dbHive[m.dbname].Exec(delSql)
	return err
}
<commit_msg>fix bug<commit_after>package orm

import (
	"database/sql"
	"errors"
	"fmt"
	"log"
	"reflect"
	"strings"
)

var dbHive map[string]*sql.DB = make(map[string]*sql.DB)

//create new database
func NewDatabase(dbname, dbtype, url string) {
	db, err := sql.Open(dbtype, url)
	if err != nil {
		panic(err.Error())
	}
	err = db.Ping()
	if err != nil {
		panic(err.Error())
	}
	dbHive[dbname] = db
}

//module
type Module struct {
	columnstr string //select field
	tableName string //table
	filters   string //condition
	orderby   string //orderby
	limit     string //limit
	join      string //join
	pk        string //pk
	dbname    string //dbname
}

//create new Module
func NewModule(tableName string) *Module {
	m := &Module{tableName: tableName, columnstr: "*", dbname: "default", pk: "id"}
	return m
}

func (m *Module) Clean() *Module {
	m.columnstr = "*"
	m.filters = ""
	m.orderby = ""
	m.limit = ""
	m.join = ""
	m.pk = "id"
	return m
}

func (m *Module) GetDB() *sql.DB {
	return dbHive[m.dbname]
}

//change db
func (m *Module) User(dbname string) *Module {
	m.dbname = dbname
	return m
}

//select fields
func (m *Module) Select(fields ...string) *Module {
	m.columnstr = ""
	for _, f := range fields {
		m.columnstr = m.columnstr + f + ","
	}
	return m
}

//Filter
func (m *Module) Filter(param ...interface{}) *Module {
	for _, p := range param {
		m.filters += fmt.Sprintf("%v", p)
	}
	return m
}

//orderBy
func (m *Module) OrderBy(param string) *Module {
	m.orderby = fmt.Sprintf("ORDER By %v", param)
	return m
}

//limit
func (m *Module) Limit(size ...int) *Module {
	if len(size) > 1 {
		m.limit = fmt.Sprintf("Limit %d,%d", size[0], size[1])
		return m
	} else {
		m.limit = fmt.Sprintf("Limit %d", size[0])
		return m
	}
}

//leftJoin
func (m *Module) LeftJoin(table, condition string) *Module {
	m.join = fmt.Sprintf("LEFT JOIN %v ON %v", table, condition)
	return m
}

//rightJoin
func (m *Module) RightJoin(table, condition string) *Module {
	m.join = fmt.Sprintf("RIGHT JOIN %v ON %v", table, condition)
	return m
}

//join
func (m *Module) Join(table, condition string) *Module {
	m.join = fmt.Sprintf("INNER JOIN %v ON %v", table, condition)
	return m
}

//fulljoin
func (m *Module) FullJoin(table, condition string) *Module {
	m.join = fmt.Sprintf("FULL JOIN %v ON %v", table, condition)
	return m
}

func (m *Module) getSqlString() string {
	columnstr := m.columnstr
	if l := len(columnstr); l > 1 {
		columnstr = columnstr[:l-1]
	}
	query := m.buildSql(columnstr)
	log.Println("sql = ", query)
	return query
}

func (m *Module) buildSql(columnstr string) string {
	where := m.filters
	where = strings.TrimSpace(where)
	if len(where) > 0 {
		where = "where " + where
	}
	query := fmt.Sprintf("select %v from %v %v %v %v %v", columnstr, m.tableName, m.join, where, m.orderby, m.limit)
	return query
}

func (m *Module) Query(callBackFunc func(*sql.Rows)) error {
	db := dbHive[m.dbname]
	rows, err := db.Query(m.getSqlString())
	if err != nil {
		return err
	}
	defer rows.Close()
	callBackFunc(rows)
	return nil
}
func (m *Module) QueryOne(callBackFunc func(*sql.Row)) {
	db := dbHive[m.dbname]
	row := db.QueryRow(m.getSqlString())
	callBackFunc(row)
}

func (m *Module) IsExist() (bool, error) {
	db := dbHive[m.dbname]
	row := db.QueryRow(m.buildSql("count(*)"))
	var count int
	err := row.Scan(&count)
	if count > 0 {
		return true, nil
	}
	return false, err
}

func (m *Module) OneRecord() (record Record, err error) {
	rs, err := m.Limit(1).AllRecords()
	if err != nil {
		return record, err
	}
	if len(rs) == 0 {
		return NewRecord(), errors.New("not fond record")
	}
	return rs[0], nil
}

func (m *Module) AllRecords() ([]Record, error) {
	db := dbHive[m.dbname]
	rows, err := db.Query(m.getSqlString())
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	records := make([]Record, 0)
	columns, _ := rows.Columns()
	values := make([]sql.RawBytes, len(columns))
	scanargs := make([]interface{}, len(values))
	for i := range values {
		scanargs[i] = &values[i]
	}
	for rows.Next() {
		err := rows.Scan(scanargs...)
		if err != nil {
			fmt.Println(err)
		}
		record := NewRecord()
		for i, v := range values {
			record.result[columns[i]] = v
		}
		records = append(records, record)
	}
	return records, nil
}
func (m *Module) SetPK(pk string) *Module {
	m.pk = pk
	return m
}

func (m *Module) FindRecordById(id int) *Module {
	m.Filter(m.pk, "=", id)
	return m
}

func (m *Module) Insert(record Record) error {
	columns := ""
	values := ""
	for c, v := range record.param {
		columns = columns + c + ","
		rv := reflect.ValueOf(v)
		switch rv.Kind() {
		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Bool:
			values = values + fmt.Sprintf("%v", v) + ","
		default:
			values = values + fmt.Sprintf("'%v'", v) + ","
		}
	}
	if l := len(columns); l > 0 {
		columns = columns[:l-1]
	}
	if l := len(values); l > 0 {
		values = values[:l-1]
	}
	insertSql := fmt.Sprintf("insert into %v(%v) values(%v)", m.tableName, columns, values)
	fmt.Println(insertSql)
	db := dbHive[m.dbname]
	_, err := db.Exec(insertSql)
	return err
}

func (m *Module) DeleteById(id int) error {
	m.Filter(m.pk, "=", id)
	return m.Delete()
}

func (m *Module) Delete() error {
	where := m.filters
	where = strings.TrimSpace(where)
	if len(where) > 0 {
		where = "where " + where
	}
	delSql := fmt.Sprintf("delete %v %v", m.tableName, where)
	fmt.Println(delSql)
	_, err := dbHive[m.dbname].Exec(delSql)
	return err
}
<|endoftext|>


<commit_before>package log

import (
	"fmt"
	"runtime"
)

var (
	_logger *logger
)

func init() {
	_logger = new()
}

// RegisterHandler adds a new Log Handler and specifies what log levels
// the handler will be passed log entries for
func RegisterHandler(handler Handler, levels ...Level) {
	_logger.RegisterHandler(handler, levels...)
}

// Debug level formatted message.
func Debug(v ...interface{}) {
	e := _logger.newEntry(DebugLevel, fmt.Sprint(v...), nil, skipLevel)
	_logger.handleEntry(e)
}

// Debugf level formatted message.
func Debugf(msg string, v ...interface{}) {
	e := _logger.newEntry(DebugLevel, fmt.Sprintf(msg, v...), nil, skipLevel)
	_logger.handleEntry(e)
}

// Info level formatted message.
func Info(v ...interface{}) {
	e := _logger.newEntry(InfoLevel, fmt.Sprint(v...), nil, skipLevel)
	_logger.handleEntry(e)
}

// Infof level formatted message.
func Infof(msg string, v ...interface{}) {
	e := _logger.newEntry(InfoLevel, fmt.Sprintf(msg, v...), nil, skipLevel)
	_logger.handleEntry(e)
}

// Warn level formatted message.
func Warn(v ...interface{}) {
	e := _logger.newEntry(WarnLevel, fmt.Sprint(v...), nil, skipLevel)
	_logger.handleEntry(e)
}

// Warnf level formatted message.
func Warnf(msg string, v ...interface{}) {
	e := _logger.newEntry(WarnLevel, fmt.Sprintf(msg, v...), nil, skipLevel)
	_logger.handleEntry(e)
}

// Error level formatted message
func Error(v ...interface{}) {
	e := _logger.newEntry(ErrorLevel, fmt.Sprint(v...), nil, skipLevel)
	_logger.handleEntry(e)
}

// Errorf level formatted message
func Errorf(msg string, v ...interface{}) {
	e := _logger.newEntry(ErrorLevel, fmt.Sprintf(msg, v...), nil, skipLevel)
	_logger.handleEntry(e)
}

// Panic level formatted message
func Panic(v ...interface{}) {
	s := fmt.Sprint(v...)
	e := _logger.newEntry(PanicLevel, s, nil, skipLevel)
	_logger.handleEntry(e)
	panic(s)
}

// Panicf level formatted message
func Panicf(msg string, v ...interface{}) {
	s := fmt.Sprintf(msg, v...)
	e := _logger.newEntry(PanicLevel, s, nil, skipLevel)
	_logger.handleEntry(e)
	panic(s)
}

// Fatal level formatted message, followed by an exit.
func Fatal(v ...interface{}) {
	e := _logger.newEntry(FatalLevel, fmt.Sprint(v...), nil, skipLevel)
	_logger.handleEntry(e)
	exitFunc(1)
}

// Fatalf level formatted message, followed by an exit.
func Fatalf(msg string, v ...interface{}) {
	e := _logger.newEntry(FatalLevel, fmt.Sprintf(msg, v...), nil, skipLevel)
	_logger.handleEntry(e)
	exitFunc(1)
}

// WithFields returns a log Entry with fields set
func WithFields(fields Fields) Logger {
	e := _logger.newEntry(InfoLevel, "", fields, skipLevel)
	return e
}

// StackTrace creates a new log Entry with pre-populated field with stack trace.
func StackTrace() Logger {
	trace := make([]byte, 1<<16)
	n := runtime.Stack(trace, true)
	if n > 7000 {
		n = 7000
	}
	customFields := Fields{
		"stack_trace": string(trace[:n]) + "\n",
	}
	return _logger.newEntry(DebugLevel, "", customFields, skipLevel)
}

// SetAppID set a constant application key
// that will be set on all log Entry objects
func SetAppID(id string) {
	_logger.appID = id
}

// AppID return an application key
func AppID() string {
	return _logger.appID
}
<commit_msg>add golang context support<commit_after>package log

import (
	"context"
	"fmt"
	"runtime"
)

var (
	_logger *logger
)

const (
	requestIdKey int = iota
)

func init() {
	_logger = new()
}

// RegisterHandler adds a new Log Handler and specifies what log levels
// the handler will be passed log entries for
func RegisterHandler(handler Handler, levels ...Level) {
	_logger.RegisterHandler(handler, levels...)
}

// Debug level formatted message.
func Debug(v ...interface{}) {
	e := _logger.newEntry(DebugLevel, fmt.Sprint(v...), nil, skipLevel)
	_logger.handleEntry(e)
}

// Debugf level formatted message.
func Debugf(msg string, v ...interface{}) {
	e := _logger.newEntry(DebugLevel, fmt.Sprintf(msg, v...), nil, skipLevel)
	_logger.handleEntry(e)
}

// Info level formatted message.
func Info(v ...interface{}) {
	e := _logger.newEntry(InfoLevel, fmt.Sprint(v...), nil, skipLevel)
	_logger.handleEntry(e)
}

// Infof level formatted message.
func Infof(msg string, v ...interface{}) {
	e := _logger.newEntry(InfoLevel, fmt.Sprintf(msg, v...), nil, skipLevel)
	_logger.handleEntry(e)
}

// Warn level formatted message.
func Warn(v ...interface{}) {
	e := _logger.newEntry(WarnLevel, fmt.Sprint(v...), nil, skipLevel)
	_logger.handleEntry(e)
}

// Warnf level formatted message.
func Warnf(msg string, v ...interface{}) {
	e := _logger.newEntry(WarnLevel, fmt.Sprintf(msg, v...), nil, skipLevel)
	_logger.handleEntry(e)
}

// Error level formatted message
func Error(v ...interface{}) {
	e := _logger.newEntry(ErrorLevel, fmt.Sprint(v...), nil, skipLevel)
	_logger.handleEntry(e)
}

// Errorf level formatted message
func Errorf(msg string, v ...interface{}) {
	e := _logger.newEntry(ErrorLevel, fmt.Sprintf(msg, v...), nil, skipLevel)
	_logger.handleEntry(e)
}

// Panic level formatted message
func Panic(v ...interface{}) {
	s := fmt.Sprint(v...)
	e := _logger.newEntry(PanicLevel, s, nil, skipLevel)
	_logger.handleEntry(e)
	panic(s)
}

// Panicf level formatted message
func Panicf(msg string, v ...interface{}) {
	s := fmt.Sprintf(msg, v...)
	e := _logger.newEntry(PanicLevel, s, nil, skipLevel)
	_logger.handleEntry(e)
	panic(s)
}

// Fatal level formatted message, followed by an exit.
func Fatal(v ...interface{}) {
	e := _logger.newEntry(FatalLevel, fmt.Sprint(v...), nil, skipLevel)
	_logger.handleEntry(e)
	exitFunc(1)
}

// Fatalf level formatted message, followed by an exit.
func Fatalf(msg string, v ...interface{}) {
	e := _logger.newEntry(FatalLevel, fmt.Sprintf(msg, v...), nil, skipLevel)
	_logger.handleEntry(e)
	exitFunc(1)
}

// WithFields returns a log Entry with fields set
func WithFields(fields Fields) Logger {
	e := _logger.newEntry(InfoLevel, "", fields, skipLevel)
	return e
}

// StackTrace creates a new log Entry with pre-populated field with stack trace.
func StackTrace() Logger {
	trace := make([]byte, 1<<16)
	n := runtime.Stack(trace, true)
	if n > 7000 {
		n = 7000
	}
	customFields := Fields{
		"stack_trace": string(trace[:n]) + "\n",
	}
	return _logger.newEntry(DebugLevel, "", customFields, skipLevel)
}

// SetAppID set a constant application key
// that will be set on all log Entry objects
func SetAppID(id string) {
	_logger.appID = id
}

// AppID return an application key
func AppID() string {
	return _logger.appID
}

// NewContext return a new context with a logger value
func NewContext(ctx context.Context, logger Logger) context.Context {
	return context.WithValue(ctx, requestIdKey, logger)
}

// FromContext return a logger from the context
func FromContext(ctx context.Context) Logger {
	val, ok := ctx.Value(requestIdKey).(*Entry)
	if ok {
		return val
	}
	return _logger
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"os/exec"
	"strings"

	"kylelemons.net/go/rx/repo"
)

var preCmd = &Command{
	Name:    "prescribe",
	Usage:   "<repo> <tag>",
	Summary: "Update the repository to the given tag/rev.",
	Help: `The prescribe command updates the repository to the named tag or
revision.  The <repo> can be the suffix of the repository root path,
as long as it is unique.  The <tag> is anything understood by the
underlying version control system as a commit, usually a tag, branch,
or commit.

After updating, prescribe will test, build, and the install each package
in the updated repository.  These steps can be disabled via flags such as
"rx prescribe --test=false repo tag".

By default, this will not link and install affected binaries; to turn this
behavior on, see the --link option.`,
}

var (
	preBuild = preCmd.Flag.Bool("build", true, "build all updated packages")
	preLink  = preCmd.Flag.Bool("link", false, "link and install all updated binaries")
	preTest  = preCmd.Flag.Bool("test", true, "test all updated packages")
	preInst  = preCmd.Flag.Bool("install", true, "install all updated packages")
	preCasc  = preCmd.Flag.Bool("cascade", true, "recursively process depending packages too")
)

func preFunc(cmd *Command, args ...string) {
	if len(args) != 2 {
		cmd.BadArgs("requires two arguments")
	}
	pathSuffix := args[0]
	repoTag := args[1]

	// Scan before accessing Repos
	if err := Scan(); err != nil {
		cmd.Fatalf("scan: %s", err)
	}

	// TODO(kevlar): This seems like something we'll be doing often...
	var rep *repo.Repository
	for p, r := range Repos {
		if strings.HasSuffix(p, pathSuffix) {
			if rep != nil {
				cmd.Fatalf("non-unique suffix %q", pathSuffix)
			}
			rep = r
		}
	}
	if rep == nil {
		cmd.Fatalf("unknown repo %q", pathSuffix)
	}

	fallback, err := rep.Head()
	if err != nil {
		cmd.Fatalf("failure to determine head: %s", err)
	}
	defer func() {
		if fallback != "" {
			rep.ToRev(fallback)
		}
	}()

	if err := rep.ToRev(repoTag); err != nil {
		cmd.Fatalf("failure to change rev to %q: %s", repoTag, err)
	}

	do := func(r *repo.Repository, subCmd string) error {
		for _, pkg := range r.Packages {
			switch subCmd {
			case "test":
				if !pkg.IsTestable() {
					continue
				}
				// Install dependencies so we don't get complaints
				if *preInst {
					exec.Command("go", "test", "-i", pkg.ImportPath)
				}
			case "install":
				if !*preLink && pkg.IsBinary() {
					continue
				}
			}
			log.Printf("   - %s", pkg.ImportPath)
			cmd := exec.Command("go", subCmd, pkg.ImportPath)
			cmd.Dir = os.TempDir()
			cmd.Stdout = os.Stdout
			cmd.Stderr = os.Stderr
			if err := cmd.Run(); err != nil {
				return err
			}
		}
		return nil
	}

	rebuilt := map[*repo.Repository]bool{}
	process := func(r *repo.Repository) {
		log.Printf("Processing repo in %q...", r.Path)
		if *preBuild {
			log.Printf(" - Build")
			if err := do(r, "build"); err != nil {
				cmd.Fatalf("build failed: %q on %q broke %q",
					repoTag, rep.Path, r.Path)
			}
		}

		if *preTest {
			log.Printf(" - Test")
			if err := do(r, "test"); err != nil {
				cmd.Fatalf("test failed: %q on %q broke %q: %s",
					repoTag, rep.Path, r.Path, err)
			}
		}

		if *preInst {
			log.Printf(" - Install")
			if err := do(r, "install"); err != nil {
				cmd.Fatalf("install failed: %q on %q broke %q",
					repoTag, rep.Path, r.Path)
			}
		}

		if *preCasc {
			log.Printf(" - Cascade")
			for _, check := range Repos {
				// Don't check repos we've already rebuilt
				if _, ok := rebuilt[check]; ok {
					continue
				}
				for _, dep := range check.RepoDeps {
					// If repo `check` depends on the current repo `r`
					if dep == r.Path {
						// rebuild it
						log.Printf("   - Repository %q", check.Path)
						rebuilt[check] = false
						break
					}
				}
			}
		}
	}

	cascade := true
	rebuilt[rep] = false

	for cascade {
		cascade = false
		for rep, processed := range rebuilt {
			if !processed {
				cascade = true
				rebuilt[rep] = true
				process(rep)
			}
		}
	}

	fallback = ""
}

func init() {
	preCmd.Run = preFunc
}
<commit_msg>actually run test -i<commit_after>package main

import (
	"log"
	"os"
	"os/exec"
	"strings"

	"kylelemons.net/go/rx/repo"
)

var preCmd = &Command{
	Name:    "prescribe",
	Usage:   "<repo> <tag>",
	Summary: "Update the repository to the given tag/rev.",
	Help: `The prescribe command updates the repository to the named tag or
revision.  The <repo> can be the suffix of the repository root path,
as long as it is unique.  The <tag> is anything understood by the
underlying version control system as a commit, usually a tag, branch,
or commit.

After updating, prescribe will test, build, and the install each package
in the updated repository.  These steps can be disabled via flags such as
"rx prescribe --test=false repo tag".

By default, this will not link and install affected binaries; to turn this
behavior on, see the --link option.`,
}

var (
	preBuild = preCmd.Flag.Bool("build", true, "build all updated packages")
	preLink  = preCmd.Flag.Bool("link", false, "link and install all updated binaries")
	preTest  = preCmd.Flag.Bool("test", true, "test all updated packages")
	preInst  = preCmd.Flag.Bool("install", true, "install all updated packages")
	preCasc  = preCmd.Flag.Bool("cascade", true, "recursively process depending packages too")
)

func preFunc(cmd *Command, args ...string) {
	if len(args) != 2 {
		cmd.BadArgs("requires two arguments")
	}
	pathSuffix := args[0]
	repoTag := args[1]

	// Scan before accessing Repos
	if err := Scan(); err != nil {
		cmd.Fatalf("scan: %s", err)
	}

	// TODO(kevlar): This seems like something we'll be doing often...
	var rep *repo.Repository
	for p, r := range Repos {
		if strings.HasSuffix(p, pathSuffix) {
			if rep != nil {
				cmd.Fatalf("non-unique suffix %q", pathSuffix)
			}
			rep = r
		}
	}
	if rep == nil {
		cmd.Fatalf("unknown repo %q", pathSuffix)
	}

	fallback, err := rep.Head()
	if err != nil {
		cmd.Fatalf("failure to determine head: %s", err)
	}
	defer func() {
		if fallback != "" {
			rep.ToRev(fallback)
		}
	}()

	if err := rep.ToRev(repoTag); err != nil {
		cmd.Fatalf("failure to change rev to %q: %s", repoTag, err)
	}

	do := func(r *repo.Repository, subCmd string) error {
		for _, pkg := range r.Packages {
			switch subCmd {
			case "test":
				if !pkg.IsTestable() {
					continue
				}
				// Install dependencies so we don't get complaints
				if *preInst {
					exec.Command("go", "test", "-i", pkg.ImportPath).Run()
				}
			case "install":
				if !*preLink && pkg.IsBinary() {
					continue
				}
			}
			log.Printf("   - %s", pkg.ImportPath)
			cmd := exec.Command("go", subCmd, pkg.ImportPath)
			cmd.Dir = os.TempDir()
			cmd.Stdout = os.Stdout
			cmd.Stderr = os.Stderr
			if err := cmd.Run(); err != nil {
				return err
			}
		}
		return nil
	}

	rebuilt := map[*repo.Repository]bool{}
	process := func(r *repo.Repository) {
		log.Printf("Processing repo in %q...", r.Path)
		if *preBuild {
			log.Printf(" - Build")
			if err := do(r, "build"); err != nil {
				cmd.Fatalf("build failed: %q on %q broke %q",
					repoTag, rep.Path, r.Path)
			}
		}

		if *preTest {
			log.Printf(" - Test")
			if err := do(r, "test"); err != nil {
				cmd.Fatalf("test failed: %q on %q broke %q: %s",
					repoTag, rep.Path, r.Path, err)
			}
		}

		if *preInst {
			log.Printf(" - Install")
			if err := do(r, "install"); err != nil {
				cmd.Fatalf("install failed: %q on %q broke %q",
					repoTag, rep.Path, r.Path)
			}
		}

		if *preCasc {
			log.Printf(" - Cascade")
			for _, check := range Repos {
				// Don't check repos we've already rebuilt
				if _, ok := rebuilt[check]; ok {
					continue
				}
				for _, dep := range check.RepoDeps {
					// If repo `check` depends on the current repo `r`
					if dep == r.Path {
						// rebuild it
						log.Printf("   - Repository %q", check.Path)
						rebuilt[check] = false
						break
					}
				}
			}
		}
	}

	cascade := true
	rebuilt[rep] = false

	for cascade {
		cascade = false
		for rep, processed := range rebuilt {
			if !processed {
				cascade = true
				rebuilt[rep] = true
				process(rep)
			}
		}
	}

	fallback = ""
}

func init() {
	preCmd.Run = preFunc
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"net/http"
	"strconv"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"github.com/koding/multiconfig"
)

type DefaultConf struct {
	Board    	string `default:"car"`
	BaseUrl     string `default:"https://www.ptt.cc/bbs/"`
	CarBoardUrl string `default:"https://www.ptt.cc/bbs/car/index.html"`
	Page		int	   `default:1`
}

var conf = loadConfig()

func loadConfig() *DefaultConf {
	m := multiconfig.New()
	conf := new(DefaultConf)
	m.MustLoad(conf)

	return conf
}

func main() {

	fetchMultiPages(conf.Board, conf.Page)
}

func fetchSingle(url string) {
	resp := fetch(url)
	defer resp.Body.Close()

	doc, _ := goquery.NewDocumentFromReader(io.Reader(resp.Body))

	doc.Find("div.title").Each(func(i int, s *goquery.Selection) {
		a := s.Find("a")
		qHref, _ := a.Attr("href")
		title := strings.TrimSpace(s.Text())
		fmt.Println(title + "\t" + "https://www.ptt.cc" + qHref)
	})
}

func fetchPages(url string, ch chan int) {
	resp := fetch(url)
	defer resp.Body.Close()

	doc, _ := goquery.NewDocumentFromReader(io.Reader(resp.Body))
	href, _ := doc.Find("div.action-bar").Find("a.btn").Eq(3).Attr("href")
	pages, _ := strconv.Atoi(strings.Trim(strings.Split(strings.Split(href, "/")[3], ".")[0], "index"))
	ch <- pages + 1
}

func fetchMultiPages(board string, pre int) {
	fmt.Println("Do fetch multiple pages")
	ch := make(chan int)
	url := conf.BaseUrl + board + "/index.html"
	go fetchPages(url, ch)
	p := <-ch

	var pagesURL = make([]string, pre+1)
	for i := pre; i >= 0; i-- {
		pagesURL[i] = conf.BaseUrl + board + "/index" + strconv.Itoa(p-i) + ".html"
		fmt.Println("\n" + pagesURL[i] + "\n")
		fetchSingle(pagesURL[i])
	}
}

func fetch(url string) *http.Response {
	resp, err := http.Get(url)

	if err != nil {
		fmt.Println(err)
	}

	return resp
}
<commit_msg>Refactor<commit_after>package main

import (
	"fmt"
	"io"
	"net/http"
	"strconv"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"github.com/koding/multiconfig"
)

type DefaultConf struct {
	Board    	string `default:"car"`
	BaseUrl     string `default:"https://www.ptt.cc/bbs/"`
	CarBoardUrl string `default:"https://www.ptt.cc/bbs/car/index.html"`
	Page		int	   `default:1`
}

var conf = loadConfig()

func loadConfig() *DefaultConf {
	m := multiconfig.New()
	conf := new(DefaultConf)
	m.MustLoad(conf)

	return conf
}

func main() {

	fetchMultiPages(conf.Board, conf.Page)
}

func fetchSingle(url string) {
	resp := fetch(url)
	defer resp.Body.Close()

	doc, _ := goquery.NewDocumentFromReader(io.Reader(resp.Body))

	doc.Find("div.title").Each(func(i int, s *goquery.Selection) {
		link, _ := s.Find("a").Attr("href")
		title := strings.TrimSpace(s.Text())
		fmt.Println(title + "\t" + "https://www.ptt.cc" + link)
	})
}

func fetchPages(url string, ch chan int) {
	resp := fetch(url)
	defer resp.Body.Close()

	doc, _ := goquery.NewDocumentFromReader(io.Reader(resp.Body))
	href, _ := doc.Find("div.action-bar").Find("a.btn").Eq(3).Attr("href")
	pages, _ := strconv.Atoi(strings.Trim(strings.Split(strings.Split(href, "/")[3], ".")[0], "index"))
	ch <- pages + 1
}

func fetchMultiPages(board string, pre int) {
	fmt.Println("Do fetch multiple pages")
	ch := make(chan int)
	url := conf.BaseUrl + board + "/index.html"
	go fetchPages(url, ch)
	p := <-ch

	var pagesURL = make([]string, pre+1)
	for i := pre; i >= 0; i-- {
		pagesURL[i] = conf.BaseUrl + board + "/index" + strconv.Itoa(p-i) + ".html"
		fmt.Println("\n" + pagesURL[i] + "\n")
		fetchSingle(pagesURL[i])
	}
}

func fetch(url string) *http.Response {
	resp, err := http.Get(url)

	if err != nil {
		fmt.Println(err)
	}

	return resp
}
<|endoftext|>


<commit_before>package r

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"os"
	"os/exec"
	"strings"
)

var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
var rootDir string

type Session struct {
	Key     string
	Dir     string
	Output  string
	Command string
}

type RCall struct {
	Package   string
	Function  string
	Arguments string
}

// Installs the given packages on the server. packages is a space
// separated list with the packages.
func installPackages(packages string) error {
	cmd := exec.Command("R", "-q", "-e", "install.packages(c("+packages+"),  repos='http://cran.us.r-project.org'), dependencies=TRUE")

	var out bytes.Buffer
	cmd.Stdout = &out

	fmt.Println(out.String())
	err := cmd.Run()
	if err != nil {
		fmt.Println("Could not install packages:", out.String())
		return err
	}
	return nil
}

// Installs the package found at src. Should be a tgz ball.
func InstallPackageFromSource(src string) (string, error) {

	cmd := exec.Command("R", "CMD", "INSTALL", src)

	var out bytes.Buffer
	cmd.Stdout = &out

	err := cmd.Run()
	if err != nil {
		fmt.Println("Could not install local package:", out.String())
		return "", err
	}
	return out.String(), nil

}

func randSeq(n int) string {
	b := make([]rune, n)
	for i := range b {
		b[i] = letters[rand.Intn(len(letters))]
	}
	return string(b)
}

func newCall() (string, string, error) {

	key := "." + randSeq(5)

	wd := rootDir + "/" + key
	err := os.MkdirAll(wd, 0755)
	if err != nil {
		return "", "", err
	}

	err = os.Chdir(wd)

	if err != nil {
		return "", "", err
	}

	return key, wd, nil

}

// Executes function. Returns tmp key for use in Get
func Call(pkg, fun, args string) (*Session, error) {

	key, wd, err := newCall()
	if err != nil {
		return nil, err
	}

	// Replace argument names with real argument names using the
	// keys from previous calls. Also load data from previous
	// calls before running cmd

	argList := strings.Split(args, ",")
	loadArgs := []string{}

	if argList[0] != "" {
		finalArgs := []string{}

		for _, arg := range argList {

			// special case when we use a column vector as argument value
			// e,g. d = c(2,1,3,4) only d=c(2, is added. we need to
			// append the rest to get the full vector.
			if len(strings.Split(arg, "=")) == 1 {
				finalArgs[len(finalArgs)-1] = finalArgs[len(finalArgs)-1] + "," + arg
				continue
			}
			argName := strings.Split(arg, "=")[0]
			argVal := strings.Split(arg, "=")[1]

			if strings.HasPrefix(argVal, ".") {
				loadArgs = append(loadArgs, "load('"+rootDir+"/"+argVal+"/.RData');")
				argVal = strings.TrimPrefix(argVal, ".")
			}
			finalArgs = append(finalArgs, argName+"="+argVal)
		}
		args = strings.Join(finalArgs, ",")
	}

	varName := strings.TrimPrefix(key, ".")
	command := varName + "=" + pkg + "::" + fun + "(" + args + "); " + varName

	if len(loadArgs) > 0 {
		loadString := strings.Join(loadArgs, "")
		command = loadString + command
	}

	cmd := exec.Command("R", "--save", "-q", "-e", command)
	cmd.Dir = wd

	var stdout bytes.Buffer
	var stderr bytes.Buffer

	cmd.Stdout = &stdout
	cmd.Stderr = &stderr

	err = cmd.Run()

	var output string
	if err != nil {
		output = stderr.String()
		// remove kvik tmp key
		output = strings.Replace(output, varName+"=", "", -1)
	} else {
		output = stdout.String()
	}

	return &Session{key, wd, output, command}, err
}

// get results for a func call
func Get(key, format string) ([]byte, error) {

	dir := rootDir + "/" + key
	err := os.Chdir(dir)
	if err != nil {
		return nil, err
	}

	varName := strings.TrimPrefix(key, ".")

	extension := "." + format
	_, err = os.Stat("output" + extension)
	if err == nil {
		return ioutil.ReadFile(dir + "/output" + extension)
	}

	var command string
	if format == "csv" {
		command = "write.table(" + varName + ", sep=',', row.names=FALSE, file='output" + extension + "')"
	} else if format == "json" {
		command = "js=jsonlite::toJSON(" + varName + "); write(js, file='output" + extension + "')"
	} else if format == "pdf" {
		return ioutil.ReadFile(dir + "/Rplots.pdf")
	} else if format == "png" {
		cmd := exec.Command("pdftoppm", "-png", dir+"/Rplots.pdf", "plot")

		var out bytes.Buffer
		cmd.Stdout = &out

		err := cmd.Run()
		if err != nil {
			fmt.Println("Could not convert rplot to png:", out.String())
			return nil, err
		}

		return ioutil.ReadFile(dir + "/plot-1.png")
	} else {
		return nil, errors.New("Unknown format")
	}

	cmd := exec.Command("R", "--save", "-q", "-e", command)
	cmd.Dir = dir

	var out bytes.Buffer
	cmd.Stdout = &out

	err = cmd.Run()

	if err != nil {
		fmt.Println("ERROR", out.String())
		return nil, err
	}

	return ioutil.ReadFile(dir + "/output" + extension)

}

func ScriptHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Println("script handler")
	_, wd, err := newCall()
	if err != nil {
		fmt.Println("new call fail:", err)
		w.Write([]byte(err.Error()))
		return
	}

	file, _, err := r.FormFile("file")
	defer file.Close()

	if err != nil {
		fmt.Fprintln(w, err)
		return
	}

	fmt.Println("workdir", wd)

	fileLoc := wd + "/script.R"

	out, err := os.Create(fileLoc)
	if err != nil {
		fmt.Fprintf(w, "Failed to open the file for writing")
		return
	}
	defer out.Close()
	_, err = io.Copy(out, file)
	if err != nil {
		fmt.Fprintln(w, err)
		return
	}

	output, err := executeScript(wd)
	if err != nil {
		w.Write([]byte("ERROR, could not execute script. Output:\n" + output))
		return
	}

	cmd := exec.Command("zip", "-r", "-q", "output.zip", ".")
	cmd.Dir = wd

	var outbuffer bytes.Buffer
	cmd.Stdout = &outbuffer

	err = cmd.Run()

	if err != nil {
		fmt.Println("ERROR", outbuffer.String())
		w.Write([]byte("Could not zip output file"))
		return
	}

	http.ServeFile(w, r, "output.zip")

	return

}

func executeScript(wd string) (string, error) {

	filename := wd + "/script.R"

	cmd := exec.Command("R", "--save", "-f", filename)
	cmd.Dir = wd

	fmt.Println(cmd)

	var out bytes.Buffer
	cmd.Stdout = &out

	err := cmd.Run()

	if err != nil {

		errMessage := out.String()
		fmt.Println("ERROR", errMessage)
		return errMessage, err
	}

	return out.String(), nil
}

func InstalledPackages() ([]byte, error) {

	pkg := "utils"
	fun := "installed.packages"
	args := ""

	s, err := Call(pkg, fun, args)
	if err != nil {
		fmt.Println("Could not get installed packages")
		fmt.Println(s.Output)
		return nil, err
	}

	pkg = "base"
	fun = "as.data.frame"
	args = "x=" + s.Key + ",row.names=make.names(installed.packages(), unique=TRUE)"

	s, err = Call(pkg, fun, args)
	if err != nil {
		fmt.Println("Could not get installed packages")
		fmt.Println(s.Output)
		return nil, err
	}

	return Get(s.Key, "json")
}

type PackageInfo struct {
	Package               string
	LibPath               string
	Version               string
	Priority              string
	Depends               string
	Imports               string
	LinkingTo             string
	Suggests              string
	Enhances              string
	Licence               string
	License_is_FOSS       string
	Licence_restricts_use string
	OS_type               string
	MD5sum                string
	NeedsCompilation      string
	Built                 string
}
<commit_msg>load pkg before func call<commit_after>package r

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"os"
	"os/exec"
	"strings"
)

var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
var rootDir string

type Session struct {
	Key     string
	Dir     string
	Output  string
	Command string
}

type RCall struct {
	Package   string
	Function  string
	Arguments string
}

// Installs the given packages on the server. packages is a space
// separated list with the packages.
func installPackages(packages string) error {
	cmd := exec.Command("R", "-q", "-e", "install.packages(c("+packages+"),  repos='http://cran.us.r-project.org'), dependencies=TRUE")

	var out bytes.Buffer
	cmd.Stdout = &out

	fmt.Println(out.String())
	err := cmd.Run()
	if err != nil {
		fmt.Println("Could not install packages:", out.String())
		return err
	}
	return nil
}

// Installs the package found at src. Should be a tgz ball.
func InstallPackageFromSource(src string) (string, error) {

	cmd := exec.Command("R", "CMD", "INSTALL", src)

	var out bytes.Buffer
	cmd.Stdout = &out

	err := cmd.Run()
	if err != nil {
		fmt.Println("Could not install local package:", out.String())
		return "", err
	}
	return out.String(), nil

}

func randSeq(n int) string {
	b := make([]rune, n)
	for i := range b {
		b[i] = letters[rand.Intn(len(letters))]
	}
	return string(b)
}

func newCall() (string, string, error) {

	key := "." + randSeq(5)

	wd := rootDir + "/" + key
	err := os.MkdirAll(wd, 0755)
	if err != nil {
		return "", "", err
	}

	err = os.Chdir(wd)

	if err != nil {
		return "", "", err
	}

	return key, wd, nil

}

// Executes function. Returns tmp key for use in Get
func Call(pkg, fun, args string) (*Session, error) {

	key, wd, err := newCall()
	if err != nil {
		return nil, err
	}

	// Replace argument names with real argument names using the
	// keys from previous calls. Also load data from previous
	// calls before running cmd

	argList := strings.Split(args, ",")
	loadArgs := []string{}

	if argList[0] != "" {
		finalArgs := []string{}

		for _, arg := range argList {

			// special case when we use a column vector as argument value
			// e,g. d = c(2,1,3,4) only d=c(2, is added. we need to
			// append the rest to get the full vector.
			if len(strings.Split(arg, "=")) == 1 {
				finalArgs[len(finalArgs)-1] = finalArgs[len(finalArgs)-1] + "," + arg
				continue
			}
			argName := strings.Split(arg, "=")[0]
			argVal := strings.Split(arg, "=")[1]

			if strings.HasPrefix(argVal, ".") {
				loadArgs = append(loadArgs, "load('"+rootDir+"/"+argVal+"/.RData');")
				argVal = strings.TrimPrefix(argVal, ".")
			}
			finalArgs = append(finalArgs, argName+"="+argVal)
		}
		args = strings.Join(finalArgs, ",")
	}

	varName := strings.TrimPrefix(key, ".")
	command := varName + "=" + pkg + "::" + fun + "(" + args + "); " + varName

	if len(loadArgs) > 0 {
		loadString := strings.Join(loadArgs, "")
		command = loadString + command
	}

	// load desired package before calling function (makes sure
	// that we load packages it depends on)
	command = "library(" + pkg + ");" + command

	cmd := exec.Command("R", "--save", "-q", "-e", command)
	cmd.Dir = wd

	var stdout bytes.Buffer
	var stderr bytes.Buffer

	cmd.Stdout = &stdout
	cmd.Stderr = &stderr

	err = cmd.Run()

	var output string
	if err != nil {
		output = stderr.String()
		// remove kvik tmp key
		output = strings.Replace(output, varName+"=", "", -1)
	} else {
		output = stdout.String()
	}

	return &Session{key, wd, output, command}, err
}

// get results for a func call
func Get(key, format string) ([]byte, error) {

	dir := rootDir + "/" + key
	err := os.Chdir(dir)
	if err != nil {
		return nil, err
	}

	varName := strings.TrimPrefix(key, ".")

	extension := "." + format
	_, err = os.Stat("output" + extension)
	if err == nil {
		return ioutil.ReadFile(dir + "/output" + extension)
	}

	var command string
	if format == "csv" {
		command = "write.table(" + varName + ", sep=',', row.names=FALSE, file='output" + extension + "')"
	} else if format == "json" {
		command = "js=jsonlite::toJSON(" + varName + "); write(js, file='output" + extension + "')"
	} else if format == "pdf" {
		return ioutil.ReadFile(dir + "/Rplots.pdf")
	} else if format == "png" {
		cmd := exec.Command("pdftoppm", "-png", dir+"/Rplots.pdf", "plot")

		var stderr bytes.Buffer
		cmd.Stderr = &stderr

		err := cmd.Run()
		if err != nil {
			fmt.Println("Could not convert rplot to png:", stderr.String())
			return nil, err
		}

		return ioutil.ReadFile(dir + "/plot-1.png")
	} else {
		return nil, errors.New("Unknown format")
	}

	fmt.Println("shit", command)
	cmd := exec.Command("R", "--save", "-q", "-e", command)
	cmd.Dir = dir

	var stderr bytes.Buffer
	cmd.Stderr = &stderr

	err = cmd.Run()

	if err != nil {
		return nil, errors.New(stderr.String())
	}

	return ioutil.ReadFile(dir + "/output" + extension)

}

func ScriptHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Println("script handler")
	_, wd, err := newCall()
	if err != nil {
		fmt.Println("new call fail:", err)
		w.Write([]byte(err.Error()))
		return
	}

	file, _, err := r.FormFile("file")
	defer file.Close()

	if err != nil {
		fmt.Fprintln(w, err)
		return
	}

	fmt.Println("workdir", wd)

	fileLoc := wd + "/script.R"

	out, err := os.Create(fileLoc)
	if err != nil {
		fmt.Fprintf(w, "Failed to open the file for writing")
		return
	}
	defer out.Close()
	_, err = io.Copy(out, file)
	if err != nil {
		fmt.Fprintln(w, err)
		return
	}

	output, err := executeScript(wd)
	if err != nil {
		w.Write([]byte("ERROR, could not execute script. Output:\n" + output))
		return
	}

	cmd := exec.Command("zip", "-r", "-q", "output.zip", ".")
	cmd.Dir = wd

	var outbuffer bytes.Buffer
	cmd.Stdout = &outbuffer

	err = cmd.Run()

	if err != nil {
		fmt.Println("ERROR", outbuffer.String())
		w.Write([]byte("Could not zip output file"))
		return
	}

	http.ServeFile(w, r, "output.zip")

	return

}

func executeScript(wd string) (string, error) {

	filename := wd + "/script.R"

	cmd := exec.Command("R", "--save", "-f", filename)
	cmd.Dir = wd

	fmt.Println(cmd)

	var out bytes.Buffer
	cmd.Stdout = &out

	err := cmd.Run()

	if err != nil {

		errMessage := out.String()
		fmt.Println("ERROR", errMessage)
		return errMessage, err
	}

	return out.String(), nil
}

func InstalledPackages() ([]byte, error) {

	pkg := "utils"
	fun := "installed.packages"
	args := ""

	s, err := Call(pkg, fun, args)
	if err != nil {
		fmt.Println("Could not get installed packages")
		fmt.Println(s.Output)
		return nil, err
	}

	pkg = "base"
	fun = "as.data.frame"
	args = "x=" + s.Key + ",row.names=make.names(installed.packages(), unique=TRUE)"

	s, err = Call(pkg, fun, args)
	if err != nil {
		fmt.Println("Could not get installed packages")
		fmt.Println(s.Output, err)
		return nil, err
	}

	return Get(s.Key, "json")
}

type PackageInfo struct {
	Package               string
	LibPath               string
	Version               string
	Priority              string
	Depends               string
	Imports               string
	LinkingTo             string
	Suggests              string
	Enhances              string
	Licence               string
	License_is_FOSS       string
	Licence_restricts_use string
	OS_type               string
	MD5sum                string
	NeedsCompilation      string
	Built                 string
}
<|endoftext|>


<commit_before>package junos

// rpcCommand lists the commands that will be called.
var rpcCommand = map[string]string{
	"command":                          "<rpc><command format=\"text\">%s</command></rpc>",
	"command-xml":                      "<rpc><command format=\"xml\">%s</command></rpc>",
    "commit": "<rpc><commit-configuration/></rpc>",
	"get-rescue-information":           "<rpc><get-rescue-information><format>text</format></get-rescue-information></rpc>",
	"get-rollback-information":         "<rpc><get-rollback-information><rollback>%d</rollback><format>text</format></get-rollback-information></rpc>",
	"get-rollback-information-compare": "<rpc><get-rollback-information><rollback>0</rollback><compare>%d</compare><format>text</format></get-rollback-information></rpc>",
	"lock":                             "<rpc><lock><target><candidate/></target></lock></rpc>",
	"rollback-config": "<rpc><load-configuration rollback=\"%d\"/></rpc>",
    "unlock":                           "<rpc><unlock><target><candidate/></target></unlock></rpc>",
}
<commit_msg>Added additional RPC commands<commit_after>package junos

// rpcCommand lists the commands that will be called.
var rpcCommand = map[string]string{
	"command":                          "<rpc><command format=\"text\">%s</command></rpc>",
	"command-xml":                      "<rpc><command format=\"xml\">%s</command></rpc>",
	"commit":                           "<rpc><commit-configuration/></rpc>",
	"get-rescue-information":           "<rpc><get-rescue-information><format>text</format></get-rescue-information></rpc>",
	"get-rollback-information":         "<rpc><get-rollback-information><rollback>%d</rollback><format>text</format></get-rollback-information></rpc>",
	"get-rollback-information-compare": "<rpc><get-rollback-information><rollback>0</rollback><compare>%d</compare><format>text</format></get-rollback-information></rpc>",
	"lock":            "<rpc><lock><target><candidate/></target></lock></rpc>",
	"rescue-config":   "<rpc><load-configuration rescue=\"rescue\"/></rpc>",
	"rollback-config": "<rpc><load-configuration rollback=\"%d\"/></rpc>",
	"unlock":          "<rpc><unlock><target><candidate/></target></unlock></rpc>",
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/c-14/grue/config"
	"github.com/mmcdole/gofeed"
	"math"
	"time"
)

type FeedParser struct {
	parser   *gofeed.Parser
	sem      chan int
	messages chan *Email
}

type RSSFeed struct {
	config      config.AccountConfig
	LastFetched int64               `json:",omitempty"`
	LastQueried int64               `json:",omitempty"`
	NextQuery   int64               `json:",omitempty"`
	Tries       int                 `json:",omitempty"`
	GUIDList    map[string]struct{} `json:",omitempty"`
}

func hasNewerDate(item *gofeed.Item, lastFetched int64) (bool, time.Time) {
	if item.PublishedParsed != nil && item.PublishedParsed.Unix() > lastFetched {
		return true, *item.PublishedParsed
	} else if item.UpdatedParsed != nil && item.UpdatedParsed.Unix() > lastFetched {
		return true, *item.UpdatedParsed
	} else if date, exists := item.Extensions["dc"]["date"]; exists {
		dateParsed, err := time.Parse(time.RFC3339, date[0].Value)
		if err != nil {
			fmt.Printf("Can't parse (%v) as dc:date for (%v)\n", date, item.Link)
			return false, time.Now()
		}
		if dateParsed.Unix() > lastFetched {
			return true, dateParsed
		}
	}
	return false, time.Now()
}

func fetchFeed(fp FeedParser, feedName string, account *RSSFeed, config *config.GrueConfig) {
	// if account.UserAgent != nil {
	// 	feed.SetUserAgent(*account.UserAgent)
	// }
	now := time.Now()
	if account.NextQuery > now.Unix() {
		return
	}
	feed, err := fp.parser.ParseURL(account.config.URI)
	account.LastQueried = now.Unix()
	if err != nil {
		if account.Tries > 0 {
			account.NextQuery = now.Add(time.Duration(math.Exp2(float64(account.Tries+4))) * time.Minute).Unix()
		}
		account.Tries++
		fmt.Printf("Caught error when parsing %s: %s\n", account.config.URI, err)
		return
	}
	account.NextQuery = 0
	account.Tries = 0
	guids := account.GUIDList
	if float64(len(guids)) > 1.2*float64(len(feed.Items)) {
		account.GUIDList = make(map[string]struct{})
	}
	for _, item := range feed.Items {
		if newer, date := hasNewerDate(item, account.LastFetched); newer {
			e := createEmail(feedName, feed.Title, item, date, account.config, config)
			err = e.Send(fp.messages)
		} else {
			_, exists := guids[item.GUID]
			if !exists {
				e := createEmail(feedName, feed.Title, item, date, account.config, config)
				err = e.Send(fp.messages)
			}
			if err == nil {
				account.GUIDList[item.GUID] = struct{}{}
			}
		}
	}
	if err == nil {
		account.LastFetched = account.LastQueried
	}
	<-fp.sem
}

func fetchFeeds(ret chan error, conf *config.GrueConfig, init bool) {
	defer close(ret)
	hist, err := ReadHistory()
	if err != nil {
		ret <- err
		return
	}
	ch := make(chan *Email)
	defer close(ch)
	if !init {
		s, err := setupDialer(conf)
		if err != nil {
			ret <- err
			return
		}
		go startDialing(s, ch, ret)
	} else {
		go func() {
			for m := range ch {
				m.ret <- nil
			}
		}()
	}

	fp := FeedParser{parser: gofeed.NewParser(), sem: make(chan int, 10), messages: ch}
	for name, accountConfig := range conf.Accounts {
		fp.sem <- 1
		account, exist := hist.Feeds[name]
		account.config = accountConfig
		if !exist {
			account.GUIDList = make(map[string]struct{})
		}
		go fetchFeed(fp, name, &account, conf)
		hist.Feeds[name] = account
	}
	ret <- hist.Write()
}
<commit_msg>Block fetcFeeds until all feeds fetched.<commit_after>package main

import (
	"fmt"
	"github.com/c-14/grue/config"
	"github.com/mmcdole/gofeed"
	"math"
	"time"
)

type FeedParser struct {
	parser   *gofeed.Parser
	messages chan *Email
	sem      chan int
	finished chan int
}

type RSSFeed struct {
	config      config.AccountConfig
	LastFetched int64               `json:",omitempty"`
	LastQueried int64               `json:",omitempty"`
	NextQuery   int64               `json:",omitempty"`
	Tries       int                 `json:",omitempty"`
	GUIDList    map[string]struct{} `json:",omitempty"`
}

func hasNewerDate(item *gofeed.Item, lastFetched int64) (bool, time.Time) {
	if item.PublishedParsed != nil && item.PublishedParsed.Unix() > lastFetched {
		return true, *item.PublishedParsed
	} else if item.UpdatedParsed != nil && item.UpdatedParsed.Unix() > lastFetched {
		return true, *item.UpdatedParsed
	} else if date, exists := item.Extensions["dc"]["date"]; exists {
		dateParsed, err := time.Parse(time.RFC3339, date[0].Value)
		if err != nil {
			fmt.Printf("Can't parse (%v) as dc:date for (%v)\n", date, item.Link)
			return false, time.Now()
		}
		if dateParsed.Unix() > lastFetched {
			return true, dateParsed
		}
	}
	return false, time.Now()
}

func fetchFeed(fp FeedParser, feedName string, account *RSSFeed, config *config.GrueConfig) {
	// if account.UserAgent != nil {
	// 	feed.SetUserAgent(*account.UserAgent)
	// }
	now := time.Now()
	if account.NextQuery > now.Unix() {
		return
	}
	feed, err := fp.parser.ParseURL(account.config.URI)
	account.LastQueried = now.Unix()
	if err != nil {
		if account.Tries > 0 {
			account.NextQuery = now.Add(time.Duration(math.Exp2(float64(account.Tries+4))) * time.Minute).Unix()
		}
		account.Tries++
		fmt.Printf("Caught error when parsing %s: %s\n", account.config.URI, err)
		return
	}
	account.NextQuery = 0
	account.Tries = 0
	guids := account.GUIDList
	if float64(len(guids)) > 1.2*float64(len(feed.Items)) {
		account.GUIDList = make(map[string]struct{})
	}
	for _, item := range feed.Items {
		if newer, date := hasNewerDate(item, account.LastFetched); newer {
			e := createEmail(feedName, feed.Title, item, date, account.config, config)
			err = e.Send(fp.messages)
		} else {
			_, exists := guids[item.GUID]
			if !exists {
				e := createEmail(feedName, feed.Title, item, date, account.config, config)
				err = e.Send(fp.messages)
			}
			if err == nil {
				account.GUIDList[item.GUID] = struct{}{}
			}
		}
	}
	if err == nil {
		account.LastFetched = account.LastQueried
	}

	<-fp.sem
	fp.finished <- 1
}

func fetchFeeds(ret chan error, conf *config.GrueConfig, init bool) {
	defer close(ret)
	hist, err := ReadHistory()
	if err != nil {
		ret <- err
		return
	}
	ch := make(chan *Email)
	defer close(ch)
	if !init {
		s, err := setupDialer(conf)
		if err != nil {
			ret <- err
			return
		}
		go startDialing(s, ch, ret)
	} else {
		go func() {
			for m := range ch {
				m.ret <- nil
			}
		}()
	}

	fp := FeedParser{parser: gofeed.NewParser(), messages: ch, sem: make(chan int, 10), finished: make(chan int)}
	go func() {
		for name, accountConfig := range conf.Accounts {
			fp.sem <- 1
			account, exist := hist.Feeds[name]
			account.config = accountConfig
			if !exist {
				account.GUIDList = make(map[string]struct{})
			}
			go fetchFeed(fp, name, &account, conf)
			hist.Feeds[name] = account
		}
	}()
	for range conf.Accounts {
		<-fp.finished
	}
	ret <- hist.Write()
}
<|endoftext|>