<commit_before>package date

import (
	"database/sql/driver"
	"errors"
	"time"
)

const (
	MillisecondsInSecond = 1000
	MicrosecondsInSecond = 1000 * MillisecondsInSecond
	NanosecondsInSecond  = 1000 * MicrosecondsInSecond
)

// Date is a plain date, without time or timezone info (use time.Time for those!)
type Date struct {
	Year  int
	Month time.Month
	Day   int

	// NOTE: time.Time does not preserve timezone when gob encoded
	// location must be private to support Gob encoding
	location *time.Location
}

func At(y int, m time.Month, d int, l *time.Location) Date {
	return Date{y, m, d, l}
}

// Create a Date from Time.Now()
func Today() Date {
	t := time.Now()
	y, m, d := t.Date()
	return Date{y, m, d, t.Location()}
}

func Yesterday() Date {
	t := time.Now().AddDate(0, 0, -1)
	y, m, d := t.Date()
	return Date{y, m, d, t.Location()}
}

// Create a Date from a time.Time object
func From(t time.Time) Date {
	y, m, d := t.Date()
	return Date{y, m, d, t.Location()}
}

func (d Date) PrevDay() Date {
	if d.Day > 1 {
		return Date{d.Year, d.Month, d.Day - 1, d.location}
	} else if d.Month > 1 {
		return Date{d.Year, d.Month - 1, DaysInMonth(d.Month-1, d.Year), d.location}
	} else {
		return Date{d.Year - 1, time.December, 31, d.location}
	}
}

func (d Date) NextDay() Date {
	if d.Day < DaysInMonth(d.Month, d.Year) {
		return Date{d.Year, d.Month, d.Day + 1, d.location}
	} else if d.Month < time.December {
		return Date{d.Year, d.Month + 1, 1, d.location}
	} else {
		return Date{d.Year + 1, time.January, 1, d.location}
	}
}

// MostRecent returns today if today is the given weekday, otherwise it returns
// the date of the last day for that weekday.
//
// It is useful to get the beginning of the week:
//
//    today.MostRecent(time.Sunday) // in the US
//    today.MostRecent(time.Monday) // officially
//
func (d Date) MostRecent(weekday time.Weekday) Date {
	t := d.BeginningOfDay()
	today := t.Weekday()
	if today == weekday {
		return d
	} else if today > weekday {
		return From(t.AddDate(0, 0, int(weekday-today)))
	} else {
		return From(t.AddDate(0, 0, int(weekday-today)-7))
	}
}

func (d Date) Before(other Date) bool {
	return (d.Year < other.Year ||
		(d.Year == other.Year &&
			(d.Month < other.Month ||
				(d.Month == other.Month && d.Day < other.Day))))
}

func (d Date) After(other Date) bool {
	return (d.Year > other.Year ||
		(d.Year == other.Year &&
			(d.Month > other.Month ||
				(d.Month == other.Month && d.Day > other.Day))))
}

func (d Date) AtLeast(other Date) bool {
	return !other.After(d)
}

func (d Date) AtMost(other Date) bool {
	return !other.Before(d)
}

func (d Date) Weekday() time.Weekday {
	return d.BeginningOfDay().Weekday()
}

func (d Date) BeginningOfDay() time.Time {
	return time.Date(d.Year, d.Month, d.Day, 0, 0, 0, 0, d.location)
}

func (d Date) BeginningOfDayIn(timezone *time.Location) time.Time {
	return time.Date(d.Year, d.Month, d.Day, 0, 0, 0, 0, timezone)
}

func (d Date) EndOfDay() time.Time {
	return time.Date(d.Year, d.Month, d.Day, 23, 59, 59, NanosecondsInSecond-1, d.location)
}

func (d Date) EndOfDayIn(timezone *time.Location) time.Time {
	return time.Date(d.Year, d.Month, d.Day, 23, 59, 59, NanosecondsInSecond-1, timezone)
}

func (d Date) String() string {
	return d.BeginningOfDayIn(time.UTC).Format("2006-01-02")
}

// Implements sql.Scanner interface
func (d Date) Scan(src interface{}) error {
	t, ok := src.(time.Time)
	if !ok {
		return errors.New("unstable/date: scan value was not a Time")
	}

	d = From(t)
	return nil
}

// Implements sql.driver.Valuer interface
func (d Date) Value() (driver.Value, error) {
	return d.BeginningOfDayIn(time.UTC), nil
}

// Implements json.Marshaler interface
func (d Date) MarshalJSON() ([]byte, error) {
	return []byte(d.BeginningOfDayIn(time.UTC).Format(`"2006-01-02"`)), nil
}

// Implements json.Unmarshaler interface
func (d *Date) UnmarshalJSON(bytes []byte) error {
	t, err := time.Parse(`"2006-01-02"`, string(bytes))
	if err != nil {
		return err
	}

	*d = From(t)
	return nil
}

// TODO: Implment gob.GobEncoder and gob.GobDecoder to preserve timezone
// func (d Date) GobEncode() ([]byte, error) {}
// func (d *Date) GobDecode(bytes []byte) error {}

func IsLeapYear(year int) bool {
	return year%4 == 0 && (year%100 != 0 || year%400 == 0)
}

var DaysInNonLeapMonth = [12]int32{31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}

func DaysInMonth(m time.Month, year int) int {
	if m == time.February && IsLeapYear(year) {
		return 29
	} else {
		return int(DaysInNonLeapMonth[m-1])
	}
}
<commit_msg>date: Added TodayInTimezone, AddDays, MostSoon, and (string) Parse helper functions.<commit_after>package date

import (
	"database/sql/driver"
	"errors"
	"time"
)

const (
	MillisecondsInSecond = 1000
	MicrosecondsInSecond = 1000 * MillisecondsInSecond
	NanosecondsInSecond  = 1000 * MicrosecondsInSecond
)

const RFC3339 = "2006-01-02"

// Parse string into desired date format i.e RFC3339
func Parse(format string, source string) (Date, error) {
	t, err := time.Parse(format, source)
	return From(t), err
}

// Date is a plain date, without time or timezone info (use time.Time for those!)
type Date struct {
	Year  int
	Month time.Month
	Day   int

	// NOTE: time.Time does not preserve timezone when gob encoded
	// location must be private to support Gob encoding
	location *time.Location
}

func At(y int, m time.Month, d int, l *time.Location) Date {
	return Date{y, m, d, l}
}

// Create a Date from Time.Now()
func Today() Date {
	t := time.Now()
	y, m, d := t.Date()
	return Date{y, m, d, t.Location()}
}

func TodayInTimezone(location *time.Location) Date {
	t := time.Now().In(location)
	y, m, d := t.Date()
	return Date{y, m, d, location}
}

func Yesterday() Date {
	t := time.Now().AddDate(0, 0, -1)
	y, m, d := t.Date()
	return Date{y, m, d, t.Location()}
}

// Create a Date from a time.Time object
func From(t time.Time) Date {
	y, m, d := t.Date()
	return Date{y, m, d, t.Location()}
}

func (d Date) AddDays(num int) Date {
	t := time.Date(d.Year, d.Month, d.Day, 0, 0, 0, 0, d.location).AddDate(0, 0, num)
	year, month, day := t.Date()
	return Date{year, month, day, d.location}
}

func (d Date) PrevDay() Date {
	if d.Day > 1 {
		return Date{d.Year, d.Month, d.Day - 1, d.location}
	} else if d.Month > 1 {
		return Date{d.Year, d.Month - 1, DaysInMonth(d.Month-1, d.Year), d.location}
	} else {
		return Date{d.Year - 1, time.December, 31, d.location}
	}
}

func (d Date) NextDay() Date {
	if d.Day < DaysInMonth(d.Month, d.Year) {
		return Date{d.Year, d.Month, d.Day + 1, d.location}
	} else if d.Month < time.December {
		return Date{d.Year, d.Month + 1, 1, d.location}
	} else {
		return Date{d.Year + 1, time.January, 1, d.location}
	}
}

// MostRecent returns today if today is the given weekday, otherwise it returns
// the date of the last day for that weekday.
//
// It is useful to get the beginning of the week:
//
//    today.MostRecent(time.Sunday) // in the US
//    today.MostRecent(time.Monday) // officially
//
func (d Date) MostRecent(weekday time.Weekday) Date {
	t := d.BeginningOfDay()
	today := t.Weekday()
	if today == weekday {
		return d
	} else if today > weekday {
		return From(t.AddDate(0, 0, int(weekday-today)))
	} else {
		return From(t.AddDate(0, 0, int(weekday-today)-7))
	}
}

// MostSoon returns today if today is the given weekday, otherwise it returns
// the date of the upcoming day for that weekday.
func (d Date) MostSoon(weekday time.Weekday) Date {
	if d.Weekday() == weekday {
		return d
	} else if d.Weekday() < weekday {
		return d.AddDays(int(weekday - d.Weekday()))
	} else {
		return d.AddDays(7 - int(d.Weekday()-weekday))
	}
}

func (d Date) Before(other Date) bool {
	return (d.Year < other.Year ||
		(d.Year == other.Year &&
			(d.Month < other.Month ||
				(d.Month == other.Month && d.Day < other.Day))))
}

func (d Date) Equal(other Date) bool {
	return d == other
}

func (d Date) After(other Date) bool {
	return (d.Year > other.Year ||
		(d.Year == other.Year &&
			(d.Month > other.Month ||
				(d.Month == other.Month && d.Day > other.Day))))
}

func (d Date) AtLeast(other Date) bool {
	return !other.After(d)
}

func (d Date) AtMost(other Date) bool {
	return !other.Before(d)
}

func (d Date) Weekday() time.Weekday {
	return d.BeginningOfDay().Weekday()
}

func (d Date) BeginningOfDay() time.Time {
	return time.Date(d.Year, d.Month, d.Day, 0, 0, 0, 0, d.location)
}

func (d Date) BeginningOfDayIn(timezone *time.Location) time.Time {
	return time.Date(d.Year, d.Month, d.Day, 0, 0, 0, 0, timezone)
}

func (d Date) EndOfDay() time.Time {
	return time.Date(d.Year, d.Month, d.Day, 23, 59, 59, NanosecondsInSecond-1, d.location)
}

func (d Date) EndOfDayIn(timezone *time.Location) time.Time {
	return time.Date(d.Year, d.Month, d.Day, 23, 59, 59, NanosecondsInSecond-1, timezone)
}

func (d Date) String() string {
	return d.BeginningOfDayIn(time.UTC).Format("2006-01-02")
}

// Implements sql.Scanner interface
func (d Date) Scan(src interface{}) error {
	t, ok := src.(time.Time)
	if !ok {
		return errors.New("unstable/date: scan value was not a Time")
	}

	d = From(t)
	return nil
}

// Implements sql.driver.Valuer interface
func (d Date) Value() (driver.Value, error) {
	return d.BeginningOfDayIn(time.UTC), nil
}

// Implements json.Marshaler interface
func (d Date) MarshalJSON() ([]byte, error) {
	return []byte(d.BeginningOfDayIn(time.UTC).Format(`"2006-01-02"`)), nil
}

// Implements json.Unmarshaler interface
func (d *Date) UnmarshalJSON(bytes []byte) error {
	t, err := time.Parse(`"2006-01-02"`, string(bytes))
	if err != nil {
		return err
	}

	*d = From(t)
	return nil
}

// TODO: Implment gob.GobEncoder and gob.GobDecoder to preserve timezone
// func (d Date) GobEncode() ([]byte, error) {}
// func (d *Date) GobDecode(bytes []byte) error {}

func IsLeapYear(year int) bool {
	return year%4 == 0 && (year%100 != 0 || year%400 == 0)
}

var DaysInNonLeapMonth = [12]int32{31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}

func DaysInMonth(m time.Month, year int) int {
	if m == time.February && IsLeapYear(year) {
		return 29
	} else {
		return int(DaysInNonLeapMonth[m-1])
	}
}
<|endoftext|>


<commit_before>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package frontend

import (
	"bytes"
	"context"

	"github.com/google/safehtml"
	"github.com/google/safehtml/template"
	"github.com/google/safehtml/uncheckedconversions"
	"github.com/microcosm-cc/bluemonday"
	"github.com/yuin/goldmark"
	emoji "github.com/yuin/goldmark-emoji"
	"github.com/yuin/goldmark/extension"
	"github.com/yuin/goldmark/parser"
	"github.com/yuin/goldmark/renderer"
	goldmarkHtml "github.com/yuin/goldmark/renderer/html"
	gmtext "github.com/yuin/goldmark/text"
	"github.com/yuin/goldmark/util"
	"golang.org/x/pkgsite/internal"
	"golang.org/x/pkgsite/internal/derrors"
	"golang.org/x/pkgsite/internal/source"
)

// Heading holds data about a heading and nested headings within a readme.
// This data is used in the sidebar template to render the readme outline.
type Heading struct {
	// Level is the original level of the heading.
	Level int
	// Text is the content from the readme contained within a heading.
	Text string
	// ID corresponds to the ID attribute for a heading element
	// and is also used in an href to the corresponding section
	// within the readme outline. All ids are prefixed with readme-
	// to avoid name collisions.
	ID string
	// Children are nested headings.
	Children []*Heading
	// parent is the heading this heading is nested within. Nil for top
	// level headings.
	parent *Heading
}

// Readme holds the result of processing a REAME file.
type Readme struct {
	HTML    safehtml.HTML // rendered HTML
	Outline []*Heading    // document headings
	Links   []link        // links from the "Links" section
}

// ProcessReadme processes the README of unit u, if it has one.
// Processing includes rendering and sanitizing the HTML or Markdown,
// and extracting headings and links.
//
// Headings are prefixed with "readme-" and heading levels are adjusted to start
// at h3 in order to nest them properly within the rest of the page. The
// readme's original styling is preserved in the html by giving headings a css
// class styled identical to their original heading level.
//
//  The extracted links are for display outside of the readme contents.
//
// This function is exported for use by external tools.
func ProcessReadme(ctx context.Context, u *internal.Unit) (_ *Readme, err error) {
	defer derrors.WrapAndReport(&err, "ProcessReadme(%q, %q, %q)", u.Path, u.ModulePath, u.Version)
	return processReadme(ctx, u.Readme, u.SourceInfo)
}

func processReadme(ctx context.Context, readme *internal.Readme, sourceInfo *source.Info) (_ *Readme, err error) {
	if readme == nil || readme.Contents == "" {
		return &Readme{}, nil
	}
	if !isMarkdown(readme.Filepath) {
		t := template.Must(template.New("").Parse(`<pre class="readme">{{.}}</pre>`))
		h, err := t.ExecuteToHTML(readme.Contents)
		if err != nil {
			return nil, err
		}
		return &Readme{HTML: h}, nil
	}

	// Sets priority value so that we always use our custom transformer
	// instead of the default ones. The default values are in:
	// https://github.com/yuin/goldmark/blob/7b90f04af43131db79ec320be0bd4744079b346f/parser/parser.go#L567
	const astTransformerPriority = 10000
	el := &extractLinks{ctx: ctx}
	et := &extractTOC{ctx: ctx, removeTitle: true}
	gdMarkdown := goldmark.New(
		goldmark.WithParserOptions(
			// WithHeadingAttribute allows us to include other attributes in
			// heading tags. This is useful for our aria-level implementation of
			// increasing heading rankings.
			parser.WithHeadingAttribute(),
			// Generates an id in every heading tag. This is used in github in
			// order to generate a link with a hash that a user would scroll to
			//
goldmark
 => github.com/yuin/goldmark#goldmark
			parser.WithAutoHeadingID(),
			// Include custom ASTTransformer using the readme and module info to
			// use translateRelativeLink and translateHTML to modify the AST
			// before it is rendered.
			parser.WithASTTransformers(
				util.Prioritized(&astTransformer{
					info:   sourceInfo,
					readme: readme,
				}, astTransformerPriority),
				// Extract links after we have transformed the URLs.
				util.Prioritized(el, astTransformerPriority+1),
				util.Prioritized(et, astTransformerPriority+1),
			),
		),
		// These extensions lets users write HTML code in the README. This is
		// fine since we process the contents using bluemonday after.
		goldmark.WithRendererOptions(goldmarkHtml.WithUnsafe(), goldmarkHtml.WithXHTML()),
		goldmark.WithExtensions(
			extension.GFM, // Support Github Flavored Markdown.
			emoji.Emoji,   // Support Github markdown emoji markup.
		),
	)
	gdMarkdown.Renderer().AddOptions(
		renderer.WithNodeRenderers(
			util.Prioritized(newHTMLRenderer(sourceInfo, readme), 100),
		),
	)
	contents := []byte(readme.Contents)
	gdParser := gdMarkdown.Parser()
	reader := gmtext.NewReader(contents)
	pctx := parser.NewContext(parser.WithIDs(newIDs()))
	doc := gdParser.Parse(reader, parser.WithContext(pctx))
	gdRenderer := gdMarkdown.Renderer()

	var b bytes.Buffer
	if err := gdRenderer.Render(&b, contents, doc); err != nil {
		return &Readme{}, nil
	}
	return &Readme{
		HTML:    sanitizeHTML(&b),
		Outline: et.Headings,
		Links:   el.links,
	}, nil
}

// sanitizeHTML sanitizes HTML from a bytes.Buffer so that it is safe.
func sanitizeHTML(b *bytes.Buffer) safehtml.HTML {
	p := bluemonday.UGCPolicy()

	p.AllowAttrs("width", "align").OnElements("img")
	p.AllowAttrs("width", "align").OnElements("div")
	p.AllowAttrs("width", "align").OnElements("p")
	// Allow accessible headings (i.e
).
	p.AllowAttrs("width", "align", "role", "aria-level").OnElements("div")
	for _, h := range []string{"h1", "h2", "h3", "h4", "h5", "h6"} {
		// Needed to preserve github styles heading font-sizes
		p.AllowAttrs("class").OnElements(h)
	}

	s := string(p.SanitizeBytes(b.Bytes()))
	return uncheckedconversions.HTMLFromStringKnownToSatisfyTypeContract(s)
}
<commit_msg>internal/frontend: handle panics from goldmark renderer<commit_after>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package frontend

import (
	"bytes"
	"context"

	"github.com/google/safehtml"
	"github.com/google/safehtml/template"
	"github.com/google/safehtml/uncheckedconversions"
	"github.com/microcosm-cc/bluemonday"
	"github.com/yuin/goldmark"
	emoji "github.com/yuin/goldmark-emoji"
	"github.com/yuin/goldmark/extension"
	"github.com/yuin/goldmark/parser"
	"github.com/yuin/goldmark/renderer"
	goldmarkHtml "github.com/yuin/goldmark/renderer/html"
	gmtext "github.com/yuin/goldmark/text"
	"github.com/yuin/goldmark/util"
	"golang.org/x/pkgsite/internal"
	"golang.org/x/pkgsite/internal/derrors"
	"golang.org/x/pkgsite/internal/log"
	"golang.org/x/pkgsite/internal/source"
)

// Heading holds data about a heading and nested headings within a readme.
// This data is used in the sidebar template to render the readme outline.
type Heading struct {
	// Level is the original level of the heading.
	Level int
	// Text is the content from the readme contained within a heading.
	Text string
	// ID corresponds to the ID attribute for a heading element
	// and is also used in an href to the corresponding section
	// within the readme outline. All ids are prefixed with readme-
	// to avoid name collisions.
	ID string
	// Children are nested headings.
	Children []*Heading
	// parent is the heading this heading is nested within. Nil for top
	// level headings.
	parent *Heading
}

// Readme holds the result of processing a REAME file.
type Readme struct {
	HTML    safehtml.HTML // rendered HTML
	Outline []*Heading    // document headings
	Links   []link        // links from the "Links" section
}

// ProcessReadme processes the README of unit u, if it has one.
// Processing includes rendering and sanitizing the HTML or Markdown,
// and extracting headings and links.
//
// Headings are prefixed with "readme-" and heading levels are adjusted to start
// at h3 in order to nest them properly within the rest of the page. The
// readme's original styling is preserved in the html by giving headings a css
// class styled identical to their original heading level.
//
//  The extracted links are for display outside of the readme contents.
//
// This function is exported for use by external tools.
func ProcessReadme(ctx context.Context, u *internal.Unit) (_ *Readme, err error) {
	defer derrors.WrapAndReport(&err, "ProcessReadme(%q, %q, %q)", u.Path, u.ModulePath, u.Version)
	return processReadme(ctx, u.Readme, u.SourceInfo)
}

func processReadme(ctx context.Context, readme *internal.Readme, sourceInfo *source.Info) (frontendReadme *Readme, err error) {
	if readme == nil || readme.Contents == "" {
		return &Readme{}, nil
	}
	if !isMarkdown(readme.Filepath) {
		t := template.Must(template.New("").Parse(`<pre class="readme">{{.}}</pre>`))
		h, err := t.ExecuteToHTML(readme.Contents)
		if err != nil {
			return nil, err
		}
		return &Readme{HTML: h}, nil
	}

	// Sets priority value so that we always use our custom transformer
	// instead of the default ones. The default values are in:
	// https://github.com/yuin/goldmark/blob/7b90f04af43131db79ec320be0bd4744079b346f/parser/parser.go#L567
	const astTransformerPriority = 10000
	el := &extractLinks{ctx: ctx}
	et := &extractTOC{ctx: ctx, removeTitle: true}
	gdMarkdown := goldmark.New(
		goldmark.WithParserOptions(
			// WithHeadingAttribute allows us to include other attributes in
			// heading tags. This is useful for our aria-level implementation of
			// increasing heading rankings.
			parser.WithHeadingAttribute(),
			// Generates an id in every heading tag. This is used in github in
			// order to generate a link with a hash that a user would scroll to
			//
goldmark
 => github.com/yuin/goldmark#goldmark
			parser.WithAutoHeadingID(),
			// Include custom ASTTransformer using the readme and module info to
			// use translateRelativeLink and translateHTML to modify the AST
			// before it is rendered.
			parser.WithASTTransformers(
				util.Prioritized(&astTransformer{
					info:   sourceInfo,
					readme: readme,
				}, astTransformerPriority),
				// Extract links after we have transformed the URLs.
				util.Prioritized(el, astTransformerPriority+1),
				util.Prioritized(et, astTransformerPriority+1),
			),
		),
		// These extensions lets users write HTML code in the README. This is
		// fine since we process the contents using bluemonday after.
		goldmark.WithRendererOptions(goldmarkHtml.WithUnsafe(), goldmarkHtml.WithXHTML()),
		goldmark.WithExtensions(
			extension.GFM, // Support Github Flavored Markdown.
			emoji.Emoji,   // Support Github markdown emoji markup.
		),
	)
	gdMarkdown.Renderer().AddOptions(
		renderer.WithNodeRenderers(
			util.Prioritized(newHTMLRenderer(sourceInfo, readme), 100),
		),
	)
	contents := []byte(readme.Contents)
	gdParser := gdMarkdown.Parser()
	reader := gmtext.NewReader(contents)
	pctx := parser.NewContext(parser.WithIDs(newIDs()))
	doc := gdParser.Parse(reader, parser.WithContext(pctx))
	gdRenderer := gdMarkdown.Renderer()

	var b bytes.Buffer
	defer func() {
		// It's possible for gdRenderer.Render to panic. For exmaple,
		// https://pkg.go.dev/github.com/jinghzhu/k8scrd/pkg/crd/jinghzhu/v1
		// results in a panic because gdRenderer.Render tries to index a slice
		// out of bounds.
		//
		// In case of a panic from gdRenderer.Render, treat this as a normal
		// error from that function.
		if p := recover(); p != nil {
			log.Debugf(ctx, "gdRenderer.Render: %v", p)
			frontendReadme = &Readme{}
			err = nil
		}
	}()
	if err := gdRenderer.Render(&b, contents, doc); err != nil {
		log.Debugf(ctx, "gdRenderer.Render: %v", err)
		return &Readme{}, nil
	}
	return &Readme{
		HTML:    sanitizeHTML(&b),
		Outline: et.Headings,
		Links:   el.links,
	}, nil
}

// sanitizeHTML sanitizes HTML from a bytes.Buffer so that it is safe.
func sanitizeHTML(b *bytes.Buffer) safehtml.HTML {
	p := bluemonday.UGCPolicy()

	p.AllowAttrs("width", "align").OnElements("img")
	p.AllowAttrs("width", "align").OnElements("div")
	p.AllowAttrs("width", "align").OnElements("p")
	// Allow accessible headings (i.e
).
	p.AllowAttrs("width", "align", "role", "aria-level").OnElements("div")
	for _, h := range []string{"h1", "h2", "h3", "h4", "h5", "h6"} {
		// Needed to preserve github styles heading font-sizes
		p.AllowAttrs("class").OnElements(h)
	}

	s := string(p.SanitizeBytes(b.Bytes()))
	return uncheckedconversions.HTMLFromStringKnownToSatisfyTypeContract(s)
}
<|endoftext|>


<commit_before>// Copyright 2020 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package lfs

import (
	"encoding/json"
	"io"
	"net/http"
	"os"
	"strconv"

	log "unknwon.dev/clog/v2"

	"gogs.io/gogs/internal/conf"
	"gogs.io/gogs/internal/context"
	"gogs.io/gogs/internal/db"
	"gogs.io/gogs/internal/lfsutil"
	"gogs.io/gogs/internal/strutil"
)

const transferBasic = "basic"
const (
	basicOperationUpload   = "upload"
	basicOperationDownload = "download"
)

// GET /{owner}/{repo}.git/info/lfs/object/basic/{oid}
func serveBasicDownload(c *context.Context, repo *db.Repository, oid lfsutil.OID) {
	object, err := db.LFS.GetObjectByOID(repo.ID, oid)
	if err != nil {
		if db.IsErrLFSObjectNotExist(err) {
			responseJSON(c.Resp, http.StatusNotFound, responseError{
				Message: "Object does not exist",
			})
		} else {
			internalServerError(c.Resp)
			log.Error("Failed to get object [repo_id: %d, oid: %s]: %v", repo.ID, oid, err)
		}
		return
	}

	fpath := lfsutil.StorageLocalPath(conf.LFS.ObjectsPath, object.OID)
	r, err := os.Open(fpath)
	if err != nil {
		internalServerError(c.Resp)
		log.Error("Failed to open object file [path: %s]: %v", fpath, err)
		return
	}
	defer r.Close()

	c.Header().Set("Content-Type", "application/octet-stream")
	c.Header().Set("Content-Length", strconv.FormatInt(object.Size, 10))
	c.Status(http.StatusOK)

	_, err = io.Copy(c.Resp, r)
	if err != nil {
		log.Error("Failed to copy object file: %v", err)
		return
	}
}

// PUT /{owner}/{repo}.git/info/lfs/object/basic/{oid}
func serveBasicUpload(c *context.Context, repo *db.Repository, oid lfsutil.OID) {
	err := db.LFS.CreateObject(repo.ID, oid, c.Req.Request.Body, lfsutil.StorageLocal)
	if err != nil {
		internalServerError(c.Resp)
		log.Error("Failed to create object [repo_id: %d, oid: %s]: %v", repo.ID, oid, err)
		return
	}
	c.Status(http.StatusOK)

	log.Trace("[LFS] Object created %q", oid)
}

// POST /{owner}/{repo}.git/info/lfs/object/basic/verify
func serveBasicVerify(c *context.Context, repo *db.Repository) {
	var request basicVerifyRequest
	defer c.Req.Request.Body.Close()
	err := json.NewDecoder(c.Req.Request.Body).Decode(&request)
	if err != nil {
		responseJSON(c.Resp, http.StatusBadRequest, responseError{
			Message: strutil.ToUpperFirst(err.Error()),
		})
		return
	}

	if !lfsutil.ValidOID(request.Oid) {
		responseJSON(c.Resp, http.StatusBadRequest, responseError{
			Message: "Invalid oid",
		})
		return
	}

	object, err := db.LFS.GetObjectByOID(repo.ID, lfsutil.OID(request.Oid))
	if err != nil {
		if db.IsErrLFSObjectNotExist(err) {
			responseJSON(c.Resp, http.StatusNotFound, responseError{
				Message: "Object does not exist",
			})
		} else {
			internalServerError(c.Resp)
			log.Error("Failed to get object [repo_id: %d, oid: %s]: %v", repo.ID, request.Oid, err)
		}
		return
	}

	if object.Size != request.Size {
		responseJSON(c.Resp, http.StatusNotFound, responseError{
			Message: "Object size mismatch",
		})
		return
	}
	c.Status(http.StatusOK)
}

type basicVerifyRequest struct {
	Oid  lfsutil.OID `json:"oid"`
	Size int64       `json:"size"`
}
<commit_msg>lfs: support upload retry (#6061)<commit_after>// Copyright 2020 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package lfs

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"

	log "unknwon.dev/clog/v2"

	"gogs.io/gogs/internal/conf"
	"gogs.io/gogs/internal/context"
	"gogs.io/gogs/internal/db"
	"gogs.io/gogs/internal/lfsutil"
	"gogs.io/gogs/internal/strutil"
)

const transferBasic = "basic"
const (
	basicOperationUpload   = "upload"
	basicOperationDownload = "download"
)

// GET /{owner}/{repo}.git/info/lfs/object/basic/{oid}
func serveBasicDownload(c *context.Context, repo *db.Repository, oid lfsutil.OID) {
	object, err := db.LFS.GetObjectByOID(repo.ID, oid)
	if err != nil {
		if db.IsErrLFSObjectNotExist(err) {
			responseJSON(c.Resp, http.StatusNotFound, responseError{
				Message: "Object does not exist",
			})
		} else {
			internalServerError(c.Resp)
			log.Error("Failed to get object [repo_id: %d, oid: %s]: %v", repo.ID, oid, err)
		}
		return
	}

	fpath := lfsutil.StorageLocalPath(conf.LFS.ObjectsPath, object.OID)
	r, err := os.Open(fpath)
	if err != nil {
		internalServerError(c.Resp)
		log.Error("Failed to open object file [path: %s]: %v", fpath, err)
		return
	}
	defer r.Close()

	c.Header().Set("Content-Type", "application/octet-stream")
	c.Header().Set("Content-Length", strconv.FormatInt(object.Size, 10))
	c.Status(http.StatusOK)

	_, err = io.Copy(c.Resp, r)
	if err != nil {
		log.Error("Failed to copy object file: %v", err)
		return
	}
}

// PUT /{owner}/{repo}.git/info/lfs/object/basic/{oid}
func serveBasicUpload(c *context.Context, repo *db.Repository, oid lfsutil.OID) {
	// NOTE: LFS client will retry upload the same object if there was a partial failure,
	// therefore we would like to skip ones that already exist.
	_, err := db.LFS.GetObjectByOID(repo.ID, oid)
	if err == nil {
		// Object exists, drain the request body and we're good.
		_, _ = io.Copy(ioutil.Discard, c.Req.Request.Body)
		c.Req.Request.Body.Close()
		c.Status(http.StatusOK)
		return
	} else if !db.IsErrLFSObjectNotExist(err) {
		internalServerError(c.Resp)
		log.Error("Failed to get object [repo_id: %d, oid: %s]: %v", repo.ID, oid, err)
		return
	}

	err = db.LFS.CreateObject(repo.ID, oid, c.Req.Request.Body, lfsutil.StorageLocal)
	if err != nil {
		internalServerError(c.Resp)
		log.Error("Failed to create object [repo_id: %d, oid: %s]: %v", repo.ID, oid, err)
		return
	}
	c.Status(http.StatusOK)

	log.Trace("[LFS] Object created %q", oid)
}

// POST /{owner}/{repo}.git/info/lfs/object/basic/verify
func serveBasicVerify(c *context.Context, repo *db.Repository) {
	var request basicVerifyRequest
	defer c.Req.Request.Body.Close()
	err := json.NewDecoder(c.Req.Request.Body).Decode(&request)
	if err != nil {
		responseJSON(c.Resp, http.StatusBadRequest, responseError{
			Message: strutil.ToUpperFirst(err.Error()),
		})
		return
	}

	if !lfsutil.ValidOID(request.Oid) {
		responseJSON(c.Resp, http.StatusBadRequest, responseError{
			Message: "Invalid oid",
		})
		return
	}

	object, err := db.LFS.GetObjectByOID(repo.ID, lfsutil.OID(request.Oid))
	if err != nil {
		if db.IsErrLFSObjectNotExist(err) {
			responseJSON(c.Resp, http.StatusNotFound, responseError{
				Message: "Object does not exist",
			})
		} else {
			internalServerError(c.Resp)
			log.Error("Failed to get object [repo_id: %d, oid: %s]: %v", repo.ID, request.Oid, err)
		}
		return
	}

	if object.Size != request.Size {
		responseJSON(c.Resp, http.StatusNotFound, responseError{
			Message: "Object size mismatch",
		})
		return
	}
	c.Status(http.StatusOK)
}

type basicVerifyRequest struct {
	Oid  lfsutil.OID `json:"oid"`
	Size int64       `json:"size"`
}
<|endoftext|>


<commit_before>package sshd

import (
	"context"
	"errors"
	"net"
	"strings"
	"time"

	"golang.org/x/crypto/ssh"
	"golang.org/x/sync/semaphore"
	grpccodes "google.golang.org/grpc/codes"
	grpcstatus "google.golang.org/grpc/status"

	"gitlab.com/gitlab-org/gitlab-shell/client"
	"gitlab.com/gitlab-org/gitlab-shell/internal/config"
	"gitlab.com/gitlab-org/gitlab-shell/internal/metrics"

	"gitlab.com/gitlab-org/labkit/log"
)

const KeepAliveMsg = "keepalive@openssh.com"

var EOFTimeout = 10 * time.Second

type connection struct {
	cfg                      *config.Config
	concurrentSessions       *semaphore.Weighted
	nconn                    net.Conn
	maxSessions              int64
	remoteAddr               string
	started                  time.Time
	establishSessionDuration float64
}

type channelHandler func(*ssh.ServerConn, ssh.Channel, <-chan *ssh.Request) error

func newConnection(cfg *config.Config, nconn net.Conn) *connection {
	maxSessions := cfg.Server.ConcurrentSessionsLimit

	return &connection{
		cfg:                cfg,
		maxSessions:        maxSessions,
		concurrentSessions: semaphore.NewWeighted(maxSessions),
		nconn:              nconn,
		remoteAddr:         nconn.RemoteAddr().String(),
		started:            time.Now(),
	}
}

func (c *connection) handle(ctx context.Context, srvCfg *ssh.ServerConfig, handler channelHandler) {
	sconn, chans, err := c.initServerConn(ctx, srvCfg)
	if err != nil {
		return
	}

	if c.cfg.Server.ClientAliveInterval > 0 {
		ticker := time.NewTicker(time.Duration(c.cfg.Server.ClientAliveInterval))
		defer ticker.Stop()
		go c.sendKeepAliveMsg(ctx, sconn, ticker)
	}

	c.handleRequests(ctx, sconn, chans, handler)

	reason := sconn.Wait()
	log.WithContextFields(ctx, log.Fields{
		"duration_s":                   time.Since(c.started).Seconds(),
		"establish_session_duration_s": c.establishSessionDuration,
		"reason":                       reason,
	}).Info("server: handleConn: done")
}

func (c *connection) initServerConn(ctx context.Context, srvCfg *ssh.ServerConfig) (*ssh.ServerConn, <-chan ssh.NewChannel, error) {
	if c.cfg.Server.LoginGraceTime > 0 {
		c.nconn.SetDeadline(time.Now().Add(time.Duration(c.cfg.Server.LoginGraceTime)))
		defer c.nconn.SetDeadline(time.Time{})
	}

	sconn, chans, reqs, err := ssh.NewServerConn(c.nconn, srvCfg)
	if err != nil {
		msg := "connection: initServerConn: failed to initialize SSH connection"
		logger := log.WithContextFields(ctx, log.Fields{"remote_addr": c.remoteAddr}).WithError(err)

		if strings.Contains(err.Error(), "no common algorithm for host key") || err.Error() == "EOF" {
			logger.Debug(msg)
		} else {
			logger.Warn(msg)
		}

		return nil, nil, err
	}
	go ssh.DiscardRequests(reqs)

	return sconn, chans, err
}

func (c *connection) handleRequests(ctx context.Context, sconn *ssh.ServerConn, chans <-chan ssh.NewChannel, handler channelHandler) {
	ctxlog := log.WithContextFields(ctx, log.Fields{"remote_addr": c.remoteAddr})

	for newChannel := range chans {
		ctxlog.WithField("channel_type", newChannel.ChannelType()).Info("connection: handle: new channel requested")
		if newChannel.ChannelType() != "session" {
			ctxlog.Info("connection: handleRequests: unknown channel type")
			newChannel.Reject(ssh.UnknownChannelType, "unknown channel type")
			continue
		}
		if !c.concurrentSessions.TryAcquire(1) {
			ctxlog.Info("connection: handleRequests: too many concurrent sessions")
			newChannel.Reject(ssh.ResourceShortage, "too many concurrent sessions")
			metrics.SshdHitMaxSessions.Inc()
			continue
		}
		channel, requests, err := newChannel.Accept()
		if err != nil {
			ctxlog.WithError(err).Error("connection: handleRequests: accepting channel failed")
			c.concurrentSessions.Release(1)
			continue
		}

		go func() {
			defer func(started time.Time) {
				metrics.SshdSessionDuration.Observe(time.Since(started).Seconds())
			}(time.Now())
			c.establishSessionDuration = time.Since(c.started).Seconds()

			defer c.concurrentSessions.Release(1)

			// Prevent a panic in a single session from taking out the whole server
			defer func() {
				if err := recover(); err != nil {
					ctxlog.WithField("recovered_error", err).Warn("panic handling session")
				}
			}()

			metrics.SliSshdSessionsTotal.Inc()
			err := handler(sconn, channel, requests)
			if err != nil {
				c.trackError(err)
			}

			ctxlog.Info("connection: handleRequests: done")
		}()
	}

	// When a connection has been prematurely closed we block execution until all concurrent sessions are released
	// in order to allow Gitaly complete the operations and close all the channels gracefully.
	// If it didn't happen within timeout, we unblock the execution
	// Related issue: https://gitlab.com/gitlab-org/gitlab-shell/-/issues/563
	ctx, cancel := context.WithTimeout(ctx, EOFTimeout)
	defer cancel()
	c.concurrentSessions.Acquire(ctx, c.maxSessions)
}

func (c *connection) sendKeepAliveMsg(ctx context.Context, sconn *ssh.ServerConn, ticker *time.Ticker) {
	ctxlog := log.WithContextFields(ctx, log.Fields{"remote_addr": c.remoteAddr})

	for {
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
			ctxlog.Debug("connection: sendKeepAliveMsg: send keepalive message to a client")

			sconn.SendRequest(KeepAliveMsg, true, nil)
		}
	}
}

func (c *connection) trackError(err error) {
	var apiError *client.ApiError
	if errors.As(err, &apiError) {
		return
	}

	grpcCode := grpcstatus.Code(err)
	if grpcCode == grpccodes.Canceled || grpcCode == grpccodes.Unavailable {
		return
	}

	metrics.SliSshdSessionsErrorsTotal.Inc()
}
<commit_msg>Add missing SshdSessionEstablishedDuration metrics<commit_after>package sshd

import (
	"context"
	"errors"
	"net"
	"strings"
	"time"

	"golang.org/x/crypto/ssh"
	"golang.org/x/sync/semaphore"
	grpccodes "google.golang.org/grpc/codes"
	grpcstatus "google.golang.org/grpc/status"

	"gitlab.com/gitlab-org/gitlab-shell/client"
	"gitlab.com/gitlab-org/gitlab-shell/internal/config"
	"gitlab.com/gitlab-org/gitlab-shell/internal/metrics"

	"gitlab.com/gitlab-org/labkit/log"
)

const KeepAliveMsg = "keepalive@openssh.com"

var EOFTimeout = 10 * time.Second

type connection struct {
	cfg                      *config.Config
	concurrentSessions       *semaphore.Weighted
	nconn                    net.Conn
	maxSessions              int64
	remoteAddr               string
	started                  time.Time
	establishSessionDuration float64
}

type channelHandler func(*ssh.ServerConn, ssh.Channel, <-chan *ssh.Request) error

func newConnection(cfg *config.Config, nconn net.Conn) *connection {
	maxSessions := cfg.Server.ConcurrentSessionsLimit

	return &connection{
		cfg:                cfg,
		maxSessions:        maxSessions,
		concurrentSessions: semaphore.NewWeighted(maxSessions),
		nconn:              nconn,
		remoteAddr:         nconn.RemoteAddr().String(),
		started:            time.Now(),
	}
}

func (c *connection) handle(ctx context.Context, srvCfg *ssh.ServerConfig, handler channelHandler) {
	sconn, chans, err := c.initServerConn(ctx, srvCfg)
	if err != nil {
		return
	}

	if c.cfg.Server.ClientAliveInterval > 0 {
		ticker := time.NewTicker(time.Duration(c.cfg.Server.ClientAliveInterval))
		defer ticker.Stop()
		go c.sendKeepAliveMsg(ctx, sconn, ticker)
	}

	c.handleRequests(ctx, sconn, chans, handler)

	reason := sconn.Wait()
	log.WithContextFields(ctx, log.Fields{
		"duration_s":                   time.Since(c.started).Seconds(),
		"establish_session_duration_s": c.establishSessionDuration,
		"reason":                       reason,
	}).Info("server: handleConn: done")
}

func (c *connection) initServerConn(ctx context.Context, srvCfg *ssh.ServerConfig) (*ssh.ServerConn, <-chan ssh.NewChannel, error) {
	if c.cfg.Server.LoginGraceTime > 0 {
		c.nconn.SetDeadline(time.Now().Add(time.Duration(c.cfg.Server.LoginGraceTime)))
		defer c.nconn.SetDeadline(time.Time{})
	}

	sconn, chans, reqs, err := ssh.NewServerConn(c.nconn, srvCfg)
	if err != nil {
		msg := "connection: initServerConn: failed to initialize SSH connection"
		logger := log.WithContextFields(ctx, log.Fields{"remote_addr": c.remoteAddr}).WithError(err)

		if strings.Contains(err.Error(), "no common algorithm for host key") || err.Error() == "EOF" {
			logger.Debug(msg)
		} else {
			logger.Warn(msg)
		}

		return nil, nil, err
	}
	go ssh.DiscardRequests(reqs)

	return sconn, chans, err
}

func (c *connection) handleRequests(ctx context.Context, sconn *ssh.ServerConn, chans <-chan ssh.NewChannel, handler channelHandler) {
	ctxlog := log.WithContextFields(ctx, log.Fields{"remote_addr": c.remoteAddr})

	for newChannel := range chans {
		ctxlog.WithField("channel_type", newChannel.ChannelType()).Info("connection: handle: new channel requested")
		if newChannel.ChannelType() != "session" {
			ctxlog.Info("connection: handleRequests: unknown channel type")
			newChannel.Reject(ssh.UnknownChannelType, "unknown channel type")
			continue
		}
		if !c.concurrentSessions.TryAcquire(1) {
			ctxlog.Info("connection: handleRequests: too many concurrent sessions")
			newChannel.Reject(ssh.ResourceShortage, "too many concurrent sessions")
			metrics.SshdHitMaxSessions.Inc()
			continue
		}
		channel, requests, err := newChannel.Accept()
		if err != nil {
			ctxlog.WithError(err).Error("connection: handleRequests: accepting channel failed")
			c.concurrentSessions.Release(1)
			continue
		}

		go func() {
			defer func(started time.Time) {
				metrics.SshdSessionDuration.Observe(time.Since(started).Seconds())
			}(time.Now())
			c.establishSessionDuration = time.Since(c.started).Seconds()
			metrics.SshdSessionEstablishedDuration.Observe(c.establishSessionDuration)

			defer c.concurrentSessions.Release(1)

			// Prevent a panic in a single session from taking out the whole server
			defer func() {
				if err := recover(); err != nil {
					ctxlog.WithField("recovered_error", err).Warn("panic handling session")
				}
			}()

			metrics.SliSshdSessionsTotal.Inc()
			err := handler(sconn, channel, requests)
			if err != nil {
				c.trackError(err)
			}

			ctxlog.Info("connection: handleRequests: done")
		}()
	}

	// When a connection has been prematurely closed we block execution until all concurrent sessions are released
	// in order to allow Gitaly complete the operations and close all the channels gracefully.
	// If it didn't happen within timeout, we unblock the execution
	// Related issue: https://gitlab.com/gitlab-org/gitlab-shell/-/issues/563
	ctx, cancel := context.WithTimeout(ctx, EOFTimeout)
	defer cancel()
	c.concurrentSessions.Acquire(ctx, c.maxSessions)
}

func (c *connection) sendKeepAliveMsg(ctx context.Context, sconn *ssh.ServerConn, ticker *time.Ticker) {
	ctxlog := log.WithContextFields(ctx, log.Fields{"remote_addr": c.remoteAddr})

	for {
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
			ctxlog.Debug("connection: sendKeepAliveMsg: send keepalive message to a client")

			sconn.SendRequest(KeepAliveMsg, true, nil)
		}
	}
}

func (c *connection) trackError(err error) {
	var apiError *client.ApiError
	if errors.As(err, &apiError) {
		return
	}

	grpcCode := grpcstatus.Code(err)
	if grpcCode == grpccodes.Canceled || grpcCode == grpccodes.Unavailable {
		return
	}

	metrics.SliSshdSessionsErrorsTotal.Inc()
}
<|endoftext|>


<commit_before>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package stanza

import (
	// Register parsers and transformers for stanza-based log receivers
	_ "github.com/open-telemetry/opentelemetry-log-collection/operator/builtin/parser/json"
	_ "github.com/open-telemetry/opentelemetry-log-collection/operator/builtin/parser/regex"
	_ "github.com/open-telemetry/opentelemetry-log-collection/operator/builtin/parser/severity"
	_ "github.com/open-telemetry/opentelemetry-log-collection/operator/builtin/parser/time"
	_ "github.com/open-telemetry/opentelemetry-log-collection/operator/builtin/parser/trace"
	_ "github.com/open-telemetry/opentelemetry-log-collection/operator/builtin/transformer/copy"
	_ "github.com/open-telemetry/opentelemetry-log-collection/operator/builtin/transformer/metadata"
	_ "github.com/open-telemetry/opentelemetry-log-collection/operator/builtin/transformer/recombine"
	_ "github.com/open-telemetry/opentelemetry-log-collection/operator/builtin/transformer/restructure"
	_ "github.com/open-telemetry/opentelemetry-log-collection/operator/builtin/transformer/router"
)
<commit_msg>Add more stanza operators (#4312)<commit_after>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package stanza

import (
	// Register parsers and transformers for stanza-based log receivers
	_ "github.com/open-telemetry/opentelemetry-log-collection/operator/builtin/parser/csv"
	_ "github.com/open-telemetry/opentelemetry-log-collection/operator/builtin/parser/json"
	_ "github.com/open-telemetry/opentelemetry-log-collection/operator/builtin/parser/regex"
	_ "github.com/open-telemetry/opentelemetry-log-collection/operator/builtin/parser/severity"
	_ "github.com/open-telemetry/opentelemetry-log-collection/operator/builtin/parser/time"
	_ "github.com/open-telemetry/opentelemetry-log-collection/operator/builtin/parser/trace"
	_ "github.com/open-telemetry/opentelemetry-log-collection/operator/builtin/parser/uri"
	_ "github.com/open-telemetry/opentelemetry-log-collection/operator/builtin/transformer/add"
	_ "github.com/open-telemetry/opentelemetry-log-collection/operator/builtin/transformer/copy"
	_ "github.com/open-telemetry/opentelemetry-log-collection/operator/builtin/transformer/filter"
	_ "github.com/open-telemetry/opentelemetry-log-collection/operator/builtin/transformer/flatten"
	_ "github.com/open-telemetry/opentelemetry-log-collection/operator/builtin/transformer/metadata"
	_ "github.com/open-telemetry/opentelemetry-log-collection/operator/builtin/transformer/move"
	_ "github.com/open-telemetry/opentelemetry-log-collection/operator/builtin/transformer/recombine"
	_ "github.com/open-telemetry/opentelemetry-log-collection/operator/builtin/transformer/restructure"
	_ "github.com/open-telemetry/opentelemetry-log-collection/operator/builtin/transformer/retain"
	_ "github.com/open-telemetry/opentelemetry-log-collection/operator/builtin/transformer/router"
)
<|endoftext|>


<commit_before>// Copyright (c) 2019 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

// Package main runs a stress test on each peer list implementation,
// concurrently adding, removing, connecting, disconnecting, and choosing
// peers.
//
// Output:
//
//  name                     stress  workers  min       max         mean       choices  updates
//  round-robin              true    1        245       8770915     146851     5253     46150
//  round-robin              false   1        207       418011      291        971224   1
//  round-robin              true    10       233       16849329    387306     19074    50669
//  round-robin              false   10       223       1578697     5346       1194757  1
//  round-robin              true    100      230       27465221    908544     75258    46810
//  round-robin              false   100      226       7461220     67418      1032094  1
//  round-robin              true    1000     233       22398858    2024969    330804   4576
//  round-robin              false   1000     229       21501281    802412     839121   1
//  round-robin              true    10000    233       92130003    19808992   346389   297
//  round-robin              false   10000    276       64225586    21145554   314307   1
//  round-robin              true    100000   230       938109206   206172150  420222   8
//  round-robin              false   100000   234       752404056   162852049  555573   1
//  random                   true    1        290       9613482     147458     5411     51211
//  random                   false   1        218       4010623     320        945794   1
//  random                   true    10       251       13586138    292117     25511    51602
//  random                   false   10       234       6359156     6460       945427   1
//  random                   true    100      245       22698254    635979     106915   41375
//  random                   false   100      245       11036988    90960      740277   1
//  random                   true    1000     248       16201139    1991433    335656   8817
//  random                   false   1000     241       20870812    1309373    510240   1
//  random                   true    10000    289       78923469    23178028   288479   125
//  random                   false   10000    297       80162200    24015477   278955   1
//  random                   true    100000   241       765837860   237899834  351979   25
//  random                   false   100000   289       694941549   212820591  365262   1
//  fewest-pending-requests  true    1        390       4419885     224512     3200     45697
//  fewest-pending-requests  false   1        313       1388333     437        811476   1
//  fewest-pending-requests  true    10       341       15878684    238641     29821    38477
//  fewest-pending-requests  false   10       328       5097597     8860       719393   1
//  fewest-pending-requests  true    100      338       12701474    452931     150094   35696
//  fewest-pending-requests  false   100      339       9182151     96564      705745   1
//  fewest-pending-requests  true    1000     329       13381772    2351399    285263   1816
//  fewest-pending-requests  false   1000     312       16021295    1546288    430947   1
//  fewest-pending-requests  true    10000    421       85027409    29813414   227209   135
//  fewest-pending-requests  false   10000    404       86757606    27587847   244877   1
//  fewest-pending-requests  true    100000   381       969925927   278125286  320719   11
//  fewest-pending-requests  false   100000   342       759484399   256068224  351255   1
//  two-random-choices       true    1        296       12530130    167011     4873     56520
//  two-random-choices       false   1        234       2157414     331        973400   1
//  two-random-choices       true    10       257       14773053    264627     27980    49152
//  two-random-choices       false   10       254       6038348     6769       896344   1
//  two-random-choices       true    100      264       16509807    566482     120169   47062
//  two-random-choices       false   100      265       8954712     78887      840535   1
//  two-random-choices       true    1000     262       16153194    1959869    341938   6304
//  two-random-choices       false   1000     257       14265730    1299014    512357   1
//  two-random-choices       true    10000    312       65980563    25155737   263431   75
//  two-random-choices       false   10000    315       68347456    23842068   282351   1
//  two-random-choices       true    100000   259       1164075075  114441088  663750   31
//  two-random-choices       false   100000   269       836529911   164734573  545225   1
package main

import (
	"fmt"
	"time"

	"go.uber.org/yarpc/api/peer"
	"go.uber.org/yarpc/peer/pendingheap"
	"go.uber.org/yarpc/peer/randpeer"
	"go.uber.org/yarpc/peer/roundrobin"
	"go.uber.org/yarpc/peer/tworandomchoices"
	"go.uber.org/yarpc/yarpctest"
)

func main() {
	fmt.Printf("name,stress,workers,min,max,mean,choices,updates\n")
	log := logger{}
	for _, c := range []struct {
		name    string
		newFunc func(peer.Transport) peer.ChooserList
	}{
		{
			name: "round-robin",
			newFunc: func(trans peer.Transport) peer.ChooserList {
				return roundrobin.New(trans)
			},
		},
		{
			name: "random",
			newFunc: func(trans peer.Transport) peer.ChooserList {
				return randpeer.New(trans)
			},
		},
		{
			name: "fewest-pending-requests",
			newFunc: func(trans peer.Transport) peer.ChooserList {
				return pendingheap.New(trans)
			},
		},
		{
			name: "two-random-choices",
			newFunc: func(trans peer.Transport) peer.ChooserList {
				return tworandomchoices.New(trans)
			},
		},
	} {
		for i := 1; i <= 100000; i *= 10 {
			for _, lowStress := range []bool{false, true} {
				report := yarpctest.ListStressTest{
					Workers:   i,
					Duration:  time.Second,
					Timeout:   time.Millisecond * time.Duration(i),
					LowStress: lowStress,
					New:       c.newFunc,
				}.Run(log)
				if report.Choices != 0 {
					fmt.Printf("%s,%v,%d,%d,%d,%d,%d,%d\n",
						c.name,
						!lowStress,
						uint64(i),
						uint64(report.Min),
						uint64(report.Max),
						uint64(report.Total/time.Duration(report.Choices)),
						report.Choices,
						report.Updates,
					)
				}
			}
		}
	}
}

type logger struct{}

func (logger) Logf(format string, args ...interface{}) {
	fmt.Printf(format, args...)
}
<commit_msg>Reduce peer list stress test range to stay under goroutine limit (#1833)<commit_after>// Copyright (c) 2019 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

// Package main runs a stress test on each peer list implementation,
// concurrently adding, removing, connecting, disconnecting, and choosing
// peers.
//
// Output:
//
//  name                     stress  workers  min       max         mean       choices  updates
//  round-robin              true    1        245       8770915     146851     5253     46150
//  round-robin              false   1        207       418011      291        971224   1
//  round-robin              true    10       233       16849329    387306     19074    50669
//  round-robin              false   10       223       1578697     5346       1194757  1
//  round-robin              true    100      230       27465221    908544     75258    46810
//  round-robin              false   100      226       7461220     67418      1032094  1
//  round-robin              true    1000     233       22398858    2024969    330804   4576
//  round-robin              false   1000     229       21501281    802412     839121   1
//  round-robin              true    10000    233       92130003    19808992   346389   297
//  round-robin              false   10000    276       64225586    21145554   314307   1
//  round-robin              true    100000   230       938109206   206172150  420222   8
//  round-robin              false   100000   234       752404056   162852049  555573   1
//  random                   true    1        290       9613482     147458     5411     51211
//  random                   false   1        218       4010623     320        945794   1
//  random                   true    10       251       13586138    292117     25511    51602
//  random                   false   10       234       6359156     6460       945427   1
//  random                   true    100      245       22698254    635979     106915   41375
//  random                   false   100      245       11036988    90960      740277   1
//  random                   true    1000     248       16201139    1991433    335656   8817
//  random                   false   1000     241       20870812    1309373    510240   1
//  random                   true    10000    289       78923469    23178028   288479   125
//  random                   false   10000    297       80162200    24015477   278955   1
//  random                   true    100000   241       765837860   237899834  351979   25
//  random                   false   100000   289       694941549   212820591  365262   1
//  fewest-pending-requests  true    1        390       4419885     224512     3200     45697
//  fewest-pending-requests  false   1        313       1388333     437        811476   1
//  fewest-pending-requests  true    10       341       15878684    238641     29821    38477
//  fewest-pending-requests  false   10       328       5097597     8860       719393   1
//  fewest-pending-requests  true    100      338       12701474    452931     150094   35696
//  fewest-pending-requests  false   100      339       9182151     96564      705745   1
//  fewest-pending-requests  true    1000     329       13381772    2351399    285263   1816
//  fewest-pending-requests  false   1000     312       16021295    1546288    430947   1
//  fewest-pending-requests  true    10000    421       85027409    29813414   227209   135
//  fewest-pending-requests  false   10000    404       86757606    27587847   244877   1
//  fewest-pending-requests  true    100000   381       969925927   278125286  320719   11
//  fewest-pending-requests  false   100000   342       759484399   256068224  351255   1
//  two-random-choices       true    1        296       12530130    167011     4873     56520
//  two-random-choices       false   1        234       2157414     331        973400   1
//  two-random-choices       true    10       257       14773053    264627     27980    49152
//  two-random-choices       false   10       254       6038348     6769       896344   1
//  two-random-choices       true    100      264       16509807    566482     120169   47062
//  two-random-choices       false   100      265       8954712     78887      840535   1
//  two-random-choices       true    1000     262       16153194    1959869    341938   6304
//  two-random-choices       false   1000     257       14265730    1299014    512357   1
//  two-random-choices       true    10000    312       65980563    25155737   263431   75
//  two-random-choices       false   10000    315       68347456    23842068   282351   1
//  two-random-choices       true    100000   259       1164075075  114441088  663750   31
//  two-random-choices       false   100000   269       836529911   164734573  545225   1
package main

import (
	"fmt"
	"time"

	"go.uber.org/yarpc/api/peer"
	"go.uber.org/yarpc/peer/pendingheap"
	"go.uber.org/yarpc/peer/randpeer"
	"go.uber.org/yarpc/peer/roundrobin"
	"go.uber.org/yarpc/peer/tworandomchoices"
	"go.uber.org/yarpc/yarpctest"
)

func main() {
	fmt.Printf("name,stress,workers,min,max,mean,choices,updates\n")
	log := logger{}
	for _, c := range []struct {
		name    string
		newFunc func(peer.Transport) peer.ChooserList
	}{
		{
			name: "round-robin",
			newFunc: func(trans peer.Transport) peer.ChooserList {
				return roundrobin.New(trans)
			},
		},
		{
			name: "random",
			newFunc: func(trans peer.Transport) peer.ChooserList {
				return randpeer.New(trans)
			},
		},
		{
			name: "fewest-pending-requests",
			newFunc: func(trans peer.Transport) peer.ChooserList {
				return pendingheap.New(trans)
			},
		},
		{
			name: "two-random-choices",
			newFunc: func(trans peer.Transport) peer.ChooserList {
				return tworandomchoices.New(trans)
			},
		},
	} {
		for i := 1; i <= 1000; i *= 10 {
			for _, lowStress := range []bool{false, true} {
				report := yarpctest.ListStressTest{
					Workers:   i,
					Duration:  time.Second,
					Timeout:   time.Millisecond * time.Duration(i),
					LowStress: lowStress,
					New:       c.newFunc,
				}.Run(log)
				if report.Choices != 0 {
					fmt.Printf("%s,%v,%d,%d,%d,%d,%d,%d\n",
						c.name,
						!lowStress,
						uint64(i),
						uint64(report.Min),
						uint64(report.Max),
						uint64(report.Total/time.Duration(report.Choices)),
						report.Choices,
						report.Updates,
					)
				}
			}
		}
	}
}

type logger struct{}

func (logger) Logf(format string, args ...interface{}) {
	fmt.Printf(format, args...)
}
<|endoftext|>


<commit_before>package matrix

type CVector struct {
	column int
	matrix Matrix
}

func (v *CVector) Matrix() Matrix {
	return v.matrix
}

func (v *CVector) Size() int {
	row, _ := v.matrix.Shape()
	return row
}
<commit_msg>Implement methods similar to "*RVector"'s methods for "*CVector".<commit_after>package matrix

type CVector struct {
	column    int
	undefined bool
	matrix    Matrix
}

func NewCVector(matrix Matrix, column int) *CVector {
	_, columns := matrix.Shape()

	v := &CVector{
		column:    column,
		undefined: column >= columns,
		matrix:    matrix,
	}

	return v
}

func (v *CVector) Matrix() Matrix {
	return v.matrix
}

func (v *CVector) Size() int {
	row, _ := v.matrix.Shape()
	return row
}

func (v *CVector) IsUndefined() bool {
	return v.undefined || v.matrix.IsUndefined()
}

func (v *CVector) Get(index int) (float64, error) {
	return v.matrix.Get(index, v.column)
}

func (v *CVector) Update(index int, value float64) *CVector {
	v.matrix.Update(index, v.column, value)

	return v
}
<|endoftext|>


<commit_before>// The matrix package contains various utilities for dealing with raw matrices.
// The interface is loosely based on the NumPy package in Python.
//
// Certain linear algebra methods, particularly in the Matrix interface, rely
// on BLAS. In order to use it, you will need to register an appropriate engine.
// See the documentation at https://github.com/gonum/blas for details. You can
// register a default (native Go) engine by calling InitDefaultBlas().
// If you see a panic message like
// "mat64: no blas engine registered: call Register()"
// then you need to register a BLAS engine. If you don't see this error, then
// you probably don't need to worry about it.
package matrix

import (
	"fmt"
	"math/rand"
	"time"
)

func init() {
	rand.Seed(time.Now().UnixNano())
}

// ArraySparsity indicates the representation type of the matrix
type ArraySparsity int

const (
	DenseArray ArraySparsity = iota
	SparseCooMatrix
	SparseDiagMatrix
)

// A NDArray is an n-dimensional array of numbers which can be manipulated in
// various ways. Concrete implementations can differ; for instance, sparse
// and dense representations are possible.
type NDArray interface {

	// Return the element-wise sum of this array and one or more others
	Add(others ...NDArray) NDArray

	// Returns true if and only if all items are nonzero
	All() bool

	// Returns true if f is true for all array elements
	AllF(f func(v float64) bool) bool

	// Returns true if f is true for all pairs of array elements in the same position
	AllF2(f func(v1, v2 float64) bool, other NDArray) bool

	// Returns true if and only if any item is nonzero
	Any() bool

	// Returns true if f is true for any array element
	AnyF(f func(v float64) bool) bool

	// Returns true if f is true for any pair of array elements in the same position
	AnyF2(f func(v1, v2 float64) bool, other NDArray) bool

	// Return the result of applying a function to all elements
	Apply(f func(float64) float64) NDArray

	// Get the matrix data as a flattened 1D array; sparse matrices will make
	// a copy first.
	Array() []float64

	// Create a new array by concatenating this with another array along the
	// specified axis. The array shapes must be equal along all other axes.
	// It is legal to add a new axis.
	Concat(axis int, others ...NDArray) NDArray

	// Returns a duplicate of this array
	Copy() NDArray

	// Counts the number of nonzero elements in the array
	CountNonzero() int

	// Returns a dense copy of the array
	Dense() NDArray

	// Return the element-wise quotient of this array and one or more others.
	// This function defines 0 / 0 = 0, so it's useful for sparse arrays.
	Div(others ...NDArray) NDArray

	// Returns true if and only if all elements in the two arrays are equal
	Equal(other NDArray) bool

	// Set all array elements to the given value
	Fill(value float64)

	// Get an array element in a flattened verison of this array
	FlatItem(index int) float64

	// Set an array element in a flattened version of this array
	FlatItemSet(value float64, index int)

	// Return an iterator over populated matrix entries
	FlatIter() FlatNDArrayIterator

	// Get an array element
	Item(index ...int) float64

	// Return the result of adding a scalar value to each array element
	ItemAdd(value float64) NDArray

	// Return the result of dividing each array element by a scalar value
	ItemDiv(value float64) NDArray

	// Return the reuslt of multiplying each array element by a scalar value
	ItemProd(value float64) NDArray

	// Return the result of subtracting a scalar value from each array element
	ItemSub(value float64) NDArray

	// Set an array element
	ItemSet(value float64, index ...int)

	// Return an iterator over populated matrix entries
	Iter() CoordNDArrayIterator

	// Returns the array as a matrix. This is only possible for 1D and 2D arrays;
	// 1D arrays of length n are converted into n x 1 vectors.
	M() Matrix

	// Get the value of the largest array element
	Max() float64

	// Get the value of the smallest array element
	Min() float64

	// Return the element-wise product of this array and one or more others
	Prod(others ...NDArray) NDArray

	// The number of dimensions in the matrix
	NDim() int

	// Return a copy of the array, normalized to sum to 1
	Normalize() NDArray

	// Get a 1D copy of the array, in 'C' order: rightmost axes change fastest
	Ravel() NDArray

	// A slice giving the size of all array dimensions
	Shape() []int

	// The total number of elements in the matrix
	Size() int

	// Get an array containing a rectangular slice of this array.
	// `from` and `to` should both have one index per axis. The indices
	// in `from` and `to` define the first and just-past-last indices you wish
	// to select along each axis.
	Slice(from []int, to []int) NDArray

	// Ask whether the matrix has a sparse representation (useful for optimization)
	Sparsity() ArraySparsity

	// Return the element-wise difference of this array and one or more others
	Sub(others ...NDArray) NDArray

	// Return the sum of all array elements
	Sum() float64

	// Return the same matrix, but with axes transposed. The same data is used,
	// for speed and memory efficiency. Use Copy() to create a new array.
	// A 1D array is unchanged; create a 2D analog to rotate a vector.
	T() NDArray
}

// Create an array from literal data
func A(shape []int, values ...float64) NDArray {
	size := 1
	for _, sz := range shape {
		size *= sz
	}
	if len(values) != size {
		panic(fmt.Sprintf("Expected %d array elements but got %d", size, len(values)))
	}
	array := &DenseF64Array{
		shape: shape,
		array: make([]float64, len(values)),
	}
	copy(array.array[:], values[:])
	return array
}

// Create a 1D array
func A1(values ...float64) NDArray {
	return A([]int{len(values)}, values...)
}

// Create a 2D array
func A2(values ...[]float64) NDArray {
	array := &DenseF64Array{
		shape: []int{len(values), len(values[0])},
		array: make([]float64, len(values)*len(values[0])),
	}
	for i0 := 0; i0 < array.shape[0]; i0++ {
		if len(values[i0]) != array.shape[1] {
			panic(fmt.Sprintf("A2 got inconsistent array lengths %d and %d", array.shape[1], len(values[i0])))
		}
		for i1 := 0; i1 < array.shape[1]; i1++ {
			array.ItemSet(values[i0][i1], i0, i1)
		}
	}
	return array
}

// Create an NDArray of float64 values, initialized to zero
func Dense(size ...int) NDArray {
	totalSize := 1
	for _, sz := range size {
		totalSize *= sz
	}
	return &DenseF64Array{
		shape: size,
		array: make([]float64, totalSize),
	}
}

// Create an NDArray of float64 values, initialized to value
func WithValue(value float64, size ...int) NDArray {
	array := Dense(size...)
	array.Fill(value)
	return array
}

// Create an NDArray of float64 values, initialized to zero
func Zeros(size ...int) NDArray {
	return Dense(size...)
}

// Create an NDArray of float64 values, initialized to one
func Ones(size ...int) NDArray {
	return WithValue(1.0, size...)
}

// Create a dense NDArray of float64 values, initialized to uniformly random
// values in [0, 1).
func Rand(size ...int) NDArray {
	array := Dense(size...)

	max := array.Size()
	for i := 0; i < max; i++ {
		array.FlatItemSet(rand.Float64(), i)
	}

	return array
}

// Create a dense NDArray of float64 values, initialized to random values in
// [-math.MaxFloat64, +math.MaxFloat64] distributed on the standard Normal
// distribution.
func RandN(size ...int) NDArray {
	array := Dense(size...)

	max := array.Size()
	for i := 0; i < max; i++ {
		array.FlatItemSet(rand.NormFloat64(), i)
	}

	return array
}
<commit_msg>updated package docs<commit_after>// The matrix package contains various utilities for dealing with raw matrices.
// The interface is loosely based on the NumPy package in Python.
//
// The key interfaces here are:
// * NDArray – A multidimensional array, with dense and (2D-only) sparse
//             implementations.
// * Matrix – A two-dimensional array, with various methods only available when
//            working in two dimensions.  A two-dimensional NDArray can be
//            trivially converted to the Matrix type by calling arr.M().
//
// When possible, function implementations take advantage of matrix sparsity.
// For instance, MProd(), the matrix multiplication function, performs the
// minimum amount of work required based on the types of its arguments.
//
// Certain linear algebra methods, particularly in the Matrix interface, rely
// on BLAS. In order to use it, you will need to register an appropriate engine.
// See the documentation at https://github.com/gonum/blas for details. You can
// register a default (native Go) engine by calling InitDefaultBlas().
// If you see a panic message like
// "mat64: no blas engine registered: call Register()"
// then you need to register a BLAS engine. If you don't see this error, then
// you probably don't need to worry about it.
package matrix

import (
	"fmt"
	"math/rand"
	"time"
)

func init() {
	rand.Seed(time.Now().UnixNano())
}

// ArraySparsity indicates the representation type of the matrix
type ArraySparsity int

const (
	DenseArray ArraySparsity = iota
	SparseCooMatrix
	SparseDiagMatrix
)

// A NDArray is an n-dimensional array of numbers which can be manipulated in
// various ways. Concrete implementations can differ; for instance, sparse
// and dense representations are possible.
type NDArray interface {

	// Return the element-wise sum of this array and one or more others
	Add(others ...NDArray) NDArray

	// Returns true if and only if all items are nonzero
	All() bool

	// Returns true if f is true for all array elements
	AllF(f func(v float64) bool) bool

	// Returns true if f is true for all pairs of array elements in the same position
	AllF2(f func(v1, v2 float64) bool, other NDArray) bool

	// Returns true if and only if any item is nonzero
	Any() bool

	// Returns true if f is true for any array element
	AnyF(f func(v float64) bool) bool

	// Returns true if f is true for any pair of array elements in the same position
	AnyF2(f func(v1, v2 float64) bool, other NDArray) bool

	// Return the result of applying a function to all elements
	Apply(f func(float64) float64) NDArray

	// Get the matrix data as a flattened 1D array; sparse matrices will make
	// a copy first.
	Array() []float64

	// Create a new array by concatenating this with another array along the
	// specified axis. The array shapes must be equal along all other axes.
	// It is legal to add a new axis.
	Concat(axis int, others ...NDArray) NDArray

	// Returns a duplicate of this array
	Copy() NDArray

	// Counts the number of nonzero elements in the array
	CountNonzero() int

	// Returns a dense copy of the array
	Dense() NDArray

	// Return the element-wise quotient of this array and one or more others.
	// This function defines 0 / 0 = 0, so it's useful for sparse arrays.
	Div(others ...NDArray) NDArray

	// Returns true if and only if all elements in the two arrays are equal
	Equal(other NDArray) bool

	// Set all array elements to the given value
	Fill(value float64)

	// Get an array element in a flattened verison of this array
	FlatItem(index int) float64

	// Set an array element in a flattened version of this array
	FlatItemSet(value float64, index int)

	// Return an iterator over populated matrix entries
	FlatIter() FlatNDArrayIterator

	// Get an array element
	Item(index ...int) float64

	// Return the result of adding a scalar value to each array element
	ItemAdd(value float64) NDArray

	// Return the result of dividing each array element by a scalar value
	ItemDiv(value float64) NDArray

	// Return the reuslt of multiplying each array element by a scalar value
	ItemProd(value float64) NDArray

	// Return the result of subtracting a scalar value from each array element
	ItemSub(value float64) NDArray

	// Set an array element
	ItemSet(value float64, index ...int)

	// Return an iterator over populated matrix entries
	Iter() CoordNDArrayIterator

	// Returns the array as a matrix. This is only possible for 1D and 2D arrays;
	// 1D arrays of length n are converted into n x 1 vectors.
	M() Matrix

	// Get the value of the largest array element
	Max() float64

	// Get the value of the smallest array element
	Min() float64

	// Return the element-wise product of this array and one or more others
	Prod(others ...NDArray) NDArray

	// The number of dimensions in the matrix
	NDim() int

	// Return a copy of the array, normalized to sum to 1
	Normalize() NDArray

	// Get a 1D copy of the array, in 'C' order: rightmost axes change fastest
	Ravel() NDArray

	// A slice giving the size of all array dimensions
	Shape() []int

	// The total number of elements in the matrix
	Size() int

	// Get an array containing a rectangular slice of this array.
	// `from` and `to` should both have one index per axis. The indices
	// in `from` and `to` define the first and just-past-last indices you wish
	// to select along each axis. Negative indexing is supported: when slicing,
	// index -1 refers to the item just past the last and -arr.Size() refers to
	// the first element.
	Slice(from []int, to []int) NDArray

	// Ask whether the matrix has a sparse representation (useful for optimization)
	Sparsity() ArraySparsity

	// Return the element-wise difference of this array and one or more others
	Sub(others ...NDArray) NDArray

	// Return the sum of all array elements
	Sum() float64

	// Return the same matrix, but with axes transposed. The same data is used,
	// for speed and memory efficiency. Use Copy() to create a new array.
	// A 1D array is unchanged; create a 2D analog to rotate a vector.
	T() NDArray
}

// Create an array from literal data
func A(shape []int, values ...float64) NDArray {
	size := 1
	for _, sz := range shape {
		size *= sz
	}
	if len(values) != size {
		panic(fmt.Sprintf("Expected %d array elements but got %d", size, len(values)))
	}
	array := &DenseF64Array{
		shape: shape,
		array: make([]float64, len(values)),
	}
	copy(array.array[:], values[:])
	return array
}

// Create a 1D array
func A1(values ...float64) NDArray {
	return A([]int{len(values)}, values...)
}

// Create a 2D array
func A2(values ...[]float64) NDArray {
	array := &DenseF64Array{
		shape: []int{len(values), len(values[0])},
		array: make([]float64, len(values)*len(values[0])),
	}
	for i0 := 0; i0 < array.shape[0]; i0++ {
		if len(values[i0]) != array.shape[1] {
			panic(fmt.Sprintf("A2 got inconsistent array lengths %d and %d", array.shape[1], len(values[i0])))
		}
		for i1 := 0; i1 < array.shape[1]; i1++ {
			array.ItemSet(values[i0][i1], i0, i1)
		}
	}
	return array
}

// Create an NDArray of float64 values, initialized to zero
func Dense(size ...int) NDArray {
	totalSize := 1
	for _, sz := range size {
		totalSize *= sz
	}
	return &DenseF64Array{
		shape: size,
		array: make([]float64, totalSize),
	}
}

// Create an NDArray of float64 values, initialized to value
func WithValue(value float64, size ...int) NDArray {
	array := Dense(size...)
	array.Fill(value)
	return array
}

// Create an NDArray of float64 values, initialized to zero
func Zeros(size ...int) NDArray {
	return Dense(size...)
}

// Create an NDArray of float64 values, initialized to one
func Ones(size ...int) NDArray {
	return WithValue(1.0, size...)
}

// Create a dense NDArray of float64 values, initialized to uniformly random
// values in [0, 1).
func Rand(size ...int) NDArray {
	array := Dense(size...)

	max := array.Size()
	for i := 0; i < max; i++ {
		array.FlatItemSet(rand.Float64(), i)
	}

	return array
}

// Create a dense NDArray of float64 values, initialized to random values in
// [-math.MaxFloat64, +math.MaxFloat64] distributed on the standard Normal
// distribution.
func RandN(size ...int) NDArray {
	array := Dense(size...)

	max := array.Size()
	for i := 0; i < max; i++ {
		array.FlatItemSet(rand.NormFloat64(), i)
	}

	return array
}
<|endoftext|>


<commit_before>package ioutilmore

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	"github.com/grokify/gotilla/encoding/jsonutil"
	"github.com/grokify/gotilla/type/maputil"
)

func CopyFile(src, dst string) (err error) {
	r, err := os.Open(src)
	if err != nil {
		return
	}
	defer r.Close()

	w, err := os.Create(dst)
	if err != nil {
		return
	}
	defer func() {
		if e := w.Close(); e != nil {
			err = e
		}
	}()

	_, err = io.Copy(w, r)
	if err != nil {
		return
	}

	err = w.Sync()
	if err != nil {
		return
	}

	si, err := os.Stat(src)
	if err != nil {
		return
	}
	err = os.Chmod(dst, si.Mode())
	if err != nil {
		return
	}

	return
}

func ReadDirSplit(dirname string, inclDotDirs bool) ([]os.FileInfo, []os.FileInfo, error) {
	all, err := ioutil.ReadDir(dirname)
	if err != nil {
		return []os.FileInfo{}, []os.FileInfo{}, err
	}
	subdirs, regular := FileInfosSplit(all, inclDotDirs)
	return subdirs, regular, nil
}

func FileInfosSplit(all []os.FileInfo, inclDotDirs bool) ([]os.FileInfo, []os.FileInfo) {
	subdirs := []os.FileInfo{}
	regular := []os.FileInfo{}
	for _, f := range all {
		if f.Mode().IsDir() {
			if f.Name() == "." && f.Name() == ".." {
				if inclDotDirs {
					subdirs = append(subdirs, f)
				}
			} else {
				subdirs = append(subdirs, f)
			}
		} else {
			regular = append(regular, f)
		}
	}
	return subdirs, regular
}

func DirEntriesReSizeGt0(dir string, rx1 *regexp.Regexp) ([]os.FileInfo, error) {
	filesMatch := []os.FileInfo{}
	filesAll, e := ioutil.ReadDir(dir)
	if e != nil {
		return filesMatch, e
	}
	for _, f := range filesAll {
		if f.Name() == "." || f.Name() == ".." {
			continue
		}
		if f.Size() > int64(0) {
			rs1 := rx1.FindStringSubmatch(f.Name())
			if len(rs1) > 0 {
				filesMatch = append(filesMatch, f)
			}
		}
	}
	return filesMatch, nil
}

// DirEntriesRegexpGreatest takes a directory, regular expression and boolean to indicate
// whether to include zero size files and returns the greatest of a single match in the
// regular expression.
func DirFilesRegexpSubmatchGreatest(dir string, rx1 *regexp.Regexp, nonZeroFilesOnly bool) ([]os.FileInfo, error) {
	files := map[string][]os.FileInfo{}

	filesAll, e := ioutil.ReadDir(dir)
	if e != nil {
		return []os.FileInfo{}, e
	}
	for _, f := range filesAll {
		if f.Name() == "." || f.Name() == ".." ||
			(nonZeroFilesOnly && f.Size() <= int64(0)) {
			continue
		}

		if rs1 := rx1.FindStringSubmatch(f.Name()); len(rs1) > 1 {
			extract := rs1[1]
			if _, ok := files[extract]; !ok {
				files[extract] = []os.FileInfo{}
			}
			files[extract] = append(files[extract], f)
		}
	}
	keysSorted := maputil.StringKeysSorted(files)
	greatest := keysSorted[len(keysSorted)-1]
	return files[greatest], nil
}

// DirFilesRegexpSubmatchGreatestSubmatch takes a directory, regular expression and boolean to indicate
// whether to include zero size files and returns the greatest of a single match in the
// regular expression.
func DirFilesRegexpSubmatchGreatestSubmatch(dir string, rx1 *regexp.Regexp, nonZeroFilesOnly bool) (string, error) {
	filesAll, err := ioutil.ReadDir(dir)
	if err != nil {
		return "", err
	}
	strs := []string{}
	for _, f := range filesAll {
		if nonZeroFilesOnly && f.Size() <= int64(0) {
			continue
		}
		rs1 := rx1.FindStringSubmatch(f.Name())
		if len(rs1) > 1 {
			strs = append(strs, rs1[1])
		}
	}
	sort.Strings(strs)
	if len(strs) == 0 {
		return "", fmt.Errorf("No matches found")
	}
	return strs[len(strs)-1], nil
}

func DirFromPath(path string) (string, error) {
	path = strings.TrimRight(path, "/\\")
	f, err := os.Open(path)
	if err != nil {
		return "", err
	}
	defer f.Close()
	fi, err := f.Stat()
	if err != nil {
		return "", err
	}
	isFile := false
	switch mode := fi.Mode(); {
	case mode.IsDir():
		return path, nil
	case mode.IsRegular():
		isFile = true
	}
	if isFile == false {
		return "", nil
	}
	rx1 := regexp.MustCompile(`^(.+)[/\\][^/\\]+`)
	rs1 := rx1.FindStringSubmatch(path)
	dir := ""
	if len(rs1) > 1 {
		dir = rs1[1]
	}
	return dir, nil
}

func IsDir(name string) (bool, error) {
	if fi, err := os.Stat(name); err != nil {
		return false, err
	} else if !fi.Mode().IsDir() {
		return false, nil
	}
	return true, nil
}

func IsFile(name string) (bool, error) {
	if fi, err := os.Stat(name); err != nil {
		return false, err
	} else if !fi.Mode().IsRegular() {
		return false, nil
	}
	return true, nil
}

func Exists(name string) (bool, error) {
	_, err := os.Stat(name)
	if os.IsNotExist(err) {
		return false, nil
	}
	return err != nil, err
}

// IsFileWithSizeGtZero verifies a path exists, is a file and is not empty,
// returning an error otherwise. An os file not exists check can be done
// with os.IsNotExist(err) which acts on error from os.Stat()
func IsFileWithSizeGtZero(name string) (bool, error) {
	if fi, err := os.Stat(name); err != nil {
		return false, err
	} else if !fi.Mode().IsRegular() {
		return false, nil
		// return fmt.Errorf("Filepath [%v] exists but is not a file.", name)
	} else if fi.Size() <= 0 {
		return false, nil
		// return fmt.Errorf("Filepath [%v] exists but is empty with size [%v].", name, fi.Size())
	}
	return true, nil
}

func SplitBetter(path string) (dir, file string) {
	isDir, err := IsDir(path)
	if err != nil && isDir {
		return dir, ""
	}
	return filepath.Split(path)
}

func SplitBest(path string) (dir, file string, err error) {
	isDir, err := IsDir(path)
	if err != nil {
		return "", "", err
	} else if isDir {
		return path, "", nil
	}
	isFile, err := IsFile(path)
	if err != nil {
		return "", "", err
	} else if isFile {
		dir, file := filepath.Split(path)
		return dir, file, nil
	}
	return "", "", fmt.Errorf("Path is valid but not file or directory: [%v]", path)
}

func FilterFilenamesSizeGtZero(filepaths ...string) []string {
	filepathsExist := []string{}

	for _, envPathVal := range filepaths {
		envPathVals := strings.Split(envPathVal, ",")
		for _, envPath := range envPathVals {
			envPath = strings.TrimSpace(envPath)

			if isFile, err := IsFileWithSizeGtZero(envPath); isFile && err == nil {
				filepathsExist = append(filepathsExist, envPath)
			}
		}
	}
	return filepathsExist
}

func RemoveAllChildren(dir string) error {
	isDir, err := IsDir(dir)
	if err != nil {
		return err
	}
	if isDir == false {
		err = errors.New("400: Path Is Not Directory")
		return err
	}
	filesAll, err := ioutil.ReadDir(dir)
	if err != nil {
		return err
	}
	for _, fi := range filesAll {
		if fi.Name() == "." || fi.Name() == ".." {
			continue
		}
		filepath := path.Join(dir, fi.Name())
		if fi.IsDir() {
			err = os.RemoveAll(filepath)
			if err != nil {
				return err
			}
		} else {
			err = os.Remove(filepath)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func FileinfosNames(fis []os.FileInfo) []string {
	s := []string{}
	for _, e := range fis {
		s = append(s, e.Name())
	}
	return s
}

// ReaderToBytes reads from an io.Reader, e.g. io.ReadCloser
func ReaderToBytes(ior io.Reader) []byte {
	buf := new(bytes.Buffer)
	buf.ReadFrom(ior)
	return buf.Bytes()
}

// ReadFileJSON reads and unmarshals a file.
func ReadFileJSON(file string, v interface{}) error {
	bytes, err := ioutil.ReadFile(file)
	if err != nil {
		return err
	}
	return json.Unmarshal(bytes, v)
}

func WriteFileJSON(filepath string, data interface{}, perm os.FileMode, prefix, indent string) error {
	var bytes []byte
	bytes, err := jsonutil.MarshalSimple(data, prefix, indent)
	if err != nil {
		return err
	}
	/*
		if wantPretty {
			bytesTry, err := json.MarshalIndent(data, "", "  ")
			if err != nil {
				return err
			}
			bytes = bytesTry
		} else {
			bytesTry, err := json.Marshal(data)
			if err != nil {
				return err
			}
			bytes = bytesTry
		}*/
	return ioutil.WriteFile(filepath, bytes, perm)
}

type FileWriter struct {
	File   *os.File
	Writer *bufio.Writer
}

func NewFileWriter(path string) (FileWriter, error) {
	fw := FileWriter{}
	file, err := os.Create(path)
	if err != nil {
		return fw, err
	}

	fw.File = file
	fw.Writer = bufio.NewWriter(file)

	return fw, nil
}

func (f *FileWriter) Close() {
	f.Writer.Flush()
	f.File.Close()
}
<commit_msg>add ioutilmore.FileinfosToFilepaths<commit_after>package ioutilmore

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	"github.com/grokify/gotilla/encoding/jsonutil"
	"github.com/grokify/gotilla/type/maputil"
)

func CopyFile(src, dst string) (err error) {
	r, err := os.Open(src)
	if err != nil {
		return
	}
	defer r.Close()

	w, err := os.Create(dst)
	if err != nil {
		return
	}
	defer func() {
		if e := w.Close(); e != nil {
			err = e
		}
	}()

	_, err = io.Copy(w, r)
	if err != nil {
		return
	}

	err = w.Sync()
	if err != nil {
		return
	}

	si, err := os.Stat(src)
	if err != nil {
		return
	}
	err = os.Chmod(dst, si.Mode())
	if err != nil {
		return
	}

	return
}

func ReadDirSplit(dirname string, inclDotDirs bool) ([]os.FileInfo, []os.FileInfo, error) {
	all, err := ioutil.ReadDir(dirname)
	if err != nil {
		return []os.FileInfo{}, []os.FileInfo{}, err
	}
	subdirs, regular := FileInfosSplit(all, inclDotDirs)
	return subdirs, regular, nil
}

func FileInfosSplit(all []os.FileInfo, inclDotDirs bool) ([]os.FileInfo, []os.FileInfo) {
	subdirs := []os.FileInfo{}
	regular := []os.FileInfo{}
	for _, f := range all {
		if f.Mode().IsDir() {
			if f.Name() == "." && f.Name() == ".." {
				if inclDotDirs {
					subdirs = append(subdirs, f)
				}
			} else {
				subdirs = append(subdirs, f)
			}
		} else {
			regular = append(regular, f)
		}
	}
	return subdirs, regular
}

func DirEntriesReSizeGt0(dir string, rx1 *regexp.Regexp) ([]os.FileInfo, error) {
	filesMatch := []os.FileInfo{}
	filesAll, e := ioutil.ReadDir(dir)
	if e != nil {
		return filesMatch, e
	}
	for _, f := range filesAll {
		if f.Name() == "." || f.Name() == ".." {
			continue
		}
		if f.Size() > int64(0) {
			rs1 := rx1.FindStringSubmatch(f.Name())
			if len(rs1) > 0 {
				filesMatch = append(filesMatch, f)
			}
		}
	}
	return filesMatch, nil
}

// DirEntriesRegexpGreatest takes a directory, regular expression and boolean to indicate
// whether to include zero size files and returns the greatest of a single match in the
// regular expression.
func DirFilesRegexpSubmatchGreatest(dir string, rx1 *regexp.Regexp, nonZeroFilesOnly bool) ([]os.FileInfo, error) {
	files := map[string][]os.FileInfo{}

	filesAll, e := ioutil.ReadDir(dir)
	if e != nil {
		return []os.FileInfo{}, e
	}
	for _, f := range filesAll {
		if f.Name() == "." || f.Name() == ".." ||
			(nonZeroFilesOnly && f.Size() <= int64(0)) {
			continue
		}

		if rs1 := rx1.FindStringSubmatch(f.Name()); len(rs1) > 1 {
			extract := rs1[1]
			if _, ok := files[extract]; !ok {
				files[extract] = []os.FileInfo{}
			}
			files[extract] = append(files[extract], f)
		}
	}
	keysSorted := maputil.StringKeysSorted(files)
	greatest := keysSorted[len(keysSorted)-1]
	return files[greatest], nil
}

// DirFilesRegexpSubmatchGreatestSubmatch takes a directory, regular expression and boolean to indicate
// whether to include zero size files and returns the greatest of a single match in the
// regular expression.
func DirFilesRegexpSubmatchGreatestSubmatch(dir string, rx1 *regexp.Regexp, nonZeroFilesOnly bool) (string, error) {
	filesAll, err := ioutil.ReadDir(dir)
	if err != nil {
		return "", err
	}
	strs := []string{}
	for _, f := range filesAll {
		if nonZeroFilesOnly && f.Size() <= int64(0) {
			continue
		}
		rs1 := rx1.FindStringSubmatch(f.Name())
		if len(rs1) > 1 {
			strs = append(strs, rs1[1])
		}
	}
	sort.Strings(strs)
	if len(strs) == 0 {
		return "", fmt.Errorf("No matches found")
	}
	return strs[len(strs)-1], nil
}

func DirFromPath(path string) (string, error) {
	path = strings.TrimRight(path, "/\\")
	f, err := os.Open(path)
	if err != nil {
		return "", err
	}
	defer f.Close()
	fi, err := f.Stat()
	if err != nil {
		return "", err
	}
	isFile := false
	switch mode := fi.Mode(); {
	case mode.IsDir():
		return path, nil
	case mode.IsRegular():
		isFile = true
	}
	if isFile == false {
		return "", nil
	}
	rx1 := regexp.MustCompile(`^(.+)[/\\][^/\\]+`)
	rs1 := rx1.FindStringSubmatch(path)
	dir := ""
	if len(rs1) > 1 {
		dir = rs1[1]
	}
	return dir, nil
}

func IsDir(name string) (bool, error) {
	if fi, err := os.Stat(name); err != nil {
		return false, err
	} else if !fi.Mode().IsDir() {
		return false, nil
	}
	return true, nil
}

func IsFile(name string) (bool, error) {
	if fi, err := os.Stat(name); err != nil {
		return false, err
	} else if !fi.Mode().IsRegular() {
		return false, nil
	}
	return true, nil
}

func Exists(name string) (bool, error) {
	_, err := os.Stat(name)
	if os.IsNotExist(err) {
		return false, nil
	}
	return err != nil, err
}

// IsFileWithSizeGtZero verifies a path exists, is a file and is not empty,
// returning an error otherwise. An os file not exists check can be done
// with os.IsNotExist(err) which acts on error from os.Stat()
func IsFileWithSizeGtZero(name string) (bool, error) {
	if fi, err := os.Stat(name); err != nil {
		return false, err
	} else if !fi.Mode().IsRegular() {
		return false, nil
		// return fmt.Errorf("Filepath [%v] exists but is not a file.", name)
	} else if fi.Size() <= 0 {
		return false, nil
		// return fmt.Errorf("Filepath [%v] exists but is empty with size [%v].", name, fi.Size())
	}
	return true, nil
}

func SplitBetter(path string) (dir, file string) {
	isDir, err := IsDir(path)
	if err != nil && isDir {
		return dir, ""
	}
	return filepath.Split(path)
}

func SplitBest(path string) (dir, file string, err error) {
	isDir, err := IsDir(path)
	if err != nil {
		return "", "", err
	} else if isDir {
		return path, "", nil
	}
	isFile, err := IsFile(path)
	if err != nil {
		return "", "", err
	} else if isFile {
		dir, file := filepath.Split(path)
		return dir, file, nil
	}
	return "", "", fmt.Errorf("Path is valid but not file or directory: [%v]", path)
}

func FileinfosToFilepaths(dir string, fileInfos []os.FileInfo) []string {
	dir = strings.TrimSpace(dir)
	paths := []string{}
	for _, fi := range fileInfos {
		if len(dir) > 0 {
			paths = append(paths, filepath.Join(dir, fi.Name()))
		} else {
			paths = append(paths, fi.Name())
		}
	}
	return paths
}

func FilterFilenamesSizeGtZero(filepaths ...string) []string {
	filepathsExist := []string{}

	for _, envPathVal := range filepaths {
		envPathVals := strings.Split(envPathVal, ",")
		for _, envPath := range envPathVals {
			envPath = strings.TrimSpace(envPath)

			if isFile, err := IsFileWithSizeGtZero(envPath); isFile && err == nil {
				filepathsExist = append(filepathsExist, envPath)
			}
		}
	}
	return filepathsExist
}

func RemoveAllChildren(dir string) error {
	isDir, err := IsDir(dir)
	if err != nil {
		return err
	}
	if isDir == false {
		err = errors.New("400: Path Is Not Directory")
		return err
	}
	filesAll, err := ioutil.ReadDir(dir)
	if err != nil {
		return err
	}
	for _, fi := range filesAll {
		if fi.Name() == "." || fi.Name() == ".." {
			continue
		}
		filepath := path.Join(dir, fi.Name())
		if fi.IsDir() {
			err = os.RemoveAll(filepath)
			if err != nil {
				return err
			}
		} else {
			err = os.Remove(filepath)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func FileinfosNames(fis []os.FileInfo) []string {
	s := []string{}
	for _, e := range fis {
		s = append(s, e.Name())
	}
	return s
}

// ReaderToBytes reads from an io.Reader, e.g. io.ReadCloser
func ReaderToBytes(ior io.Reader) []byte {
	buf := new(bytes.Buffer)
	buf.ReadFrom(ior)
	return buf.Bytes()
}

// ReadFileJSON reads and unmarshals a file.
func ReadFileJSON(file string, v interface{}) error {
	bytes, err := ioutil.ReadFile(file)
	if err != nil {
		return err
	}
	return json.Unmarshal(bytes, v)
}

func WriteFileJSON(filepath string, data interface{}, perm os.FileMode, prefix, indent string) error {
	var bytes []byte
	bytes, err := jsonutil.MarshalSimple(data, prefix, indent)
	if err != nil {
		return err
	}
	/*
		if wantPretty {
			bytesTry, err := json.MarshalIndent(data, "", "  ")
			if err != nil {
				return err
			}
			bytes = bytesTry
		} else {
			bytesTry, err := json.Marshal(data)
			if err != nil {
				return err
			}
			bytes = bytesTry
		}*/
	return ioutil.WriteFile(filepath, bytes, perm)
}

type FileWriter struct {
	File   *os.File
	Writer *bufio.Writer
}

func NewFileWriter(path string) (FileWriter, error) {
	fw := FileWriter{}
	file, err := os.Create(path)
	if err != nil {
		return fw, err
	}

	fw.File = file
	fw.Writer = bufio.NewWriter(file)

	return fw, nil
}

func (f *FileWriter) Close() {
	f.Writer.Flush()
	f.File.Close()
}
<|endoftext|>