<commit_before>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package containerd

import (
	"context"
	"fmt"
	"runtime"
	"strings"
	"testing"

	. "github.com/containerd/containerd"
	"github.com/containerd/containerd/errdefs"
	"github.com/containerd/containerd/images"
	"github.com/containerd/containerd/platforms"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)

func TestImageIsUnpacked(t *testing.T) {
	if runtime.GOOS == "windows" {
		t.Skip()
	}

	const imageName = "k8s.gcr.io/pause:3.4.1"
	ctx, cancel := testContext(t)
	defer cancel()

	client, err := newClient(t, address)
	if err != nil {
		t.Fatal(err)
	}
	defer client.Close()

	// Cleanup
	opts := []images.DeleteOpt{images.SynchronousDelete()}
	err = client.ImageService().Delete(ctx, imageName, opts...)
	if err != nil && !errdefs.IsNotFound(err) {
		t.Fatal(err)
	}

	// By default pull does not unpack an image
	image, err := client.Pull(ctx, imageName, WithPlatform("linux/amd64"))
	if err != nil {
		t.Fatal(err)
	}

	// Check that image is not unpacked
	unpacked, err := image.IsUnpacked(ctx, DefaultSnapshotter)
	if err != nil {
		t.Fatal(err)
	}
	if unpacked {
		t.Fatalf("image should not be unpacked")
	}

	// Check that image is unpacked
	err = image.Unpack(ctx, DefaultSnapshotter)
	if err != nil {
		t.Fatal(err)
	}
	unpacked, err = image.IsUnpacked(ctx, DefaultSnapshotter)
	if err != nil {
		t.Fatal(err)
	}
	if !unpacked {
		t.Fatalf("image should be unpacked")
	}
}

func TestImagePullWithDistSourceLabel(t *testing.T) {
	if runtime.GOOS == "windows" {
		t.Skip()
	}
	var (
		source   = "k8s.gcr.io"
		repoName = "pause"
		tag      = "3.4.1"
	)

	ctx, cancel := testContext(t)
	defer cancel()

	client, err := newClient(t, address)
	if err != nil {
		t.Fatal(err)
	}
	defer client.Close()

	imageName := fmt.Sprintf("%s/%s:%s", source, repoName, tag)
	pMatcher := platforms.Default()

	// pull content without unpack and add distribution source label
	image, err := client.Pull(ctx, imageName, WithPlatformMatcher(pMatcher))
	if err != nil {
		t.Fatal(err)
	}
	defer client.ImageService().Delete(ctx, imageName)

	cs := client.ContentStore()
	key := fmt.Sprintf("containerd.io/distribution.source.%s", source)

	// only check the target platform
	childrenHandler := images.LimitManifests(images.ChildrenHandler(cs), pMatcher, 1)

	checkLabelHandler := func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
		children, err := childrenHandler(ctx, desc)
		if err != nil {
			return nil, err
		}

		info, err := cs.Info(ctx, desc.Digest)
		if err != nil {
			return nil, err
		}

		// check the label
		if got := info.Labels[key]; !strings.Contains(got, repoName) {
			return nil, fmt.Errorf("expected to have %s repo name in label, but got %s", repoName, got)
		}
		return children, nil
	}

	if err := images.Dispatch(ctx, images.HandlerFunc(checkLabelHandler), nil, image.Target()); err != nil {
		t.Fatal(err)
	}
}

func TestImageUsage(t *testing.T) {
	if testing.Short() || runtime.GOOS == "windows" {
		t.Skip()
	}

	imageName := "k8s.gcr.io/pause:3.4.1"
	ctx, cancel := testContext(t)
	defer cancel()

	client, err := newClient(t, address)
	if err != nil {
		t.Fatal(err)
	}
	defer client.Close()

	// Cleanup
	err = client.ImageService().Delete(ctx, imageName, images.SynchronousDelete())
	if err != nil && !errdefs.IsNotFound(err) {
		t.Fatal(err)
	}

	testPlatform := platforms.Only(ocispec.Platform{
		OS:           "linux",
		Architecture: "amd64",
	})

	// Pull single platform, do not unpack
	image, err := client.Pull(ctx, imageName, WithPlatformMatcher(testPlatform))
	if err != nil {
		t.Fatal(err)
	}

	s1, err := image.Usage(ctx, WithUsageManifestLimit(1))
	if err != nil {
		t.Fatal(err)
	}

	if _, err := image.Usage(ctx, WithUsageManifestLimit(0), WithManifestUsage()); err == nil {
		t.Fatal("expected NotFound with missing manifests")
	} else if !errdefs.IsNotFound(err) {
		t.Fatalf("unexpected error: %+v", err)
	}

	// Pin image name to specific version for future fetches
	imageName = imageName + "@" + image.Target().Digest.String()
	defer client.ImageService().Delete(ctx, imageName, images.SynchronousDelete())

	// Fetch single platforms, but all manifests pulled
	if _, err := client.Fetch(ctx, imageName, WithPlatformMatcher(testPlatform), WithAllMetadata()); err != nil {
		t.Fatal(err)
	}

	if s, err := image.Usage(ctx, WithUsageManifestLimit(1)); err != nil {
		t.Fatal(err)
	} else if s != s1 {
		t.Fatalf("unexpected usage %d, expected %d", s, s1)
	}

	s2, err := image.Usage(ctx, WithUsageManifestLimit(0))
	if err != nil {
		t.Fatal(err)
	}

	if s2 <= s1 {
		t.Fatalf("Expected larger usage counting all manifests: %d <= %d", s2, s1)
	}

	s3, err := image.Usage(ctx, WithUsageManifestLimit(0), WithManifestUsage())
	if err != nil {
		t.Fatal(err)
	}

	if s3 <= s2 {
		t.Fatalf("Expected larger usage counting all manifest reported sizes: %d <= %d", s3, s2)
	}

	// Fetch everything
	if _, err = client.Fetch(ctx, imageName); err != nil {
		t.Fatal(err)
	}

	if s, err := image.Usage(ctx); err != nil {
		t.Fatal(err)
	} else if s != s3 {
		t.Fatalf("Expected actual usage to equal manifest reported usage of %d: got %d", s3, s)
	}

	err = image.Unpack(ctx, DefaultSnapshotter)
	if err != nil {
		t.Fatal(err)
	}

	if s, err := image.Usage(ctx, WithSnapshotUsage()); err != nil {
		t.Fatal(err)
	} else if s <= s3 {
		t.Fatalf("Expected actual usage with snapshots to be greater: %d <= %d", s, s3)
	}
}

func TestImageSupportedBySnapshotter_Error(t *testing.T) {
	var unsupportedImage string
	if runtime.GOOS == "windows" {
		unsupportedImage = "k8s.gcr.io/pause-amd64:3.2"
	} else {
		unsupportedImage = "mcr.microsoft.com/windows/nanoserver:1809"
	}

	ctx, cancel := testContext(t)
	defer cancel()

	client, err := newClient(t, address)
	if err != nil {
		t.Fatal(err)
	}
	defer client.Close()

	// Cleanup
	err = client.ImageService().Delete(ctx, unsupportedImage)
	if err != nil && !errdefs.IsNotFound(err) {
		t.Fatal(err)
	}

	_, err = client.Pull(ctx, unsupportedImage,
		WithSchema1Conversion,
		WithPlatform(platforms.DefaultString()),
		WithPullSnapshotter(DefaultSnapshotter),
		WithPullUnpack,
		WithUnpackOpts([]UnpackOpt{WithSnapshotterPlatformCheck()}),
	)

	if err == nil {
		t.Fatalf("expected unpacking %s for snapshotter %s to fail", unsupportedImage, DefaultSnapshotter)
	}
}
<commit_msg>Enable some image-pull tests on Windows<commit_after>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package containerd

import (
	"context"
	"fmt"
	"runtime"
	"strings"
	"testing"

	. "github.com/containerd/containerd"
	"github.com/containerd/containerd/errdefs"
	"github.com/containerd/containerd/images"
	"github.com/containerd/containerd/platforms"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)

func TestImageIsUnpacked(t *testing.T) {
	const imageName = "k8s.gcr.io/pause:3.4.1"
	ctx, cancel := testContext(t)
	defer cancel()

	client, err := newClient(t, address)
	if err != nil {
		t.Fatal(err)
	}
	defer client.Close()

	// Cleanup
	opts := []images.DeleteOpt{images.SynchronousDelete()}
	err = client.ImageService().Delete(ctx, imageName, opts...)
	if err != nil && !errdefs.IsNotFound(err) {
		t.Fatal(err)
	}

	// By default pull does not unpack an image
	image, err := client.Pull(ctx, imageName, WithPlatformMatcher(platforms.Default()))
	if err != nil {
		t.Fatal(err)
	}

	// Check that image is not unpacked
	unpacked, err := image.IsUnpacked(ctx, DefaultSnapshotter)
	if err != nil {
		t.Fatal(err)
	}
	if unpacked {
		t.Fatalf("image should not be unpacked")
	}

	// Check that image is unpacked
	err = image.Unpack(ctx, DefaultSnapshotter)
	if err != nil {
		t.Fatal(err)
	}
	unpacked, err = image.IsUnpacked(ctx, DefaultSnapshotter)
	if err != nil {
		t.Fatal(err)
	}
	if !unpacked {
		t.Fatalf("image should be unpacked")
	}
}

func TestImagePullWithDistSourceLabel(t *testing.T) {
	var (
		source   = "k8s.gcr.io"
		repoName = "pause"
		tag      = "3.4.1"
	)

	ctx, cancel := testContext(t)
	defer cancel()

	client, err := newClient(t, address)
	if err != nil {
		t.Fatal(err)
	}
	defer client.Close()

	imageName := fmt.Sprintf("%s/%s:%s", source, repoName, tag)
	pMatcher := platforms.Default()

	// pull content without unpack and add distribution source label
	image, err := client.Pull(ctx, imageName, WithPlatformMatcher(pMatcher))
	if err != nil {
		t.Fatal(err)
	}
	defer client.ImageService().Delete(ctx, imageName)

	cs := client.ContentStore()
	key := fmt.Sprintf("containerd.io/distribution.source.%s", source)

	// only check the target platform
	childrenHandler := images.LimitManifests(images.ChildrenHandler(cs), pMatcher, 1)

	checkLabelHandler := func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
		children, err := childrenHandler(ctx, desc)
		if err != nil {
			return nil, err
		}

		info, err := cs.Info(ctx, desc.Digest)
		if err != nil {
			return nil, err
		}

		// check the label
		if got := info.Labels[key]; !strings.Contains(got, repoName) {
			return nil, fmt.Errorf("expected to have %s repo name in label, but got %s", repoName, got)
		}
		return children, nil
	}

	if err := images.Dispatch(ctx, images.HandlerFunc(checkLabelHandler), nil, image.Target()); err != nil {
		t.Fatal(err)
	}
}

func TestImageUsage(t *testing.T) {
	if testing.Short() {
		t.Skip()
	}

	imageName := "k8s.gcr.io/pause:3.4.1"
	ctx, cancel := testContext(t)
	defer cancel()

	client, err := newClient(t, address)
	if err != nil {
		t.Fatal(err)
	}
	defer client.Close()

	// Cleanup
	err = client.ImageService().Delete(ctx, imageName, images.SynchronousDelete())
	if err != nil && !errdefs.IsNotFound(err) {
		t.Fatal(err)
	}

	pMatcher := platforms.Default()

	// Pull single platform, do not unpack
	image, err := client.Pull(ctx, imageName, WithPlatformMatcher(pMatcher))
	if err != nil {
		t.Fatal(err)
	}

	s1, err := image.Usage(ctx, WithUsageManifestLimit(1))
	if err != nil {
		t.Fatal(err)
	}

	if _, err := image.Usage(ctx, WithUsageManifestLimit(0), WithManifestUsage()); err == nil {
		t.Fatal("expected NotFound with missing manifests")
	} else if !errdefs.IsNotFound(err) {
		t.Fatalf("unexpected error: %+v", err)
	}

	// Pin image name to specific version for future fetches
	imageName = imageName + "@" + image.Target().Digest.String()
	defer client.ImageService().Delete(ctx, imageName, images.SynchronousDelete())

	// Fetch single platforms, but all manifests pulled
	if _, err := client.Fetch(ctx, imageName, WithPlatformMatcher(pMatcher), WithAllMetadata()); err != nil {
		t.Fatal(err)
	}

	if s, err := image.Usage(ctx, WithUsageManifestLimit(1)); err != nil {
		t.Fatal(err)
	} else if s != s1 {
		t.Fatalf("unexpected usage %d, expected %d", s, s1)
	}

	s2, err := image.Usage(ctx, WithUsageManifestLimit(0))
	if err != nil {
		t.Fatal(err)
	}

	if s2 <= s1 {
		t.Fatalf("Expected larger usage counting all manifests: %d <= %d", s2, s1)
	}

	s3, err := image.Usage(ctx, WithUsageManifestLimit(0), WithManifestUsage())
	if err != nil {
		t.Fatal(err)
	}

	if s3 <= s2 {
		t.Fatalf("Expected larger usage counting all manifest reported sizes: %d <= %d", s3, s2)
	}

	// Fetch everything
	if _, err = client.Fetch(ctx, imageName); err != nil {
		t.Fatal(err)
	}

	if s, err := image.Usage(ctx); err != nil {
		t.Fatal(err)
	} else if s != s3 {
		t.Fatalf("Expected actual usage to equal manifest reported usage of %d: got %d", s3, s)
	}

	err = image.Unpack(ctx, DefaultSnapshotter)
	if err != nil {
		t.Fatal(err)
	}

	if s, err := image.Usage(ctx, WithSnapshotUsage()); err != nil {
		t.Fatal(err)
	} else if s <= s3 {
		t.Fatalf("Expected actual usage with snapshots to be greater: %d <= %d", s, s3)
	}
}

func TestImageSupportedBySnapshotter_Error(t *testing.T) {
	var unsupportedImage string
	if runtime.GOOS == "windows" {
		unsupportedImage = "k8s.gcr.io/pause-amd64:3.2"
	} else {
		unsupportedImage = "mcr.microsoft.com/windows/nanoserver:1809"
	}

	ctx, cancel := testContext(t)
	defer cancel()

	client, err := newClient(t, address)
	if err != nil {
		t.Fatal(err)
	}
	defer client.Close()

	// Cleanup
	err = client.ImageService().Delete(ctx, unsupportedImage)
	if err != nil && !errdefs.IsNotFound(err) {
		t.Fatal(err)
	}

	_, err = client.Pull(ctx, unsupportedImage,
		WithSchema1Conversion,
		WithPlatform(platforms.DefaultString()),
		WithPullSnapshotter(DefaultSnapshotter),
		WithPullUnpack,
		WithUnpackOpts([]UnpackOpt{WithSnapshotterPlatformCheck()}),
	)

	if err == nil {
		t.Fatalf("expected unpacking %s for snapshotter %s to fail", unsupportedImage, DefaultSnapshotter)
	}
}
<|endoftext|>


<commit_before>//  Copyright (c) 2015 ikawaha.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package lattice

import (
	"bytes"
	"testing"
	"unicode/utf8"

	"github.com/ikawaha/kagome/internal/dic"
)

func TestLatticeBuild01(t *testing.T) {
	la := New(dic.SysDic(), nil)
	if la == nil {
		t.Error("cannot new a lattice")
	}
	defer la.Free()

	inp := ""
	la.Build(inp)
	if la.Input != inp {
		t.Errorf("got %v, expected %v", la.Input, inp)
	}
	boseos := node{ID: -1}
	if len(la.list) != 2 {
		t.Errorf("lattice initialize error: got %v, expected has 2 eos/bos nodes", la.list)
	} else if len(la.list[0]) != 1 || *la.list[0][0] != boseos {
		t.Errorf("lattice initialize error: got %v, expected %v", *la.list[0][0], boseos)
	} else if len(la.list[1]) != 1 || *la.list[1][0] != boseos {
		t.Errorf("lattice initialize error: got %v, expected %v", *la.list[1][0], boseos)
	}
	if len(la.Output) != 0 {
		t.Errorf("lattice initialize error: got %v, expected empty", la.Output)
	}

	if la.dic == nil {
		t.Errorf("lattice initialize error: dic is nil")
	}
	if la.udic != nil {
		t.Errorf("lattice initialize error: got %v, expected empty", la.udic)
	}
}

func TestLatticeBuild02(t *testing.T) {
	la := New(dic.SysDic(), nil)
	if la == nil {
		t.Fatal("cannot new a lattice")
	}
	defer la.Free()

	inp := "あ"
	la.Build(inp)
	if la.Input != inp {
		t.Errorf("got %v, expected %v", la.Input, inp)
	}
	bos := node{ID: -1}
	eos := node{ID: -1, Start: 1}
	if len(la.list) != 3 {
		t.Errorf("lattice initialize error: got %v, expected has 2 eos/bos nodes", la.list)
	} else if len(la.list[0]) != 1 || *la.list[0][0] != bos {
		t.Errorf("lattice initialize error: got %v, expected %v", *la.list[0][0], bos)
	} else if len(la.list[2]) != 1 || *la.list[2][0] != eos {
		t.Errorf("lattice initialize error: got %v, expected %v", *la.list[2][0], eos)
	}

	expected := 4
	if len(la.list[1]) != expected {
		t.Errorf("lattice initialize error: got %v, expected %v", len(la.list[1]), expected)
	} else {
		l := la.list[1]
		callAndResponse := []struct {
			in  int
			out node
		}{
			{in: 0, out: node{122, 0, KNOWN, 0, 3, 3, 5549, "あ", nil}},
			{in: 1, out: node{123, 0, KNOWN, 0, 776, 776, 6690, "あ", nil}},
			{in: 2, out: node{124, 0, KNOWN, 0, 2, 2, 4262, "あ", nil}},
			{in: 3, out: node{125, 0, KNOWN, 0, 1118, 1118, 9035, "あ", nil}},
		}
		for _, cr := range callAndResponse {
			if *l[cr.in] != cr.out {
				t.Errorf("lattice initialize error: got %v, expected %v", l[cr.in], cr.out)
			}
		}
	}
	if len(la.Output) != 0 {
		t.Errorf("lattice initialize error: got %v, expected empty", la.Output)
	}
	if la.dic == nil {
		t.Errorf("lattice initialize error: dic is nil")
	}
	if la.udic != nil {
		t.Errorf("lattice initialize error: got %v, expected empty", la.udic)
	}
}

func TestLatticeBuild03(t *testing.T) {

	const udicPath = "../../_sample/userdic.txt"

	udic, e := dic.NewUserDic(udicPath)
	if e != nil {
		t.Fatalf("unexpected error: cannot load user dic, %v", e)
	}
	la := New(dic.SysDic(), udic)
	if la == nil {
		t.Fatal("cannot new a lattice")
	}
	defer la.Free()

	inp := "朝青龍"
	la.Build(inp)
	if la.Input != inp {
		t.Errorf("got %v, expected %v", la.Input, inp)
	}

	if la.list[3][0].Class != USER {
		t.Errorf("%+v", la)
	}

	if len(la.Output) != 0 {
		t.Errorf("lattice initialize error: got %v, expected empty", la.Output)
	}
	if la.dic == nil {
		t.Errorf("lattice initialize error: dic is nil")
	}
	if la.udic == nil {
		t.Errorf("lattice initialize error: got %v, expected not empty", la.udic)
	}
}

func TestLatticeBuild04(t *testing.T) {
	la := New(dic.SysDic(), nil)
	if la == nil {
		t.Fatal("cannot new a lattice")
	}
	defer la.Free()

	inp := "ポポピ"
	la.Build(inp)
	if la.Input != inp {
		t.Errorf("got %v, expected %v", la.Input, inp)
	}
	bos := node{ID: -1}
	eos := node{ID: -1, Start: 3}
	if len(la.list) != 5 {
		t.Errorf("lattice initialize error: got %v, expected has 2 eos/bos nodes", la.list)
	} else if len(la.list[0]) != 1 || *la.list[0][0] != bos {
		t.Errorf("lattice initialize error: got %v, expected %v", *la.list[0][0], bos)
	} else if len(la.list[len(la.list)-1]) != 1 || *la.list[len(la.list)-1][0] != eos {
		t.Errorf("lattice initialize error: got %v, expected %v", *la.list[len(la.list)-1][0], eos)
	}

	expected := 7
	if len(la.list[1]) != expected {
		t.Errorf("lattice initialize error: got %v, expected %v", len(la.list[1]), expected)
	} else {
		l := la.list[1]
		callAndResponse := []struct {
			in  int
			out node
		}{
			{in: 0, out: node{98477, 0, KNOWN, 0, 1285, 1285, 4279, "ポ", nil}},
			{in: 1, out: node{31, 0, UNKNOWN, 0, 1289, 1289, 13581, "ポ", nil}},
			{in: 2, out: node{32, 0, UNKNOWN, 0, 1285, 1285, 9461, "ポ", nil}},
			{in: 3, out: node{33, 0, UNKNOWN, 0, 1293, 1293, 13661, "ポ", nil}},
			{in: 4, out: node{34, 0, UNKNOWN, 0, 1292, 1292, 10922, "ポ", nil}},
			{in: 5, out: node{35, 0, UNKNOWN, 0, 1288, 1288, 10521, "ポ", nil}},
			{in: 6, out: node{36, 0, UNKNOWN, 0, 3, 3, 14138, "ポ", nil}},
		}
		for _, cr := range callAndResponse {
			if *l[cr.in] != cr.out {
				t.Errorf("lattice initialize error: got %v, expected %v", l[cr.in], cr.out)
			}
		}
	}
	if len(la.Output) != 0 {
		t.Errorf("lattice initialize error: got %v, expected empty", la.Output)
	}
	if la.dic == nil {
		t.Errorf("lattice initialize error: dic is nil")
	}
	if la.udic != nil {
		t.Errorf("lattice initialize error: got %v, expected empty", la.udic)
	}
}

func TestLatticeBuild05(t *testing.T) {

	la := New(dic.SysDic(), nil)
	if la == nil {
		t.Fatal("cannot new a lattice")
	}
	defer la.Free()

	inp := "ポポピポンポコナーノ"
	var b bytes.Buffer
	for i, step := 0, utf8.RuneCountInString(inp); i < maximumUnknownWordLength; i = i + step {
		b.WriteString(inp)
	}
	la.Build(b.String())
	for i := range la.list {
		for j := range la.list[i] {
			l := utf8.RuneCountInString(la.list[i][j].Surface)
			if l > maximumUnknownWordLength {
				t.Errorf("too long unknown word, %v", l)
			}
		}
	}
}

func TestKanjiOnly01(t *testing.T) {
	callAndResponse := []struct {
		in  string
		out bool
	}{
		{in: "ひらがな", out: false},
		{in: "カタカナ", out: false},
		{in: "漢字", out: true},
		{in: "かな漢字交じり", out: false},
		{in: "123", out: false},
		{in: "#$%", out: false},
		{in: "", out: false},
	}
	for _, cr := range callAndResponse {
		if rsp := kanjiOnly(cr.in); rsp != cr.out {
			t.Errorf("in: %v, got %v, expected: %v", cr.in, rsp, cr.out)
		}
	}
}

func TestLatticeString(t *testing.T) {
	la := New(dic.SysDic(), nil)
	if la == nil {
		t.Fatal("cannot new a lattice")
	}
	defer la.Free()

	expected := ""
	str := la.String()
	if str != expected {
		t.Errorf("got %v, expected: %v", str, expected)
	}

	la.Build("わたしまけましたわ")
	str = la.String()
	if str == "" {
		t.Errorf("got empty string")
	}
}

func TestLatticeDot(t *testing.T) {
	la := New(dic.SysDic(), nil)
	if la == nil {
		t.Fatal("cannot new a lattice")
	}
	defer la.Free()

	expected := "graph lattice {\n\tdpi=48;\n\tgraph [style=filled, rankdir=LR]\n}\n"
	var b bytes.Buffer
	la.Dot(&b)
	if b.String() != expected {
		t.Errorf("got %v, expected: %v", b.String(), expected)
	}
	b.Reset()
	la.Build("わたしまけましたわ")
	la.Dot(&b)
	if b.String() == "" {
		t.Errorf("got empty string")
	}
}

func TestLatticeNewAndFree(t *testing.T) {
	la := New(dic.SysDic(), nil)
	if la == nil {
		t.Fatal("unexpected error: cannot new a lattice")
	}
	if la.Input != "" {
		t.Fatalf("unexpected error: lattice input initialize error, %+v", la.Input)
	}
	if len(la.Output) != 0 {
		t.Fatal("unexpected error: lattice output initialize error, %+v", la.Output)
	}
	if len(la.list) != 0 {
		t.Fatal("unexpected error: lattice list initialize error, %+v", la.list)
	}
	la.Build("すべては科学する心に宿るのだ")
	la.Free()

	// renew
	la = New(dic.SysDic(), nil)
	if la == nil {
		t.Fatal("unexpected error: cannot new a lattice")
	}
	if la.Input != "" {
		t.Fatalf("unexpected error: lattice input initialize error, %+v", la.Input)
	}
	if len(la.Output) != 0 {
		t.Fatal("unexpected error: lattice output initialize error, %+v", la.Output)
	}
	if len(la.list) != 0 {
		t.Fatal("unexpected error: lattice list initialize error, %+v", la.list)
	}
}

func TestForward(t *testing.T) {
	la := New(dic.SysDic(), nil)
	if la == nil {
		t.Fatal("unexpected error: cannot new a lattice")
	}

	la.Forward(Normal)
	la.Forward(Search)
	la.Forward(Extended)

	for _, m := range []TokenizeMode{Normal, Search, Extended} {
		la.Build("わたしまけましたわ")
		la.Forward(m)
	}
}

func TestBackward(t *testing.T) {
	la := New(dic.SysDic(), nil)
	if la == nil {
		t.Fatal("unexpected error: cannot new a lattice")
	}

	// only run
	la.Backward(Normal)
	la.Backward(Search)
	la.Backward(Extended)

	for _, m := range []TokenizeMode{Normal, Search, Extended} {
		la.Build("わたしまけましたわ")
		la.Forward(m)
		la.Backward(m)
	}
}
<commit_msg>Fix tests<commit_after>//  Copyright (c) 2015 ikawaha.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package lattice

import (
	"bytes"
	"testing"
	"unicode/utf8"

	"github.com/ikawaha/kagome/internal/dic"
)

func TestLatticeBuild01(t *testing.T) {
	la := New(dic.SysDic(), nil)
	if la == nil {
		t.Error("cannot new a lattice")
	}
	defer la.Free()

	inp := ""
	la.Build(inp)
	if la.Input != inp {
		t.Errorf("got %v, expected %v", la.Input, inp)
	}
	boseos := node{ID: -1}
	if len(la.list) != 2 {
		t.Errorf("lattice initialize error: got %v, expected has 2 eos/bos nodes", la.list)
	} else if len(la.list[0]) != 1 || *la.list[0][0] != boseos {
		t.Errorf("lattice initialize error: got %v, expected %v", *la.list[0][0], boseos)
	} else if len(la.list[1]) != 1 || *la.list[1][0] != boseos {
		t.Errorf("lattice initialize error: got %v, expected %v", *la.list[1][0], boseos)
	}
	if len(la.Output) != 0 {
		t.Errorf("lattice initialize error: got %v, expected empty", la.Output)
	}

	if la.dic == nil {
		t.Errorf("lattice initialize error: dic is nil")
	}
	if la.udic != nil {
		t.Errorf("lattice initialize error: got %v, expected empty", la.udic)
	}
}

func TestLatticeBuild02(t *testing.T) {
	la := New(dic.SysDic(), nil)
	if la == nil {
		t.Fatal("cannot new a lattice")
	}
	defer la.Free()

	inp := "あ"
	la.Build(inp)
	if la.Input != inp {
		t.Errorf("got %v, expected %v", la.Input, inp)
	}
	bos := node{ID: -1}
	eos := node{ID: -1, Start: 1}
	if len(la.list) != 3 {
		t.Errorf("lattice initialize error: got %v, expected has 2 eos/bos nodes", la.list)
	} else if len(la.list[0]) != 1 || *la.list[0][0] != bos {
		t.Errorf("lattice initialize error: got %v, expected %v", *la.list[0][0], bos)
	} else if len(la.list[2]) != 1 || *la.list[2][0] != eos {
		t.Errorf("lattice initialize error: got %v, expected %v", *la.list[2][0], eos)
	}

	expected := 4
	if len(la.list[1]) != expected {
		t.Errorf("lattice initialize error: got %v, expected %v", len(la.list[1]), expected)
	} else {
		l := la.list[1]
		callAndResponse := []struct {
			in  int
			out node
		}{
			{in: 0, out: node{122, 0, KNOWN, 0, 3, 3, 5549, "あ", nil}},
			{in: 1, out: node{123, 0, KNOWN, 0, 776, 776, 6690, "あ", nil}},
			{in: 2, out: node{124, 0, KNOWN, 0, 2, 2, 4262, "あ", nil}},
			{in: 3, out: node{125, 0, KNOWN, 0, 1118, 1118, 9035, "あ", nil}},
		}
		for _, cr := range callAndResponse {
			if *l[cr.in] != cr.out {
				t.Errorf("lattice initialize error: got %v, expected %v", l[cr.in], cr.out)
			}
		}
	}
	if len(la.Output) != 0 {
		t.Errorf("lattice initialize error: got %v, expected empty", la.Output)
	}
	if la.dic == nil {
		t.Errorf("lattice initialize error: dic is nil")
	}
	if la.udic != nil {
		t.Errorf("lattice initialize error: got %v, expected empty", la.udic)
	}
}

func TestLatticeBuild03(t *testing.T) {

	const udicPath = "../../_sample/userdic.txt"

	udic, e := dic.NewUserDic(udicPath)
	if e != nil {
		t.Fatalf("unexpected error: cannot load user dic, %v", e)
	}
	la := New(dic.SysDic(), udic)
	if la == nil {
		t.Fatal("cannot new a lattice")
	}
	defer la.Free()

	inp := "朝青龍"
	la.Build(inp)
	if la.Input != inp {
		t.Errorf("got %v, expected %v", la.Input, inp)
	}

	if la.list[3][0].Class != USER {
		t.Errorf("%+v", la)
	}

	if len(la.Output) != 0 {
		t.Errorf("lattice initialize error: got %v, expected empty", la.Output)
	}
	if la.dic == nil {
		t.Errorf("lattice initialize error: dic is nil")
	}
	if la.udic == nil {
		t.Errorf("lattice initialize error: got %v, expected not empty", la.udic)
	}
}

func TestLatticeBuild04(t *testing.T) {
	la := New(dic.SysDic(), nil)
	if la == nil {
		t.Fatal("cannot new a lattice")
	}
	defer la.Free()

	inp := "ポポピ"
	la.Build(inp)
	if la.Input != inp {
		t.Errorf("got %v, expected %v", la.Input, inp)
	}
	bos := node{ID: -1}
	eos := node{ID: -1, Start: 3}
	if len(la.list) != 5 {
		t.Errorf("lattice initialize error: got %v, expected has 2 eos/bos nodes", la.list)
	} else if len(la.list[0]) != 1 || *la.list[0][0] != bos {
		t.Errorf("lattice initialize error: got %v, expected %v", *la.list[0][0], bos)
	} else if len(la.list[len(la.list)-1]) != 1 || *la.list[len(la.list)-1][0] != eos {
		t.Errorf("lattice initialize error: got %v, expected %v", *la.list[len(la.list)-1][0], eos)
	}

	expected := 7
	if len(la.list[1]) != expected {
		t.Errorf("lattice initialize error: got %v, expected %v", len(la.list[1]), expected)
	} else {
		l := la.list[1]
		callAndResponse := []struct {
			in  int
			out node
		}{
			{in: 0, out: node{98477, 0, KNOWN, 0, 1285, 1285, 4279, "ポ", nil}},
			{in: 1, out: node{31, 0, UNKNOWN, 0, 1289, 1289, 13581, "ポ", nil}},
			{in: 2, out: node{32, 0, UNKNOWN, 0, 1285, 1285, 9461, "ポ", nil}},
			{in: 3, out: node{33, 0, UNKNOWN, 0, 1293, 1293, 13661, "ポ", nil}},
			{in: 4, out: node{34, 0, UNKNOWN, 0, 1292, 1292, 10922, "ポ", nil}},
			{in: 5, out: node{35, 0, UNKNOWN, 0, 1288, 1288, 10521, "ポ", nil}},
			{in: 6, out: node{36, 0, UNKNOWN, 0, 3, 3, 14138, "ポ", nil}},
		}
		for _, cr := range callAndResponse {
			if *l[cr.in] != cr.out {
				t.Errorf("lattice initialize error: got %v, expected %v", l[cr.in], cr.out)
			}
		}
	}
	if len(la.Output) != 0 {
		t.Errorf("lattice initialize error: got %v, expected empty", la.Output)
	}
	if la.dic == nil {
		t.Errorf("lattice initialize error: dic is nil")
	}
	if la.udic != nil {
		t.Errorf("lattice initialize error: got %v, expected empty", la.udic)
	}
}

func TestLatticeBuild05(t *testing.T) {

	la := New(dic.SysDic(), nil)
	if la == nil {
		t.Fatal("cannot new a lattice")
	}
	defer la.Free()

	inp := "ポポピポンポコナーノ"
	var b bytes.Buffer
	for i, step := 0, utf8.RuneCountInString(inp); i < maximumUnknownWordLength; i = i + step {
		b.WriteString(inp)
	}
	la.Build(b.String())
	for i := range la.list {
		for j := range la.list[i] {
			l := utf8.RuneCountInString(la.list[i][j].Surface)
			if l > maximumUnknownWordLength {
				t.Errorf("too long unknown word, %v", l)
			}
		}
	}
}

func TestKanjiOnly01(t *testing.T) {
	callAndResponse := []struct {
		in  string
		out bool
	}{
		{in: "ひらがな", out: false},
		{in: "カタカナ", out: false},
		{in: "漢字", out: true},
		{in: "かな漢字交じり", out: false},
		{in: "123", out: false},
		{in: "#$%", out: false},
		{in: "", out: false},
	}
	for _, cr := range callAndResponse {
		if rsp := kanjiOnly(cr.in); rsp != cr.out {
			t.Errorf("in: %v, got %v, expected: %v", cr.in, rsp, cr.out)
		}
	}
}

func TestLatticeString(t *testing.T) {
	la := New(dic.SysDic(), nil)
	if la == nil {
		t.Fatal("cannot new a lattice")
	}
	defer la.Free()

	expected := ""
	str := la.String()
	if str != expected {
		t.Errorf("got %v, expected: %v", str, expected)
	}

	la.Build("わたしまけましたわ")
	str = la.String()
	if str == "" {
		t.Errorf("got empty string")
	}
}

func TestLatticeDot(t *testing.T) {
	la := New(dic.SysDic(), nil)
	if la == nil {
		t.Fatal("cannot new a lattice")
	}
	defer la.Free()

	expected := "graph lattice {\n\tdpi=48;\n\tgraph [style=filled, rankdir=LR]\n}\n"
	var b bytes.Buffer
	la.Dot(&b)
	if b.String() != expected {
		t.Errorf("got %v, expected: %v", b.String(), expected)
	}
	b.Reset()
	la.Build("わたしまけましたわ")
	la.Dot(&b)
	if b.String() == "" {
		t.Errorf("got empty string")
	}
}

func TestLatticeNewAndFree(t *testing.T) {
	for i := 0; i < 100; i++ {
		la := New(dic.SysDic(), nil)
		if la == nil {
			t.Fatal("unexpected error: cannot new a lattice")
		}
		if la.Input != "" {
			t.Fatalf("unexpected error: lattice input initialize error, %+v", la.Input)
		}
		if len(la.Output) != 0 {
			t.Fatal("unexpected error: lattice output initialize error, %+v", la.Output)
		}
		if len(la.list) != 0 {
			t.Fatal("unexpected error: lattice list initialize error, %+v", la.list)
		}
		la.Build("すべては科学する心に宿るのだ")
		la.Free()

		// renew
		la = New(dic.SysDic(), nil)
		if la == nil {
			t.Fatal("unexpected error: cannot new a lattice")
		}
		if la.Input != "" {
			t.Fatalf("unexpected error: lattice input initialize error, %+v", la.Input)
		}
		if len(la.Output) != 0 {
			t.Fatal("unexpected error: lattice output initialize error, %+v", la.Output)
		}
		if len(la.list) != 0 {
			t.Fatal("unexpected error: lattice list initialize error, %+v", la.list)
		}
		la.Free()
	}
}

func TestForward(t *testing.T) {
	la := New(dic.SysDic(), nil)
	if la == nil {
		t.Fatal("unexpected error: cannot new a lattice")
	}

	la.Forward(Normal)
	la.Forward(Search)
	la.Forward(Extended)

	for _, m := range []TokenizeMode{Normal, Search, Extended} {
		la.Build("わたしまけましたわ")
		la.Forward(m)
	}
}

func TestBackward(t *testing.T) {
	la := New(dic.SysDic(), nil)
	if la == nil {
		t.Fatal("unexpected error: cannot new a lattice")
	}

	// only run
	la.Backward(Normal)
	la.Backward(Search)
	la.Backward(Extended)

	for _, m := range []TokenizeMode{Normal, Search, Extended} {
		la.Build("わたしまけましたわ")
		la.Forward(m)
		la.Backward(m)
	}
}
<|endoftext|>


<commit_before>package oauth

import (
    "fmt"
    "os"
)

// Stores access token information to the given filename.
//
// Format is the same as the return from the server.
func (o *OAuth) Save(fileName string) (err os.Error) {
    file, err := os.Open(fileName, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0600)
    defer file.Close()
    if err != nil {
        return
    }

    fmt.Fprintf(file, "oauth_token=%s&oauth_token_secret=%s", o.accessToken, o.accessSecret)
    if o.userId != 0 {
        fmt.Fprintf(file, "&user_id=%d", o.userId)
    }
    if o.userName != "" {
        fmt.Fprintf(file, "&screen_name=%s", o.userName)
    }

    return nil
}

// Loads access token information from a file.
func (o *OAuth) Load(fileName string) (err os.Error) {
    file, err := os.Open(fileName, os.O_RDONLY, 0600)
    defer file.Close()
    if err != nil {
        return
    }

    return o.parseResponse(200, file, TokenReq)
}

<commit_msg>Update file handling code.<commit_after>package oauth

import (
    "fmt"
    "os"
)

// Stores access token information to the given filename.
//
// Format is the same as the return from the server.
func (o *OAuth) Save(fileName string) (err os.Error) {
    file, err := os.OpenFile(fileName, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0600)
    defer file.Close()
    if err != nil {
        return
    }

    fmt.Fprintf(file, "oauth_token=%s&oauth_token_secret=%s", o.accessToken, o.accessSecret)
    if o.userId != 0 {
        fmt.Fprintf(file, "&user_id=%d", o.userId)
    }
    if o.userName != "" {
        fmt.Fprintf(file, "&screen_name=%s", o.userName)
    }

    return nil
}

// Loads access token information from a file.
func (o *OAuth) Load(fileName string) (err os.Error) {
    file, err := os.Open(fileName)
    defer file.Close()
    if err != nil {
        return
    }

    return o.parseResponse(200, file, TokenReq)
}

<|endoftext|>


<commit_before>/**
 * A thread-safe tree which caches inverted matrices.
 *
 * Copyright 2016, Peter Collins
 */

package reedsolomon

import (
	"errors"
	"sync"
)

// The tree uses a Reader-Writer mutex to make it thread-safe
// when accessing cached matrices and inserting new ones.
type inversionTree struct {
	mutex sync.RWMutex
	root  inversionNode
}

type inversionNode struct {
	matrix   matrix
	children []*inversionNode
}

// newInversionTree initializes a tree for storing inverted matrices.
// Note that the root node is the identity matrix as it implies
// there were no errors with the original data.
func newInversionTree(dataShards, parityShards int) inversionTree {
	identity, _ := identityMatrix(dataShards)
	root := inversionNode{
		matrix:   identity,
		children: make([]*inversionNode, dataShards+parityShards),
	}
	return inversionTree{
		mutex: sync.RWMutex{},
		root:  root,
	}
}

// GetInvertedMatrix returns the cached inverted matrix or nil if it
// is not found in the tree keyed on the indices of invalid rows.
func (t inversionTree) GetInvertedMatrix(invalidIndices []int) matrix {
	// Lock the tree for reading before accessing the tree.
	t.mutex.RLock()
	defer t.mutex.RUnlock()

	// If no invalid indices were give we should return the root
	// identity matrix.
	if len(invalidIndices) == 0 {
		return t.root.matrix
	}

	// Recursively search for the inverted matrix in the tree, passing in
	// 0 as the parent index as we start at the root of the tree.
	return t.root.getInvertedMatrix(invalidIndices, 0)
}

// errAlreadySet is returned if the root node matrix is overwritten
var errAlreadySet = errors.New("the root node identity matrix is already set")

// InsertInvertedMatrix inserts a new inverted matrix into the tree
// keyed by the indices of invalid rows.  The total number of shards
// is required for creating the proper length lists of child nodes for
// each node.
func (t inversionTree) InsertInvertedMatrix(invalidIndices []int, matrix matrix, shards int) error {
	// If no invalid indices were given then we are done because the
	// root node is already set with the identity matrix.
	if len(invalidIndices) == 0 {
		return errAlreadySet
	}

	if !matrix.IsSquare() {
		return errNotSquare
	}

	// Lock the tree for writing and reading before accessing the tree.
	t.mutex.Lock()
	defer t.mutex.Unlock()

	// Recursively create nodes for the inverted matrix in the tree until
	// we reach the node to insert the matrix to.  We start by passing in
	// 0 as the parent index as we start at the root of the tree.
	t.root.insertInvertedMatrix(invalidIndices, matrix, shards, 0)

	return nil
}

func (n inversionNode) getInvertedMatrix(invalidIndices []int, parent int) matrix {
	// Get the child node to search next from the list of children.  The
	// list of children starts relative to the parent index passed in
	// because the indices of invalid rows is sorted (by default).  As we
	// search recursively, the first invalid index gets popped off the list,
	// so when searching through the list of children, use that first invalid
	// index to find the child node.
	firstIndex := invalidIndices[0]
	node := n.children[firstIndex-parent]

	// If the child node doesn't exist in the list yet, fail fast by
	// returning, so we can construct and insert the proper inverted matrix.
	if node == nil {
		return nil
	}

	// If there's more than one invalid index left in the list we should
	// keep searching recursively.
	if len(invalidIndices) > 1 {
		// Search recursively on the child node by passing in the invalid indices
		// with the first index popped off the front.  Also the parent index to
		// pass down is the first index plus one.
		return node.getInvertedMatrix(invalidIndices[1:], firstIndex+1)
	}
	// If there aren't any more invalid indices to search, we've found our
	// node.  Return it, however keep in mind that the matrix could still be
	// nil because intermediary nodes in the tree are created sometimes with
	// their inversion matrices uninitialized.
	return n.matrix
}

func (n inversionNode) insertInvertedMatrix(invalidIndices []int, matrix matrix, shards, parent int) {
	// As above, get the child node to search next from the list of children.
	// The list of children starts relative to the parent index passed in
	// because the indices of invalid rows is sorted (by default).  As we
	// search recursively, the first invalid index gets popped off the list,
	// so when searching through the list of children, use that first invalid
	// index to find the child node.
	firstIndex := invalidIndices[0]
	node := n.children[firstIndex-parent]

	// If the child node doesn't exist in the list yet, create a new
	// node because we have the writer lock and add it to the list
	// of children.
	if node == nil {
		// Make the length of the list of children equal to the number
		// of shards minus the first invalid index because the list of
		// invalid indices is sorted, so only this length of errors
		// are possible in the tree.
		node = &inversionNode{
			children: make([]*inversionNode, shards-firstIndex),
		}
		// Insert the new node into the tree at the first index relative
		// to the parent index that was given in this recursive call.
		n.children[firstIndex-parent] = node
	}

	// If there's more than one invalid index left in the list we should
	// keep searching recursively in order to find the node to add our
	// matrix.
	if len(invalidIndices) > 1 {
		// As above, search recursively on the child node by passing in
		// the invalid indices with the first index popped off the front.
		// Also the total number of shards and parent index are passed down
		// which is equal to the first index plus one.
		node.insertInvertedMatrix(invalidIndices[1:], matrix, shards, firstIndex+1)
	} else {
		// If there aren't any more invalid indices to search, we've found our
		// node.  Cache the inverted matrix in this node.
		node.matrix = matrix
	}
}
<commit_msg>Don't pass RWMutex by value<commit_after>/**
 * A thread-safe tree which caches inverted matrices.
 *
 * Copyright 2016, Peter Collins
 */

package reedsolomon

import (
	"errors"
	"sync"
)

// The tree uses a Reader-Writer mutex to make it thread-safe
// when accessing cached matrices and inserting new ones.
type inversionTree struct {
	mutex *sync.RWMutex
	root  inversionNode
}

type inversionNode struct {
	matrix   matrix
	children []*inversionNode
}

// newInversionTree initializes a tree for storing inverted matrices.
// Note that the root node is the identity matrix as it implies
// there were no errors with the original data.
func newInversionTree(dataShards, parityShards int) inversionTree {
	identity, _ := identityMatrix(dataShards)
	root := inversionNode{
		matrix:   identity,
		children: make([]*inversionNode, dataShards+parityShards),
	}
	return inversionTree{
		mutex: &sync.RWMutex{},
		root:  root,
	}
}

// GetInvertedMatrix returns the cached inverted matrix or nil if it
// is not found in the tree keyed on the indices of invalid rows.
func (t inversionTree) GetInvertedMatrix(invalidIndices []int) matrix {
	// Lock the tree for reading before accessing the tree.
	t.mutex.RLock()
	defer t.mutex.RUnlock()

	// If no invalid indices were give we should return the root
	// identity matrix.
	if len(invalidIndices) == 0 {
		return t.root.matrix
	}

	// Recursively search for the inverted matrix in the tree, passing in
	// 0 as the parent index as we start at the root of the tree.
	return t.root.getInvertedMatrix(invalidIndices, 0)
}

// errAlreadySet is returned if the root node matrix is overwritten
var errAlreadySet = errors.New("the root node identity matrix is already set")

// InsertInvertedMatrix inserts a new inverted matrix into the tree
// keyed by the indices of invalid rows.  The total number of shards
// is required for creating the proper length lists of child nodes for
// each node.
func (t inversionTree) InsertInvertedMatrix(invalidIndices []int, matrix matrix, shards int) error {
	// If no invalid indices were given then we are done because the
	// root node is already set with the identity matrix.
	if len(invalidIndices) == 0 {
		return errAlreadySet
	}

	if !matrix.IsSquare() {
		return errNotSquare
	}

	// Lock the tree for writing and reading before accessing the tree.
	t.mutex.Lock()
	defer t.mutex.Unlock()

	// Recursively create nodes for the inverted matrix in the tree until
	// we reach the node to insert the matrix to.  We start by passing in
	// 0 as the parent index as we start at the root of the tree.
	t.root.insertInvertedMatrix(invalidIndices, matrix, shards, 0)

	return nil
}

func (n inversionNode) getInvertedMatrix(invalidIndices []int, parent int) matrix {
	// Get the child node to search next from the list of children.  The
	// list of children starts relative to the parent index passed in
	// because the indices of invalid rows is sorted (by default).  As we
	// search recursively, the first invalid index gets popped off the list,
	// so when searching through the list of children, use that first invalid
	// index to find the child node.
	firstIndex := invalidIndices[0]
	node := n.children[firstIndex-parent]

	// If the child node doesn't exist in the list yet, fail fast by
	// returning, so we can construct and insert the proper inverted matrix.
	if node == nil {
		return nil
	}

	// If there's more than one invalid index left in the list we should
	// keep searching recursively.
	if len(invalidIndices) > 1 {
		// Search recursively on the child node by passing in the invalid indices
		// with the first index popped off the front.  Also the parent index to
		// pass down is the first index plus one.
		return node.getInvertedMatrix(invalidIndices[1:], firstIndex+1)
	}
	// If there aren't any more invalid indices to search, we've found our
	// node.  Return it, however keep in mind that the matrix could still be
	// nil because intermediary nodes in the tree are created sometimes with
	// their inversion matrices uninitialized.
	return n.matrix
}

func (n inversionNode) insertInvertedMatrix(invalidIndices []int, matrix matrix, shards, parent int) {
	// As above, get the child node to search next from the list of children.
	// The list of children starts relative to the parent index passed in
	// because the indices of invalid rows is sorted (by default).  As we
	// search recursively, the first invalid index gets popped off the list,
	// so when searching through the list of children, use that first invalid
	// index to find the child node.
	firstIndex := invalidIndices[0]
	node := n.children[firstIndex-parent]

	// If the child node doesn't exist in the list yet, create a new
	// node because we have the writer lock and add it to the list
	// of children.
	if node == nil {
		// Make the length of the list of children equal to the number
		// of shards minus the first invalid index because the list of
		// invalid indices is sorted, so only this length of errors
		// are possible in the tree.
		node = &inversionNode{
			children: make([]*inversionNode, shards-firstIndex),
		}
		// Insert the new node into the tree at the first index relative
		// to the parent index that was given in this recursive call.
		n.children[firstIndex-parent] = node
	}

	// If there's more than one invalid index left in the list we should
	// keep searching recursively in order to find the node to add our
	// matrix.
	if len(invalidIndices) > 1 {
		// As above, search recursively on the child node by passing in
		// the invalid indices with the first index popped off the front.
		// Also the total number of shards and parent index are passed down
		// which is equal to the first index plus one.
		node.insertInvertedMatrix(invalidIndices[1:], matrix, shards, firstIndex+1)
	} else {
		// If there aren't any more invalid indices to search, we've found our
		// node.  Cache the inverted matrix in this node.
		node.matrix = matrix
	}
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/organizations"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/helper/validation"
	"log"
)

func resourceAwsOrganization() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsOrganizationCreate,
		Read:   resourceAwsOrganizationRead,
		Delete: resourceAwsOrganizationDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"arn": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"master_account_arn": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"master_account_email": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"master_account_id": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"feature_set": {
				Type:         schema.TypeString,
				Optional:     true,
				ForceNew:     true,
				Default:      organizations.OrganizationFeatureSetAll,
				ValidateFunc: validation.StringInSlice([]string{organizations.OrganizationFeatureSetAll, organizations.OrganizationFeatureSetConsolidatedBilling}, true),
			},
		},
	}
}

func resourceAwsOrganizationCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).organizationsconn

	createOpts := &organizations.CreateOrganizationInput{
		FeatureSet: aws.String(d.Get("feature_set").(string)),
	}
	log.Printf("[DEBUG] Creating Organization: %#v", createOpts)

	resp, err := conn.CreateOrganization(createOpts)
	if err != nil {
		return fmt.Errorf("Error creating organization: %s", err)
	}

	org := resp.Organization
	d.SetId(*org.Id)
	log.Printf("[INFO] Organization ID: %s", d.Id())

	return resourceAwsOrganizationRead(d, meta)
}

func resourceAwsOrganizationRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).organizationsconn

	log.Printf("[INFO] Reading Organization: %s", d.Id())
	org, err := conn.DescribeOrganization(&organizations.DescribeOrganizationInput{})
	if err != nil {
		if orgerr, ok := err.(awserr.Error); ok && orgerr.Code() == "AWSOrganizationsNotInUseException" {
			log.Printf("[WARN] Organization does not exist, removing from state: %s", d.Id())
			d.SetId("")
			return nil
		}
		return err
	}

	d.Set("arn", org.Organization.Arn)
	d.Set("feature_set", org.Organization.FeatureSet)
	d.Set("master_account_arn", org.Organization.MasterAccountArn)
	d.Set("master_account_email", org.Organization.MasterAccountEmail)
	d.Set("master_account_id", org.Organization.MasterAccountId)
	return nil
}

func resourceAwsOrganizationDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).organizationsconn

	log.Printf("[INFO] Deleting Organization: %s", d.Id())

	_, err := conn.DeleteOrganization(&organizations.DeleteOrganizationInput{})
	if err != nil {
		return fmt.Errorf("Error deleting Organization: %s", err)
	}

	d.SetId("")

	return nil
}
<commit_msg>Remove INFO log statement.<commit_after>package aws

import (
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/organizations"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/helper/validation"
	"log"
)

func resourceAwsOrganization() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsOrganizationCreate,
		Read:   resourceAwsOrganizationRead,
		Delete: resourceAwsOrganizationDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"arn": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"master_account_arn": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"master_account_email": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"master_account_id": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"feature_set": {
				Type:         schema.TypeString,
				Optional:     true,
				ForceNew:     true,
				Default:      organizations.OrganizationFeatureSetAll,
				ValidateFunc: validation.StringInSlice([]string{organizations.OrganizationFeatureSetAll, organizations.OrganizationFeatureSetConsolidatedBilling}, true),
			},
		},
	}
}

func resourceAwsOrganizationCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).organizationsconn

	createOpts := &organizations.CreateOrganizationInput{
		FeatureSet: aws.String(d.Get("feature_set").(string)),
	}
	log.Printf("[DEBUG] Creating Organization: %#v", createOpts)

	resp, err := conn.CreateOrganization(createOpts)
	if err != nil {
		return fmt.Errorf("Error creating organization: %s", err)
	}

	org := resp.Organization
	d.SetId(*org.Id)

	return resourceAwsOrganizationRead(d, meta)
}

func resourceAwsOrganizationRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).organizationsconn

	log.Printf("[INFO] Reading Organization: %s", d.Id())
	org, err := conn.DescribeOrganization(&organizations.DescribeOrganizationInput{})
	if err != nil {
		if orgerr, ok := err.(awserr.Error); ok && orgerr.Code() == "AWSOrganizationsNotInUseException" {
			log.Printf("[WARN] Organization does not exist, removing from state: %s", d.Id())
			d.SetId("")
			return nil
		}
		return err
	}

	d.Set("arn", org.Organization.Arn)
	d.Set("feature_set", org.Organization.FeatureSet)
	d.Set("master_account_arn", org.Organization.MasterAccountArn)
	d.Set("master_account_email", org.Organization.MasterAccountEmail)
	d.Set("master_account_id", org.Organization.MasterAccountId)
	return nil
}

func resourceAwsOrganizationDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).organizationsconn

	log.Printf("[INFO] Deleting Organization: %s", d.Id())

	_, err := conn.DeleteOrganization(&organizations.DeleteOrganizationInput{})
	if err != nil {
		return fmt.Errorf("Error deleting Organization: %s", err)
	}

	d.SetId("")

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/sendgrid/sendgrid-go"
	"os"
)

///////////////////////////////////////////////////
// Retrieve all categories
// GET /categories

func Retrieveallcategories() {
  apiKey := os.Getenv("SENDGRID_APIKEY")
  host := "https://api.sendgrid.com"
  request := sendgrid.GetRequest(apiKey, "/categories", host, "v3")
  request.Method = "GET"
  queryParams := make(map[string]string)
  queryParams["category"] = "test_string"
queryParams["limit"] = "1"
queryParams["offset"] = "1"
request.QueryParams = queryParams
  response, err := sendgrid.API(request)
  if err != nil {
    fmt.Println(err)
  } else {
    fmt.Println(response.StatusCode)
    fmt.Println(response.ResponseBody)
    fmt.Println(response.ResponseHeaders)
  }
}

///////////////////////////////////////////////////
// Retrieve Email Statistics for Categories
// GET /categories/stats

func RetrieveEmailStatisticsforCategories() {
  apiKey := os.Getenv("SENDGRID_APIKEY")
  host := "https://api.sendgrid.com"
  request := sendgrid.GetRequest(apiKey, "/categories/stats", host, "v3")
  request.Method = "GET"
  queryParams := make(map[string]string)
  queryParams["end_date"] = "2016-04-01"
queryParams["aggregated_by"] = "day"
queryParams["limit"] = "1"
queryParams["offset"] = "1"
queryParams["start_date"] = "2016-01-01"
queryParams["categories"] = "test_string"
request.QueryParams = queryParams
  response, err := sendgrid.API(request)
  if err != nil {
    fmt.Println(err)
  } else {
    fmt.Println(response.StatusCode)
    fmt.Println(response.ResponseBody)
    fmt.Println(response.ResponseHeaders)
  }
}

///////////////////////////////////////////////////
// Retrieve sums of email stats for each category [Needs: Stats object defined, has category ID?]
// GET /categories/stats/sums

func Retrievesumsofemailstatsforeachcategory[Needs:Statsobjectdefined,hascategoryID?]() {
  apiKey := os.Getenv("SENDGRID_APIKEY")
  host := "https://api.sendgrid.com"
  request := sendgrid.GetRequest(apiKey, "/categories/stats/sums", host, "v3")
  request.Method = "GET"
  queryParams := make(map[string]string)
  queryParams["end_date"] = "2016-04-01"
queryParams["aggregated_by"] = "day"
queryParams["limit"] = "1"
queryParams["sort_by_metric"] = "test_string"
queryParams["offset"] = "1"
queryParams["start_date"] = "2016-01-01"
queryParams["sort_by_direction"] = "asc"
request.QueryParams = queryParams
  response, err := sendgrid.API(request)
  if err != nil {
    fmt.Println(err)
  } else {
    fmt.Println(response.StatusCode)
    fmt.Println(response.ResponseBody)
    fmt.Println(response.ResponseHeaders)
  }
}

func main() {
    // add your function calls here
}
<commit_msg>Removed stray comment<commit_after>package main

import (
	"fmt"
	"github.com/sendgrid/sendgrid-go"
	"os"
)

///////////////////////////////////////////////////
// Retrieve all categories
// GET /categories

func Retrieveallcategories() {
  apiKey := os.Getenv("SENDGRID_APIKEY")
  host := "https://api.sendgrid.com"
  request := sendgrid.GetRequest(apiKey, "/categories", host, "v3")
  request.Method = "GET"
  queryParams := make(map[string]string)
  queryParams["category"] = "test_string"
queryParams["limit"] = "1"
queryParams["offset"] = "1"
request.QueryParams = queryParams
  response, err := sendgrid.API(request)
  if err != nil {
    fmt.Println(err)
  } else {
    fmt.Println(response.StatusCode)
    fmt.Println(response.ResponseBody)
    fmt.Println(response.ResponseHeaders)
  }
}

///////////////////////////////////////////////////
// Retrieve Email Statistics for Categories
// GET /categories/stats

func RetrieveEmailStatisticsforCategories() {
  apiKey := os.Getenv("SENDGRID_APIKEY")
  host := "https://api.sendgrid.com"
  request := sendgrid.GetRequest(apiKey, "/categories/stats", host, "v3")
  request.Method = "GET"
  queryParams := make(map[string]string)
  queryParams["end_date"] = "2016-04-01"
queryParams["aggregated_by"] = "day"
queryParams["limit"] = "1"
queryParams["offset"] = "1"
queryParams["start_date"] = "2016-01-01"
queryParams["categories"] = "test_string"
request.QueryParams = queryParams
  response, err := sendgrid.API(request)
  if err != nil {
    fmt.Println(err)
  } else {
    fmt.Println(response.StatusCode)
    fmt.Println(response.ResponseBody)
    fmt.Println(response.ResponseHeaders)
  }
}

///////////////////////////////////////////////////
// Retrieve sums of email stats for each category [Needs: Stats object defined, has category ID?]
// GET /categories/stats/sums

func Retrievesumsofemailstatsforeachcategory() {
  apiKey := os.Getenv("SENDGRID_APIKEY")
  host := "https://api.sendgrid.com"
  request := sendgrid.GetRequest(apiKey, "/categories/stats/sums", host, "v3")
  request.Method = "GET"
  queryParams := make(map[string]string)
  queryParams["end_date"] = "2016-04-01"
queryParams["aggregated_by"] = "day"
queryParams["limit"] = "1"
queryParams["sort_by_metric"] = "test_string"
queryParams["offset"] = "1"
queryParams["start_date"] = "2016-01-01"
queryParams["sort_by_direction"] = "asc"
request.QueryParams = queryParams
  response, err := sendgrid.API(request)
  if err != nil {
    fmt.Println(err)
  } else {
    fmt.Println(response.StatusCode)
    fmt.Println(response.ResponseBody)
    fmt.Println(response.ResponseHeaders)
  }
}

func main() {
    // add your function calls here
}
<|endoftext|>


<commit_before>package impapps

import (
	"encoding/json"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"sync"

	"github.com/spf13/cobra"
	"github.com/srinandan/apigeecli/cmd/shared"
	types "github.com/srinandan/apigeecli/cmd/types"
)

type App struct {
	Name        string            `json:"name"`
	Status      string            `json:"status"`
	Credentials *[]Credential     `json:"credentials,omitempty"`
	DeveloperID *string           `json:"developerId"`
	DisplayName string            `json:"displayName,omitempty"`
	Attributes  []types.Attribute `json:"attributes,omitempty"`
	CallbackURL string            `json:"callbackUrl,omitempty"`
	Scopes      []string          `json:"scopes,omitempty"`
}

type Credential struct {
	APIProducts    []APIProduct `json:"apiProducts,omitempty"`
	ConsumerKey    string       `json:"consumerKey`
	ConsumerSecret string       `json:"consumerSecret`
	ExpiresAt      int          `json:"expiresAt,omitempty"`
	Status         string       `json:"status,omitempty"`
	Scopes         []string     `json:"scopes,omitempty"`
}

type APIProduct struct {
	Name string `json:"apiproduct,omitempty"`
}

type ImportCredential struct {
	APIProducts    []string `json:"apiProducts,omitempty"`
	ConsumerKey    string   `json:"consumerKey,omitempty"`
	ConsumerSecret string   `json:"consumerSecret,omitempty"`
	Scopes         []string `json:"scopes,omitempty"`
}

var Cmd = &cobra.Command{
	Use:   "import",
	Short: "Import a file containing Developer Apps",
	Long:  "Import a file containing Developer Apps",
	RunE: func(cmd *cobra.Command, args []string) error {
		return createApps()
	},
}

var conn int
var file string

func init() {

	Cmd.Flags().StringVarP(&file, "file", "f",
		"", "File containing Developer Apps")
	Cmd.Flags().IntVarP(&conn, "conn", "c",
		4, "Number of connections")

	_ = Cmd.MarkFlagRequired("file")
}

func createAsyncApp(app App, wg *sync.WaitGroup) {
	defer wg.Done()

	//importing an app will be a two step process.
	//1. create the app without the credential
	//2. create/import the credential
	u, _ := url.Parse(shared.BaseURL)
	//store the developer and the credential
	developerID := *app.DeveloperID
	credentials := *app.Credentials

	//remove the developer id and credentials from the payload
	app.DeveloperID = nil
	app.Credentials = nil

	out, err := json.Marshal(app)
	if err != nil {
		shared.Error.Fatalln(err)
		return
	}

	u.Path = path.Join(u.Path, shared.RootArgs.Org, "developers", developerID, "apps")
	_, err = shared.HttpClient(true, u.String(), string(out))
	if err != nil {
		shared.Error.Fatalln(err)
		return
	}
	u, _ = url.Parse(shared.BaseURL)
	u.Path = path.Join(u.Path, shared.RootArgs.Org, "developers", developerID, "apps", app.Name, "keys", "create")
	for _, credential := range credentials {
		//construct a []string for products
		var products []string
		for _, apiProduct := range credential.APIProducts {
			products = append(products, apiProduct.Name)
		}
		//create a new credential
		importCredential := ImportCredential{}
		importCredential.APIProducts = products
		importCredential.ConsumerKey = credential.ConsumerKey
		importCredential.ConsumerSecret = credential.ConsumerSecret
		importCredential.Scopes = credential.Scopes

		impCred, err := json.Marshal(importCredential)
		if err != nil {
			shared.Error.Fatalln(err)
			return
		}
		_, err = shared.HttpClient(true, u.String(), string(impCred))
		if err != nil {
			shared.Error.Fatalln(err)
			return
		}
		shared.Warning.Println("NOTE: apiProducts are not associated with the app")
	}
	shared.Info.Printf("Completed entity: %s", app.Name)
}

//batch created a batch of products to query
func batch(entities []App, pwg *sync.WaitGroup) {

	defer pwg.Done()
	//batch workgroup
	var bwg sync.WaitGroup

	bwg.Add(len(entities))

	for _, entity := range entities {
		go createAsyncApp(entity, &bwg)
	}
	bwg.Wait()
}

func createApps() error {

	var pwg sync.WaitGroup

	entities, err := readAppsFile()
	if err != nil {
		shared.Error.Fatalln("Error reading file: ", err)
		return err
	}

	numEntities := len(entities)
	shared.Info.Printf("Found %d apps in the file\n", numEntities)
	shared.Info.Printf("Create apps with %d connections\n", conn)

	numOfLoops, remaining := numEntities/conn, numEntities%conn

	//ensure connections aren't greater then products
	if conn > numEntities {
		conn = numEntities
	}

	start := 0

	for i, end := 0, 0; i < numOfLoops; i++ {
		pwg.Add(1)
		end = (i * conn) + conn
		shared.Info.Printf("Creating batch %d of apps\n", (i + 1))
		go batch(entities[start:end], &pwg)
		start = end
		pwg.Wait()
	}

	if remaining > 0 {
		pwg.Add(1)
		shared.Info.Printf("Creating remaining %d apps\n", remaining)
		go batch(entities[start:numEntities], &pwg)
		pwg.Wait()
	}

	return nil
}

func readAppsFile() ([]App, error) {

	apps := []App{}

	jsonFile, err := os.Open(file)

	if err != nil {
		return apps, err
	}

	defer jsonFile.Close()

	byteValue, err := ioutil.ReadAll(jsonFile)

	if err != nil {
		return apps, err
	}

	err = json.Unmarshal(byteValue, &apps)

	if err != nil {
		return apps, err
	}

	return apps, nil

}
<commit_msg>fix typo in comment<commit_after>package impapps

import (
	"encoding/json"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"sync"

	"github.com/spf13/cobra"
	"github.com/srinandan/apigeecli/cmd/shared"
	types "github.com/srinandan/apigeecli/cmd/types"
)

type App struct {
	Name        string            `json:"name"`
	Status      string            `json:"status"`
	Credentials *[]Credential     `json:"credentials,omitempty"`
	DeveloperID *string           `json:"developerId"`
	DisplayName string            `json:"displayName,omitempty"`
	Attributes  []types.Attribute `json:"attributes,omitempty"`
	CallbackURL string            `json:"callbackUrl,omitempty"`
	Scopes      []string          `json:"scopes,omitempty"`
}

type Credential struct {
	APIProducts    []APIProduct `json:"apiProducts,omitempty"`
	ConsumerKey    string       `json:"consumerKey`
	ConsumerSecret string       `json:"consumerSecret`
	ExpiresAt      int          `json:"expiresAt,omitempty"`
	Status         string       `json:"status,omitempty"`
	Scopes         []string     `json:"scopes,omitempty"`
}

type APIProduct struct {
	Name string `json:"apiproduct,omitempty"`
}

type ImportCredential struct {
	APIProducts    []string `json:"apiProducts,omitempty"`
	ConsumerKey    string   `json:"consumerKey,omitempty"`
	ConsumerSecret string   `json:"consumerSecret,omitempty"`
	Scopes         []string `json:"scopes,omitempty"`
}

var Cmd = &cobra.Command{
	Use:   "import",
	Short: "Import a file containing Developer Apps",
	Long:  "Import a file containing Developer Apps",
	RunE: func(cmd *cobra.Command, args []string) error {
		return createApps()
	},
}

var conn int
var file string

func init() {

	Cmd.Flags().StringVarP(&file, "file", "f",
		"", "File containing Developer Apps")
	Cmd.Flags().IntVarP(&conn, "conn", "c",
		4, "Number of connections")

	_ = Cmd.MarkFlagRequired("file")
}

func createAsyncApp(app App, wg *sync.WaitGroup) {
	defer wg.Done()

	//importing an app will be a two step process.
	//1. create the app without the credential
	//2. create/import the credential
	u, _ := url.Parse(shared.BaseURL)
	//store the developer and the credential
	developerID := *app.DeveloperID
	credentials := *app.Credentials

	//remove the developer id and credentials from the payload
	app.DeveloperID = nil
	app.Credentials = nil

	out, err := json.Marshal(app)
	if err != nil {
		shared.Error.Fatalln(err)
		return
	}

	u.Path = path.Join(u.Path, shared.RootArgs.Org, "developers", developerID, "apps")
	_, err = shared.HttpClient(true, u.String(), string(out))
	if err != nil {
		shared.Error.Fatalln(err)
		return
	}
	u, _ = url.Parse(shared.BaseURL)
	u.Path = path.Join(u.Path, shared.RootArgs.Org, "developers", developerID, "apps", app.Name, "keys", "create")
	for _, credential := range credentials {
		//construct a []string for products
		var products []string
		for _, apiProduct := range credential.APIProducts {
			products = append(products, apiProduct.Name)
		}
		//create a new credential
		importCredential := ImportCredential{}
		importCredential.APIProducts = products
		importCredential.ConsumerKey = credential.ConsumerKey
		importCredential.ConsumerSecret = credential.ConsumerSecret
		importCredential.Scopes = credential.Scopes

		impCred, err := json.Marshal(importCredential)
		if err != nil {
			shared.Error.Fatalln(err)
			return
		}
		_, err = shared.HttpClient(true, u.String(), string(impCred))
		if err != nil {
			shared.Error.Fatalln(err)
			return
		}
		shared.Warning.Println("NOTE: apiProducts are not associated with the app")
	}
	shared.Info.Printf("Completed entity: %s", app.Name)
}

//batch creates a batch of apps to create
func batch(entities []App, pwg *sync.WaitGroup) {

	defer pwg.Done()
	//batch workgroup
	var bwg sync.WaitGroup

	bwg.Add(len(entities))

	for _, entity := range entities {
		go createAsyncApp(entity, &bwg)
	}
	bwg.Wait()
}

func createApps() error {

	var pwg sync.WaitGroup

	entities, err := readAppsFile()
	if err != nil {
		shared.Error.Fatalln("Error reading file: ", err)
		return err
	}

	numEntities := len(entities)
	shared.Info.Printf("Found %d apps in the file\n", numEntities)
	shared.Info.Printf("Create apps with %d connections\n", conn)

	numOfLoops, remaining := numEntities/conn, numEntities%conn

	//ensure connections aren't greater than products
	if conn > numEntities {
		conn = numEntities
	}

	start := 0

	for i, end := 0, 0; i < numOfLoops; i++ {
		pwg.Add(1)
		end = (i * conn) + conn
		shared.Info.Printf("Creating batch %d of apps\n", (i + 1))
		go batch(entities[start:end], &pwg)
		start = end
		pwg.Wait()
	}

	if remaining > 0 {
		pwg.Add(1)
		shared.Info.Printf("Creating remaining %d apps\n", remaining)
		go batch(entities[start:numEntities], &pwg)
		pwg.Wait()
	}

	return nil
}

func readAppsFile() ([]App, error) {

	apps := []App{}

	jsonFile, err := os.Open(file)

	if err != nil {
		return apps, err
	}

	defer jsonFile.Close()

	byteValue, err := ioutil.ReadAll(jsonFile)

	if err != nil {
		return apps, err
	}

	err = json.Unmarshal(byteValue, &apps)

	if err != nil {
		return apps, err
	}

	return apps, nil

}
<|endoftext|>


<commit_before>package interfaces

import (
	"net/http"

	"github.com/4406arthur/clean-micro-article-service/pkg/entity"
)

type ArticleInteractor interface {
	Get(title string) (*entity.Article, error)
	List() ([]*entity.Article, error)
	Create(article entity.Article) error
}

type WebserviceHandler struct {
	ArticleInteractor ArticleInteractor
}

func (handler WebserviceHandler) ApiList(res http.ResponseWriter, req *http.Request) {
	articles, _ := handler.ArticleInteractor.List()
	for _, article := range articles {

	}
}

func (handler WebserviceHandler) ApiGet(res http.ResponseWriter, req *http.Request) {
	title := req.FormValue("title")
	article, _ := handler.ArticleInteractor.Get(title)
	return article
}

func (handler WebserviceHandler) ApiCreate(res http.ResponseWriter, req *http.Request) {

	handler.ArticleInteractor.Create()
	for _, article := range articles {

	}
}
<commit_msg>delete unuse http client <commit_after><|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"context"
	"fmt"
	"io"
	"os"

	"github.com/blang/semver/v4"
	"github.com/spf13/cobra"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/klog/v2"
	"k8s.io/kops"
	"k8s.io/kops/cmd/kops/util"
	kopsapi "k8s.io/kops/pkg/apis/kops"
	kopsutil "k8s.io/kops/pkg/apis/kops/util"
	"k8s.io/kops/pkg/commands"
	"k8s.io/kops/pkg/commands/commandutils"
	"k8s.io/kops/pkg/pretty"
	"k8s.io/kops/upup/pkg/fi/cloudup"
	"k8s.io/kops/util/pkg/tables"
	"k8s.io/kubectl/pkg/util/i18n"
	"k8s.io/kubectl/pkg/util/templates"
)

var (
	upgradeClusterLong = pretty.LongDesc(i18n.T(`
	Automates checking for and applying Kubernetes updates. This upgrades a cluster to the latest recommended
	production ready Kubernetes version. After this command is run, use ` + pretty.Bash("kops update cluster") + ` and ` + pretty.Bash("kops rolling-update cluster") + `
	to finish a cluster upgrade.
	`))

	upgradeClusterExample = templates.Examples(i18n.T(`
	# Upgrade a cluster's Kubernetes version.
	kops upgrade cluster k8s-cluster.example.com --yes --state=s3://my-state-store
	`))

	upgradeClusterShort = i18n.T("Upgrade a kubernetes cluster.")
)

type UpgradeClusterOptions struct {
	ClusterName string
	Yes         bool
	Channel     string
}

func NewCmdUpgradeCluster(f *util.Factory, out io.Writer) *cobra.Command {
	options := &UpgradeClusterOptions{}

	cmd := &cobra.Command{
		Use:               "cluster [CLUSTER]",
		Short:             upgradeClusterShort,
		Long:              upgradeClusterLong,
		Example:           upgradeClusterExample,
		Args:              rootCommand.clusterNameArgs(&options.ClusterName),
		ValidArgsFunction: commandutils.CompleteClusterName(f, true, false),
		RunE: func(cmd *cobra.Command, args []string) error {
			ctx := context.TODO()

			return RunUpgradeCluster(ctx, f, out, options)
		},
	}

	cmd.Flags().BoolVarP(&options.Yes, "yes", "y", false, "Apply update")
	cmd.Flags().StringVar(&options.Channel, "channel", "", "Channel to use for upgrade")
	cmd.RegisterFlagCompletionFunc("channel", completeChannel)

	return cmd
}

type upgradeAction struct {
	Item     string
	Property string
	Old      string
	New      string

	apply func()
}

func RunUpgradeCluster(ctx context.Context, f *util.Factory, out io.Writer, options *UpgradeClusterOptions) error {
	cluster, err := GetCluster(ctx, f, options.ClusterName)
	if err != nil {
		return err
	}

	clientset, err := f.Clientset()
	if err != nil {
		return err
	}

	instanceGroups, err := commands.ReadAllInstanceGroups(ctx, clientset, cluster)
	if err != nil {
		return err
	}

	if cluster.ObjectMeta.Annotations[kopsapi.AnnotationNameManagement] == kopsapi.AnnotationValueManagementImported {
		return fmt.Errorf("upgrade is not for use with imported clusters.)")
	}

	channelLocation := options.Channel
	if channelLocation == "" {
		channelLocation = cluster.Spec.Channel
	}
	if channelLocation == "" {
		channelLocation = kopsapi.DefaultChannel
	}

	var actions []*upgradeAction
	if channelLocation != cluster.Spec.Channel {
		actions = append(actions, &upgradeAction{
			Item:     "Cluster",
			Property: "Channel",
			Old:      cluster.Spec.Channel,
			New:      channelLocation,
			apply: func() {
				cluster.Spec.Channel = channelLocation
			},
		})
	}

	channel, err := kopsapi.LoadChannel(channelLocation)
	if err != nil {
		return fmt.Errorf("error loading channel %q: %v", channelLocation, err)
	}

	var currentKubernetesVersion *semver.Version
	{
		sv, err := kopsutil.ParseKubernetesVersion(cluster.Spec.KubernetesVersion)
		if err != nil {
			klog.Warningf("error parsing KubernetesVersion %q", cluster.Spec.KubernetesVersion)
		} else {
			currentKubernetesVersion = sv
		}
	}

	proposedKubernetesVersion := kopsapi.RecommendedKubernetesVersion(channel, kops.Version)

	// We won't propose a downgrade
	// TODO: What if a kubernetes version is bad?
	if currentKubernetesVersion != nil && proposedKubernetesVersion != nil && currentKubernetesVersion.GT(*proposedKubernetesVersion) {
		klog.Warningf("cluster version %q is greater than recommended version %q", *currentKubernetesVersion, *proposedKubernetesVersion)
		proposedKubernetesVersion = currentKubernetesVersion
	}

	if proposedKubernetesVersion != nil && currentKubernetesVersion != nil && currentKubernetesVersion.NE(*proposedKubernetesVersion) {
		actions = append(actions, &upgradeAction{
			Item:     "Cluster",
			Property: "KubernetesVersion",
			Old:      cluster.Spec.KubernetesVersion,
			New:      proposedKubernetesVersion.String(),
			apply: func() {
				cluster.Spec.KubernetesVersion = proposedKubernetesVersion.String()
			},
		})
	}

	// For further calculations, default to the current kubernetes version
	if proposedKubernetesVersion == nil {
		proposedKubernetesVersion = currentKubernetesVersion
	}

	cloud, err := cloudup.BuildCloud(cluster)
	if err != nil {
		return err
	}

	// Prompt to upgrade image
	if proposedKubernetesVersion != nil {
		for _, ig := range instanceGroups {
			architecture, err := cloudup.MachineArchitecture(cloud, ig.Spec.MachineType)
			if err != nil {
				klog.Warningf("Error finding architecture for machine type %q: %v", ig.Spec.MachineType, err)
				continue
			}
			image := channel.FindImage(cloud.ProviderID(), *proposedKubernetesVersion, architecture)
			if image == nil {
				klog.Warningf("No matching images specified in channel; cannot prompt for upgrade")
				continue
			}
			if channel.HasUpstreamImagePrefix(ig.Spec.Image) {
				if ig.Spec.Image != image.Name {
					target := ig
					actions = append(actions, &upgradeAction{
						Item:     "InstanceGroup/" + target.ObjectMeta.Name,
						Property: "Image",
						Old:      target.Spec.Image,
						New:      image.Name,
						apply: func() {
							target.Spec.Image = image.Name
						},
					})
				}
			} else {
				klog.Infof("Custom image (%s) has been provided for Instance Group %q; not updating image", ig.Spec.Image, ig.GetName())
			}
		}
	}

	if len(actions) == 0 {
		// TODO: Allow --force option to force even if not needed?
		// Note stderr - we try not to print to stdout if no update is needed
		fmt.Fprintf(os.Stderr, "\nNo upgrade required\n")
		return nil
	}

	{
		t := &tables.Table{}
		t.AddColumn("ITEM", func(a *upgradeAction) string {
			return a.Item
		})
		t.AddColumn("PROPERTY", func(a *upgradeAction) string {
			return a.Property
		})
		t.AddColumn("OLD", func(a *upgradeAction) string {
			return a.Old
		})
		t.AddColumn("NEW", func(a *upgradeAction) string {
			return a.New
		})

		err := t.Render(actions, out, "ITEM", "PROPERTY", "OLD", "NEW")
		if err != nil {
			return err
		}
	}

	if !options.Yes {
		fmt.Printf("\nMust specify --yes to perform upgrade\n")
		return nil
	}
	for _, action := range actions {
		action.apply()
	}

	if err := commands.UpdateCluster(ctx, clientset, cluster, instanceGroups); err != nil {
		return err
	}

	for _, g := range instanceGroups {
		_, err := clientset.InstanceGroupsFor(cluster).Update(ctx, g, metav1.UpdateOptions{})
		if err != nil {
			return fmt.Errorf("error writing InstanceGroup %q: %v", g.ObjectMeta.Name, err)
		}
	}

	fmt.Printf("\nUpdates applied to configuration.\n")

	// TODO: automate this step
	fmt.Printf("You can now apply these changes, using `kops update cluster %s`\n", cluster.ObjectMeta.Name)

	return nil
}

func completeChannel(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
	// TODO implement completion against VFS
	return []string{"alpha", "stable"}, cobra.ShellCompDirectiveNoFileComp
}
<commit_msg>If image is empty, have kops upgrade fill it in<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"context"
	"fmt"
	"io"
	"os"

	"github.com/blang/semver/v4"
	"github.com/spf13/cobra"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/klog/v2"
	"k8s.io/kops"
	"k8s.io/kops/cmd/kops/util"
	kopsapi "k8s.io/kops/pkg/apis/kops"
	kopsutil "k8s.io/kops/pkg/apis/kops/util"
	"k8s.io/kops/pkg/commands"
	"k8s.io/kops/pkg/commands/commandutils"
	"k8s.io/kops/pkg/pretty"
	"k8s.io/kops/upup/pkg/fi/cloudup"
	"k8s.io/kops/util/pkg/tables"
	"k8s.io/kubectl/pkg/util/i18n"
	"k8s.io/kubectl/pkg/util/templates"
)

var (
	upgradeClusterLong = pretty.LongDesc(i18n.T(`
	Automates checking for and applying Kubernetes updates. This upgrades a cluster to the latest recommended
	production ready Kubernetes version. After this command is run, use ` + pretty.Bash("kops update cluster") + ` and ` + pretty.Bash("kops rolling-update cluster") + `
	to finish a cluster upgrade.
	`))

	upgradeClusterExample = templates.Examples(i18n.T(`
	# Upgrade a cluster's Kubernetes version.
	kops upgrade cluster k8s-cluster.example.com --yes --state=s3://my-state-store
	`))

	upgradeClusterShort = i18n.T("Upgrade a kubernetes cluster.")
)

type UpgradeClusterOptions struct {
	ClusterName string
	Yes         bool
	Channel     string
}

func NewCmdUpgradeCluster(f *util.Factory, out io.Writer) *cobra.Command {
	options := &UpgradeClusterOptions{}

	cmd := &cobra.Command{
		Use:               "cluster [CLUSTER]",
		Short:             upgradeClusterShort,
		Long:              upgradeClusterLong,
		Example:           upgradeClusterExample,
		Args:              rootCommand.clusterNameArgs(&options.ClusterName),
		ValidArgsFunction: commandutils.CompleteClusterName(f, true, false),
		RunE: func(cmd *cobra.Command, args []string) error {
			ctx := context.TODO()

			return RunUpgradeCluster(ctx, f, out, options)
		},
	}

	cmd.Flags().BoolVarP(&options.Yes, "yes", "y", false, "Apply update")
	cmd.Flags().StringVar(&options.Channel, "channel", "", "Channel to use for upgrade")
	cmd.RegisterFlagCompletionFunc("channel", completeChannel)

	return cmd
}

type upgradeAction struct {
	Item     string
	Property string
	Old      string
	New      string

	apply func()
}

func RunUpgradeCluster(ctx context.Context, f *util.Factory, out io.Writer, options *UpgradeClusterOptions) error {
	cluster, err := GetCluster(ctx, f, options.ClusterName)
	if err != nil {
		return err
	}

	clientset, err := f.Clientset()
	if err != nil {
		return err
	}

	instanceGroups, err := commands.ReadAllInstanceGroups(ctx, clientset, cluster)
	if err != nil {
		return err
	}

	if cluster.ObjectMeta.Annotations[kopsapi.AnnotationNameManagement] == kopsapi.AnnotationValueManagementImported {
		return fmt.Errorf("upgrade is not for use with imported clusters.)")
	}

	channelLocation := options.Channel
	if channelLocation == "" {
		channelLocation = cluster.Spec.Channel
	}
	if channelLocation == "" {
		channelLocation = kopsapi.DefaultChannel
	}

	var actions []*upgradeAction
	if channelLocation != cluster.Spec.Channel {
		actions = append(actions, &upgradeAction{
			Item:     "Cluster",
			Property: "Channel",
			Old:      cluster.Spec.Channel,
			New:      channelLocation,
			apply: func() {
				cluster.Spec.Channel = channelLocation
			},
		})
	}

	channel, err := kopsapi.LoadChannel(channelLocation)
	if err != nil {
		return fmt.Errorf("error loading channel %q: %v", channelLocation, err)
	}

	var currentKubernetesVersion *semver.Version
	{
		sv, err := kopsutil.ParseKubernetesVersion(cluster.Spec.KubernetesVersion)
		if err != nil {
			klog.Warningf("error parsing KubernetesVersion %q", cluster.Spec.KubernetesVersion)
		} else {
			currentKubernetesVersion = sv
		}
	}

	proposedKubernetesVersion := kopsapi.RecommendedKubernetesVersion(channel, kops.Version)

	// We won't propose a downgrade
	// TODO: What if a kubernetes version is bad?
	if currentKubernetesVersion != nil && proposedKubernetesVersion != nil && currentKubernetesVersion.GT(*proposedKubernetesVersion) {
		klog.Warningf("cluster version %q is greater than recommended version %q", *currentKubernetesVersion, *proposedKubernetesVersion)
		proposedKubernetesVersion = currentKubernetesVersion
	}

	if proposedKubernetesVersion != nil && currentKubernetesVersion != nil && currentKubernetesVersion.NE(*proposedKubernetesVersion) {
		actions = append(actions, &upgradeAction{
			Item:     "Cluster",
			Property: "KubernetesVersion",
			Old:      cluster.Spec.KubernetesVersion,
			New:      proposedKubernetesVersion.String(),
			apply: func() {
				cluster.Spec.KubernetesVersion = proposedKubernetesVersion.String()
			},
		})
	}

	// For further calculations, default to the current kubernetes version
	if proposedKubernetesVersion == nil {
		proposedKubernetesVersion = currentKubernetesVersion
	}

	cloud, err := cloudup.BuildCloud(cluster)
	if err != nil {
		return err
	}

	// Prompt to upgrade image
	if proposedKubernetesVersion != nil {
		for _, ig := range instanceGroups {
			architecture, err := cloudup.MachineArchitecture(cloud, ig.Spec.MachineType)
			if err != nil {
				klog.Warningf("Error finding architecture for machine type %q: %v", ig.Spec.MachineType, err)
				continue
			}
			image := channel.FindImage(cloud.ProviderID(), *proposedKubernetesVersion, architecture)
			if image == nil {
				klog.Warningf("No matching images specified in channel; cannot prompt for upgrade")
				continue
			}
			if ig.Spec.Image == "" || channel.HasUpstreamImagePrefix(ig.Spec.Image) {
				if ig.Spec.Image != image.Name {
					target := ig
					actions = append(actions, &upgradeAction{
						Item:     "InstanceGroup/" + target.ObjectMeta.Name,
						Property: "Image",
						Old:      target.Spec.Image,
						New:      image.Name,
						apply: func() {
							target.Spec.Image = image.Name
						},
					})
				}
			} else {
				klog.Infof("Custom image (%s) has been provided for Instance Group %q; not updating image", ig.Spec.Image, ig.GetName())
			}
		}
	}

	if len(actions) == 0 {
		// TODO: Allow --force option to force even if not needed?
		// Note stderr - we try not to print to stdout if no update is needed
		fmt.Fprintf(os.Stderr, "\nNo upgrade required\n")
		return nil
	}

	{
		t := &tables.Table{}
		t.AddColumn("ITEM", func(a *upgradeAction) string {
			return a.Item
		})
		t.AddColumn("PROPERTY", func(a *upgradeAction) string {
			return a.Property
		})
		t.AddColumn("OLD", func(a *upgradeAction) string {
			return a.Old
		})
		t.AddColumn("NEW", func(a *upgradeAction) string {
			return a.New
		})

		err := t.Render(actions, out, "ITEM", "PROPERTY", "OLD", "NEW")
		if err != nil {
			return err
		}
	}

	if !options.Yes {
		fmt.Printf("\nMust specify --yes to perform upgrade\n")
		return nil
	}
	for _, action := range actions {
		action.apply()
	}

	if err := commands.UpdateCluster(ctx, clientset, cluster, instanceGroups); err != nil {
		return err
	}

	for _, g := range instanceGroups {
		_, err := clientset.InstanceGroupsFor(cluster).Update(ctx, g, metav1.UpdateOptions{})
		if err != nil {
			return fmt.Errorf("error writing InstanceGroup %q: %v", g.ObjectMeta.Name, err)
		}
	}

	fmt.Printf("\nUpdates applied to configuration.\n")

	// TODO: automate this step
	fmt.Printf("You can now apply these changes, using `kops update cluster %s`\n", cluster.ObjectMeta.Name)

	return nil
}

func completeChannel(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
	// TODO implement completion against VFS
	return []string{"alpha", "stable"}, cobra.ShellCompDirectiveNoFileComp
}
<|endoftext|>