Consume concurrency from compiler

Refers to #33
This commit is contained in:
Ettore Di Giacinto 2019-12-30 14:46:45 +01:00
parent a40ecaea40
commit 475b63be95
No known key found for this signature in database
GPG Key ID: 1ADA699B145A2D1C
5 changed files with 52 additions and 32 deletions

View File

@ -80,8 +80,8 @@ func (cs *LuetCompiler) compilerWorker(i int, wg *sync.WaitGroup, cspecs chan Co
} }
} }
func (cs *LuetCompiler) CompileWithReverseDeps(concurrency int, keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error) { func (cs *LuetCompiler) CompileWithReverseDeps(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error) {
artifacts, err := cs.CompileParallel(concurrency, keepPermissions, ps) artifacts, err := cs.CompileParallel(keepPermissions, ps)
if len(err) != 0 { if len(err) != 0 {
return artifacts, err return artifacts, err
} }
@ -129,11 +129,11 @@ func (cs *LuetCompiler) CompileWithReverseDeps(concurrency int, keepPermissions
Info(" :arrow_right_hook:", u.GetPackage().GetName(), ":leaves:", u.GetPackage().GetVersion(), "(", u.GetPackage().GetCategory(), ")") Info(" :arrow_right_hook:", u.GetPackage().GetName(), ":leaves:", u.GetPackage().GetVersion(), "(", u.GetPackage().GetCategory(), ")")
} }
artifacts2, err := cs.CompileParallel(concurrency, keepPermissions, uniques) artifacts2, err := cs.CompileParallel(keepPermissions, uniques)
return append(artifacts, artifacts2...), err return append(artifacts, artifacts2...), err
} }
func (cs *LuetCompiler) CompileParallel(concurrency int, keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error) { func (cs *LuetCompiler) CompileParallel(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error) {
Spinner(22) Spinner(22)
defer SpinnerStop() defer SpinnerStop()
all := make(chan CompilationSpec) all := make(chan CompilationSpec)
@ -141,9 +141,9 @@ func (cs *LuetCompiler) CompileParallel(concurrency int, keepPermissions bool, p
mutex := &sync.Mutex{} mutex := &sync.Mutex{}
errors := make(chan error, ps.Len()) errors := make(chan error, ps.Len())
var wg = new(sync.WaitGroup) var wg = new(sync.WaitGroup)
for i := 0; i < concurrency; i++ { for i := 0; i < cs.Concurrency; i++ {
wg.Add(1) wg.Add(1)
go cs.compilerWorker(i, wg, all, &artifacts, mutex, concurrency, keepPermissions, errors) go cs.compilerWorker(i, wg, all, &artifacts, mutex, cs.Concurrency, keepPermissions, errors)
} }
for _, p := range ps.All() { for _, p := range ps.All() {
@ -472,13 +472,13 @@ func (cs *LuetCompiler) ComputeDepTree(p CompilationSpec) (solver.PackagesAssert
} }
// Compile is non-parallel // Compile is non-parallel
func (cs *LuetCompiler) Compile(concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) { func (cs *LuetCompiler) Compile(keepPermissions bool, p CompilationSpec) (Artifact, error) {
asserts, err := cs.ComputeDepTree(p) asserts, err := cs.ComputeDepTree(p)
if err != nil { if err != nil {
panic(err) panic(err)
} }
p.SetSourceAssertion(asserts) p.SetSourceAssertion(asserts)
return cs.compile(concurrency, keepPermissions, p) return cs.compile(cs.Concurrency, keepPermissions, p)
} }
func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) { func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) {

View File

@ -53,7 +53,9 @@ var _ = Describe("Compiler", func() {
Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"})) Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
spec.SetOutputPath(tmpdir) spec.SetOutputPath(tmpdir)
artifact, err := compiler.Compile(2, false, spec) compiler.SetConcurrency(2)
artifact, err := compiler.Compile(false, spec)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
@ -95,7 +97,8 @@ var _ = Describe("Compiler", func() {
spec.SetOutputPath(tmpdir) spec.SetOutputPath(tmpdir)
spec2.SetOutputPath(tmpdir) spec2.SetOutputPath(tmpdir)
artifacts, errs := compiler.CompileParallel(2, false, NewLuetCompilationspecs(spec, spec2)) compiler.SetConcurrency(2)
artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec, spec2))
Expect(errs).To(BeNil()) Expect(errs).To(BeNil())
for _, artifact := range artifacts { for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
@ -134,8 +137,9 @@ var _ = Describe("Compiler", func() {
spec.SetOutputPath(tmpdir) spec.SetOutputPath(tmpdir)
spec2.SetOutputPath(tmpdir) spec2.SetOutputPath(tmpdir)
spec3.SetOutputPath(tmpdir) spec3.SetOutputPath(tmpdir)
compiler.SetConcurrency(2)
artifacts, errs := compiler.CompileParallel(2, false, NewLuetCompilationspecs(spec, spec2, spec3)) artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec, spec2, spec3))
Expect(errs).To(BeNil()) Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(3)) Expect(len(artifacts)).To(Equal(3))
@ -181,12 +185,12 @@ var _ = Describe("Compiler", func() {
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
spec.SetOutputPath(tmpdir) spec.SetOutputPath(tmpdir)
spec2.SetOutputPath(tmpdir) spec2.SetOutputPath(tmpdir)
compiler.SetConcurrency(1)
artifacts, errs := compiler.CompileParallel(1, false, NewLuetCompilationspecs(spec)) artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil()) Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1)) Expect(len(artifacts)).To(Equal(1))
artifacts2, errs := compiler.CompileParallel(1, false, NewLuetCompilationspecs(spec2)) artifacts2, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec2))
Expect(errs).To(BeNil()) Expect(errs).To(BeNil())
Expect(len(artifacts2)).To(Equal(1)) Expect(len(artifacts2)).To(Equal(1))
@ -224,8 +228,9 @@ var _ = Describe("Compiler", func() {
// Expect(err).ToNot(HaveOccurred()) // Expect(err).ToNot(HaveOccurred())
spec.SetOutputPath(tmpdir) spec.SetOutputPath(tmpdir)
compiler.SetConcurrency(1)
artifacts, errs := compiler.CompileParallel(1, false, NewLuetCompilationspecs(spec)) artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil()) Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1)) Expect(len(artifacts)).To(Equal(1))
@ -258,8 +263,8 @@ var _ = Describe("Compiler", func() {
// Expect(err).ToNot(HaveOccurred()) // Expect(err).ToNot(HaveOccurred())
spec.SetOutputPath(tmpdir) spec.SetOutputPath(tmpdir)
compiler.SetConcurrency(1)
artifacts, errs := compiler.CompileParallel(1, false, NewLuetCompilationspecs(spec)) artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil()) Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1)) Expect(len(artifacts)).To(Equal(1))
@ -296,8 +301,9 @@ var _ = Describe("Compiler", func() {
// Expect(err).ToNot(HaveOccurred()) // Expect(err).ToNot(HaveOccurred())
spec.SetOutputPath(tmpdir) spec.SetOutputPath(tmpdir)
compiler.SetConcurrency(1)
artifacts, errs := compiler.CompileParallel(1, false, NewLuetCompilationspecs(spec)) artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil()) Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1)) Expect(len(artifacts)).To(Equal(1))
@ -336,8 +342,9 @@ var _ = Describe("Compiler", func() {
// Expect(err).ToNot(HaveOccurred()) // Expect(err).ToNot(HaveOccurred())
spec.SetOutputPath(tmpdir) spec.SetOutputPath(tmpdir)
compiler.SetConcurrency(1)
artifacts, errs := compiler.CompileParallel(1, false, NewLuetCompilationspecs(spec)) artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil()) Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1)) Expect(len(artifacts)).To(Equal(1))
Expect(len(artifacts[0].GetDependencies())).To(Equal(1)) Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
@ -379,8 +386,9 @@ var _ = Describe("Compiler", func() {
// Expect(err).ToNot(HaveOccurred()) // Expect(err).ToNot(HaveOccurred())
spec.SetOutputPath(tmpdir) spec.SetOutputPath(tmpdir)
compiler.SetConcurrency(1)
artifacts, errs := compiler.CompileParallel(1, false, NewLuetCompilationspecs(spec)) artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil()) Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1)) Expect(len(artifacts)).To(Equal(1))
Expect(len(artifacts[0].GetDependencies())).To(Equal(1)) Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
@ -421,7 +429,7 @@ var _ = Describe("Compiler", func() {
spec.SetOutputPath(tmpdir) spec.SetOutputPath(tmpdir)
artifacts, errs := compiler.CompileWithReverseDeps(1, false, NewLuetCompilationspecs(spec)) artifacts, errs := compiler.CompileWithReverseDeps(false, NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil()) Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(2)) Expect(len(artifacts)).To(Equal(2))
@ -459,7 +467,7 @@ var _ = Describe("Compiler", func() {
spec.SetOutputPath(tmpdir) spec.SetOutputPath(tmpdir)
artifacts, errs := compiler.CompileParallel(1, false, NewLuetCompilationspecs(spec)) artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil()) Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1)) Expect(len(artifacts)).To(Equal(1))
Expect(len(artifacts[0].GetDependencies())).To(Equal(6)) Expect(len(artifacts[0].GetDependencies())).To(Equal(6))
@ -494,7 +502,7 @@ var _ = Describe("Compiler", func() {
spec.SetOutputPath(tmpdir) spec.SetOutputPath(tmpdir)
artifacts, errs := compiler.CompileWithReverseDeps(1, false, NewLuetCompilationspecs(spec)) artifacts, errs := compiler.CompileWithReverseDeps(false, NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil()) Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(4)) Expect(len(artifacts)).To(Equal(4))
@ -552,7 +560,9 @@ var _ = Describe("Compiler", func() {
defer os.RemoveAll(tmpdir) // clean up defer os.RemoveAll(tmpdir) // clean up
spec.SetOutputPath(tmpdir) spec.SetOutputPath(tmpdir)
artifacts, errs := compiler.CompileParallel(2, false, NewLuetCompilationspecs(spec)) compiler.SetConcurrency(2)
artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil()) Expect(errs).To(BeNil())
for _, artifact := range artifacts { for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
@ -593,7 +603,9 @@ var _ = Describe("Compiler", func() {
defer os.RemoveAll(tmpdir) // clean up defer os.RemoveAll(tmpdir) // clean up
spec.SetOutputPath(tmpdir) spec.SetOutputPath(tmpdir)
artifacts, errs := compiler.CompileParallel(2, false, NewLuetCompilationspecs(spec)) compiler.SetConcurrency(1)
artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil()) Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1)) Expect(len(artifacts)).To(Equal(1))
Expect(len(artifacts[0].GetDependencies())).To(Equal(1)) Expect(len(artifacts[0].GetDependencies())).To(Equal(1))

View File

@ -21,9 +21,9 @@ import (
) )
type Compiler interface { type Compiler interface {
Compile(int, bool, CompilationSpec) (Artifact, error) Compile(bool, CompilationSpec) (Artifact, error)
CompileParallel(concurrency int, keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error) CompileParallel(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
CompileWithReverseDeps(concurrency int, keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error) CompileWithReverseDeps(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
ComputeDepTree(p CompilationSpec) (solver.PackagesAssertions, error) ComputeDepTree(p CompilationSpec) (solver.PackagesAssertions, error)
SetConcurrency(i int) SetConcurrency(i int)
FromPackage(pkg.Package) (CompilationSpec, error) FromPackage(pkg.Package) (CompilationSpec, error)

View File

@ -62,7 +62,9 @@ var _ = Describe("Installer", func() {
Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"})) Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
spec.SetOutputPath(tmpdir) spec.SetOutputPath(tmpdir)
artifact, err := compiler.Compile(2, false, spec) compiler.SetConcurrency(2)
artifact, err := compiler.Compile(false, spec)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
@ -172,7 +174,9 @@ uri: "`+tmpdir+`"
Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"})) Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
spec.SetOutputPath(tmpdir) spec.SetOutputPath(tmpdir)
artifact, err := compiler.Compile(2, false, spec) compiler.SetConcurrency(2)
artifact, err := compiler.Compile(false, spec)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
@ -291,7 +295,9 @@ uri: "`+tmpdir+`"
spec.SetOutputPath(tmpdir) spec.SetOutputPath(tmpdir)
spec2.SetOutputPath(tmpdir) spec2.SetOutputPath(tmpdir)
spec3.SetOutputPath(tmpdir) spec3.SetOutputPath(tmpdir)
_, errs := c.CompileParallel(2, false, compiler.NewLuetCompilationspecs(spec, spec2, spec3)) c.SetConcurrency(2)
_, errs := c.CompileParallel(false, compiler.NewLuetCompilationspecs(spec, spec2, spec3))
Expect(errs).To(BeEmpty()) Expect(errs).To(BeEmpty())

View File

@ -61,7 +61,9 @@ var _ = Describe("Repository", func() {
Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"})) Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
spec.SetOutputPath(tmpdir) spec.SetOutputPath(tmpdir)
artifact, err := compiler.Compile(2, false, spec) compiler.SetConcurrency(1)
artifact, err := compiler.Compile(false, spec)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())