go: Update reproducibility patch to fix panic errors
A message that is not in the archive (sent off-list or not yet received): [email protected]
This might have gotten lost in the fun of the YPS and the many discussions around the previous versions of then patch. Was there any more feedback or changes that folks want me to make for this patch, or is it good to go? Just curious if there was more that you needed from me?
Quoted 143 lines · Ryan Eatmon via lists.openembedded.org
Based on a discussion on the mailing list [1], there are panic errors that occur on a few platforms caused by the patch. We cannot simply remove the original patch due to the reproducibility issues that it addresses, so this patch on the original patch fixes the cause of the panic errors.
The previous version of this patch was a little too aggressive in cleaning up the environment. Some of the variables impacted by the
filerCompilerFlags()function require at least one value to remain in the array. In this case, the values for ccExe, cxxExe, and fcExe require a value or later code that access them result in a panic related to accessing a value out of range.This updated patch adds a flag that requires keeping the first value so that at least one thing remains and the assignments for the Exes set that flag to true. The first item in the array should be the executable name, so leaving it should be safe.
I have run the oe-selftest and everything passed in my setup.
There is a bug report [2] filed for the issue that this patch addresses.
[YOCTO #14976]
[1] https://lists.openembedded.org/g/openembedded-core/topic/94022663
[2] https://bugzilla.yoctoproject.org/show_bug.cgi?id=14976Signed-off-by Ryan Eatmon <[email protected]>---
v3: Added [YOCTO #14976] to commit message.
v2: Added more background and bug report link to commit message....ent-based-hash-generation-less-pedan.patch | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-)meta/recipes-devtools/go/go/0001-cmd-go-make-content-based-hash-generation-less-pedan.patch +16
-15@@ -74,7 +74,7 @@ index c88b315..a06455c 100644 + cppflags, cflags, cxxflags, fflags, ldflags, _ := b.CFlags(p, true) - ccExe := b.ccExe() -+ ccExe := filterCompilerFlags(b.ccExe()) ++ ccExe := filterCompilerFlags(b.ccExe(), true) fmt.Fprintf(h, "CC=%q %q %q %q\n", ccExe, cppflags, cflags, ldflags) // Include the C compiler tool ID so that if the C // compiler changes we rebuild the package.@@ -83,7 +83,7 @@ index c88b315..a06455c 100644 } if len(p.CXXFiles)+len(p.SwigCXXFiles) > 0 { - cxxExe := b.cxxExe() -+ cxxExe := filterCompilerFlags(b.cxxExe()) ++ cxxExe := filterCompilerFlags(b.cxxExe(), true) fmt.Fprintf(h, "CXX=%q %q\n", cxxExe, cxxflags) if cxxID, err := b.gccToolID(cxxExe[0], "c++"); err == nil { fmt.Fprintf(h, "CXX ID=%q\n", cxxID)@@ -91,7 +91,7 @@ index c88b315..a06455c 100644 } if len(p.FFiles) > 0 { - fcExe := b.fcExe() -+ fcExe := filterCompilerFlags(b.fcExe()) ++ fcExe := filterCompilerFlags(b.fcExe(), true) fmt.Fprintf(h, "FC=%q %q\n", fcExe, fflags) if fcID, err := b.gccToolID(fcExe[0], "f95"); err == nil { fmt.Fprintf(h, "FC ID=%q\n", fcID)@@ -104,20 +104,22 @@ index c88b315..a06455c 100644 } // Configuration specific to compiler toolchain. -@@ -2705,8 +2707,23 @@ func envList(key, def string) []string { +@@ -2705,8 +2707,25 @@ func envList(key, def string) []string { return args } +var filterFlags = os.Getenv("CGO_PEDANTIC") == "" + -+func filterCompilerFlags(flags []string) []string { ++func filterCompilerFlags(flags []string, keepfirst bool) []string { + var newflags []string ++ var realkeepfirst bool = keepfirst + if !filterFlags { + return flags + } + for _, flag := range flags { -+ if strings.HasPrefix(flag, "-m") { ++ if strings.HasPrefix(flag, "-m") || realkeepfirst { + newflags = append(newflags, flag) ++ realkeepfirst = false + } + } + return newflags@@ -129,21 +131,21 @@ index c88b315..a06455c 100644 defaults := "-g -O2" if cppflags, err = buildFlags("CPPFLAGS", "", p.CgoCPPFLAGS, checkCompilerFlags); err != nil { -@@ -2724,6 +2741,13 @@ func (b *Builder) CFlags(p *load.Package) (cppflags, cflags, cxxflags, fflags, l +@@ -2724,6 +2743,13 @@ func (b *Builder) CFlags(p *load.Package) (cppflags, cflags, cxxflags, fflags, l if ldflags, err = buildFlags("LDFLAGS", defaults, p.CgoLDFLAGS, checkLinkerFlags); err != nil { return } + if filtered { -+ cppflags = filterCompilerFlags(cppflags) -+ cflags = filterCompilerFlags(cflags) -+ cxxflags = filterCompilerFlags(cxxflags) -+ fflags = filterCompilerFlags(fflags) -+ ldflags = filterCompilerFlags(ldflags) ++ cppflags = filterCompilerFlags(cppflags, false) ++ cflags = filterCompilerFlags(cflags, false) ++ cxxflags = filterCompilerFlags(cxxflags, false) ++ fflags = filterCompilerFlags(fflags, false) ++ ldflags = filterCompilerFlags(ldflags, false) + } return } -@@ -2739,7 +2763,7 @@ var cgoRe = lazyregexp.New(`[/\\:]`) +@@ -2739,7 +2765,7 @@ var cgoRe = lazyregexp.New(`[/\\:]`) func (b *Builder) cgo(a *Action, cgoExe, objdir string, pcCFLAGS, pcLDFLAGS, cgofiles, gccfiles, gxxfiles, mfiles, ffiles []string) (outGo, outObj []string, err error) { p := a.Package@@ -152,7 +154,7 @@ index c88b315..a06455c 100644 if err != nil { return nil, nil, err } -@@ -3246,7 +3270,7 @@ func (b *Builder) swigIntSize(objdir string) (intsize string, err error) { +@@ -3246,7 +3272,7 @@ func (b *Builder) swigIntSize(objdir string) (intsize string, err error) { // Run SWIG on one SWIG input file. func (b *Builder) swigOne(a *Action, p *load.Package, file, objdir string, pcCFLAGS []string, cxx bool, intgosize string) (outGo, outC string, err error) { -=-=-=-=-=-=-=-=-=-=-=-Links: You receive all messages sent to this group.
View/Reply Online (#173973): https://lists.openembedded.org/g/openembedded-core/message/173973
Mute This Topic: https://lists.openembedded.org/mt/95334740/6551054
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
Signature · 3 lines
Ryan Eatmon [email protected] ----------------------------------------- Texas Instruments, Inc. - LCPD - MGTS
Hello,
Quoted 6 lines · Ryan Eatmon via lists.openembedded.org
This might have gotten lost in the fun of the YPS and the many discussions around the previous versions of then patch. Was there any more feedback or changes that folks want me to make for this patch, or is it good to go? Just curious if there was more that you needed from me?
It passed successfully on the autobuilders, it is now up to Richard to merge.
Quoted 147 lines · Ryan Eatmon via lists.openembedded.org
Quoted 130 lines
Based on a discussion on the mailing list [1], there are panic errors that occur on a few platforms caused by the patch. We cannot simply remove the original patch due to the reproducibility issues that it addresses, so this patch on the original patch fixes the cause of the panic errors.
The previous version of this patch was a little too aggressive in cleaning up the environment. Some of the variables impacted by the
filerCompilerFlags()function require at least one value to remain in the array. In this case, the values for ccExe, cxxExe, and fcExe require a value or later code that access them result in a panic related to accessing a value out of range.This updated patch adds a flag that requires keeping the first value so that at least one thing remains and the assignments for the Exes set that flag to true. The first item in the array should be the executable name, so leaving it should be safe.
I have run the oe-selftest and everything passed in my setup.
There is a bug report [2] filed for the issue that this patch addresses.
[YOCTO #14976]
[1] https://lists.openembedded.org/g/openembedded-core/topic/94022663
[2] https://bugzilla.yoctoproject.org/show_bug.cgi?id=14976Signed-off-by Ryan Eatmon <[email protected]>---
v3: Added [YOCTO #14976] to commit message.
v2: Added more background and bug report link to commit message....ent-based-hash-generation-less-pedan.patch | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-)meta/recipes-devtools/go/go/0001-cmd-go-make-content-based-hash-generation-less-pedan.patch +16
-14@@ -74,7 +74,7 @@ index c88b315..a06455c 100644 + cppflags, cflags, cxxflags, fflags, ldflags, _ := b.CFlags(p, true) - ccExe := b.ccExe() -+ ccExe := filterCompilerFlags(b.ccExe()) ++ ccExe := filterCompilerFlags(b.ccExe(), true) fmt.Fprintf(h, "CC=%q %q %q %q\n", ccExe, cppflags, cflags, ldflags) // Include the C compiler tool ID so that if the C // compiler changes we rebuild the package.@@ -83,7 +83,7 @@ index c88b315..a06455c 100644 } if len(p.CXXFiles)+len(p.SwigCXXFiles) > 0 { - cxxExe := b.cxxExe() -+ cxxExe := filterCompilerFlags(b.cxxExe()) ++ cxxExe := filterCompilerFlags(b.cxxExe(), true) fmt.Fprintf(h, "CXX=%q %q\n", cxxExe, cxxflags) if cxxID, err := b.gccToolID(cxxExe[0], "c++"); err == nil { fmt.Fprintf(h, "CXX ID=%q\n", cxxID)@@ -91,7 +91,7 @@ index c88b315..a06455c 100644 } if len(p.FFiles) > 0 { - fcExe := b.fcExe() -+ fcExe := filterCompilerFlags(b.fcExe()) ++ fcExe := filterCompilerFlags(b.fcExe(), true) fmt.Fprintf(h, "FC=%q %q\n", fcExe, fflags) if fcID, err := b.gccToolID(fcExe[0], "f95"); err == nil { fmt.Fprintf(h, "FC ID=%q\n", fcID)@@ -104,20 +104,22 @@ index c88b315..a06455c 100644 } // Configuration specific to compiler toolchain. -@@ -2705,8 +2707,23 @@ func envList(key, def string) []string { +@@ -2705,8 +2707,25 @@ func envList(key, def string) []string { return args } +var filterFlags = os.Getenv("CGO_PEDANTIC") == "" + -+func filterCompilerFlags(flags []string) []string { ++func filterCompilerFlags(flags []string, keepfirst bool) []string { + var newflags []string ++ var realkeepfirst bool = keepfirst + if !filterFlags { + return flags + } + for _, flag := range flags { -+ if strings.HasPrefix(flag, "-m") { ++ if strings.HasPrefix(flag, "-m") || realkeepfirst { + newflags = append(newflags, flag) ++ realkeepfirst = false + } + } + return newflags@@ -129,21 +131,21 @@ index c88b315..a06455c 100644 defaults := "-g -O2" if cppflags, err = buildFlags("CPPFLAGS", "", p.CgoCPPFLAGS, checkCompilerFlags); err != nil { -@@ -2724,6 +2741,13 @@ func (b *Builder) CFlags(p *load.Package) (cppflags, cflags, cxxflags, fflags, l +@@ -2724,6 +2743,13 @@ func (b *Builder) CFlags(p *load.Package) (cppflags, cflags, cxxflags, fflags, l if ldflags, err = buildFlags("LDFLAGS", defaults, p.CgoLDFLAGS, checkLinkerFlags); err != nil { return } + if filtered { -+ cppflags = filterCompilerFlags(cppflags) -+ cflags = filterCompilerFlags(cflags) -+ cxxflags = filterCompilerFlags(cxxflags) -+ fflags = filterCompilerFlags(fflags) -+ ldflags = filterCompilerFlags(ldflags) ++ cppflags = filterCompilerFlags(cppflags, false) ++ cflags = filterCompilerFlags(cflags, false) ++ cxxflags = filterCompilerFlags(cxxflags, false) ++ fflags = filterCompilerFlags(fflags, false) ++ ldflags = filterCompilerFlags(ldflags, false) + } return } -@@ -2739,7 +2763,7 @@ var cgoRe = lazyregexp.New(`[/\\:]`) +@@ -2739,7 +2765,7 @@ var cgoRe = lazyregexp.New(`[/\\:]`) func (b *Builder) cgo(a *Action, cgoExe, objdir string, pcCFLAGS, pcLDFLAGS, cgofiles, gccfiles, gxxfiles, mfiles, ffiles []string) (outGo, outObj []string, err error) { p := a.Package@@ -152,7 +154,7 @@ index c88b315..a06455c 100644 if err != nil { return nil, nil, err } -@@ -3246,7 +3270,7 @@ func (b *Builder) swigIntSize(objdir string) (intsize string, err error) { +@@ -3246,7 +3272,7 @@ func (b *Builder) swigIntSize(objdir string) (intsize string, err error) { // Run SWIG on one SWIG input file. func (b *Builder) swigOne(a *Action, p *load.Package, file, objdir string, pcCFLAGS []string, cxx bool, intgosize string) (outGo, outC string, err error) {Signature · 12 lines
Ryan Eatmon [email protected] ----------------------------------------- Texas Instruments, Inc. - LCPD - MGTS -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#174301): https://lists.openembedded.org/g/openembedded-core/message/174301 Mute This Topic: https://lists.openembedded.org/mt/95334740/3617179 Group Owner: [email protected] Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]] -=-=-=-=-=-=-=-=-=-=-=-
Signature · 3 lines
Alexandre Belloni, co-owner and COO, Bootlin Embedded Linux and Kernel engineering https://bootlin.com
Quoted 4 lines · Ryan Eatmon via lists.openembedded.org
This might have gotten lost in the fun of the YPS and the many discussions around the previous versions of then patch. Was there any more feedback or changes that folks want me to make for this patch, or is it good to go? Just curious if there was more that you needed from me?
I've merged it but I am worried there is a deeper problem here we could really do with solving properly. We should probably at least have an open bug about resolving the issue the patch addresses in a better way.
Cheers,
Richard
Hi Ryan,
I am building with this patch and it fixes the problem in my machines, riscv and am6xxx.
Thanks for fixing this issue.
Jose
Ryan Eatmon via lists.openembedded.org <reatmon= [email protected]> escreveu no dia segunda, 5/12/2022 à(s) 19:49:
Quoted 180 lines
This might have gotten lost in the fun of the YPS and the many discussions around the previous versions of then patch. Was there any more feedback or changes that folks want me to make for this patch, or is it good to go? Just curious if there was more that you needed from me?
Quoted 30 lines · Ryan Eatmon via lists.openembedded.org
Based on a discussion on the mailing list [1], there are panic errors that occur on a few platforms caused by the patch. We cannot simply remove the original patch due to the reproducibility issues that it addresses, so this patch on the original patch fixes the cause of the panic errors.
The previous version of this patch was a little too aggressive in cleaning up the environment. Some of the variables impacted by the
filerCompilerFlags()function require at least one value to remain in the array. In this case, the values for ccExe, cxxExe, and fcExe require a value or later code that access them result in a panic related to accessing a value out of range.This updated patch adds a flag that requires keeping the first value so that at least one thing remains and the assignments for the Exes set that flag to true. The first item in the array should be the executable name, so leaving it should be safe.
I have run the oe-selftest and everything passed in my setup.
There is a bug report [2] filed for the issue that this patch addresses.
[YOCTO #14976]
[1] https://lists.openembedded.org/g/openembedded-core/topic/94022663
[2] https://bugzilla.yoctoproject.org/show_bug.cgi?id=14976Signed-off-by Ryan Eatmon <[email protected]>Quoted 8 lines
---
v3: Added [YOCTO #14976] to commit message.
v2: Added more background and bug report link to commit message....ent-based-hash-generation-less-pedan.patch | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-)diff --git
a/meta/recipes-devtools/go/go/0001-cmd-go-make-content-based-hash-generation-less-pedan.patchb/meta/recipes-devtools/go/go/0001-cmd-go-make-content-based-hash-generation-less-pedan.patchQuoted 2 lines
index
17fa9d9831..43be5cd2e8100644
---
a/meta/recipes-devtools/go/go/0001-cmd-go-make-content-based-hash-generation-less-pedan.patchQuoted 1 line
+++
b/meta/recipes-devtools/go/go/0001-cmd-go-make-content-based-hash-generation-less-pedan.patchReview index c88b315..a06455c 100644 · line 74
Quoted 2 lines
(quoted diff) +0
-0@@ -74,7 +74,7 @@ index c88b315..a06455c 100644 + cppflags, cflags, cxxflags, fflags, ldflags, _ :=
b.CFlags(p, true)Review index c88b315..a06455c 100644 · line 77
Quoted 5 lines
(quoted diff) +1
-1- ccExe := b.ccExe() -+ ccExe := filterCompilerFlags(b.ccExe()) ++ ccExe := filterCompilerFlags(b.ccExe(), true) fmt.Fprintf(h, "CC=%q %q %q %q\n", ccExe, cppflags,cflags, ldflags)
Review index c88b315..a06455c 100644 · line 88
Quoted 10 lines
(quoted diff) +1
-1// Include the C compiler tool ID so that if the C // compiler changes we rebuild the package.@@ -83,7 +83,7 @@ index c88b315..a06455c 100644 } if len(p.CXXFiles)+len(p.SwigCXXFiles) > 0 { - cxxExe := b.cxxExe() -+ cxxExe := filterCompilerFlags(b.cxxExe()) ++ cxxExe := filterCompilerFlags(b.cxxExe(), true) fmt.Fprintf(h, "CXX=%q %q\n", cxxExe, cxxflags) if cxxID, err := b.gccToolID(cxxExe[0], "c++");err == nil {
Review index c88b315..a06455c 100644 · line 96
Quoted 9 lines
(quoted diff) +1
-1fmt.Fprintf(h, "CXX ID=%q\n", cxxID)@@ -91,7 +91,7 @@ index c88b315..a06455c 100644 } if len(p.FFiles) > 0 { - fcExe := b.fcExe() -+ fcExe := filterCompilerFlags(b.fcExe()) ++ fcExe := filterCompilerFlags(b.fcExe(), true) fmt.Fprintf(h, "FC=%q %q\n", fcExe, fflags) if fcID, err := b.gccToolID(fcExe[0], "f95"); err== nil {
Quoted 28 lines
(quoted diff) +5
-3fmt.Fprintf(h, "FC ID=%q\n", fcID)@@ -104,20 +104,22 @@ index c88b315..a06455c 100644 } // Configuration specific to compiler toolchain. -@@ -2705,8 +2707,23 @@ func envList(key, def string) []string { +@@ -2705,8 +2707,25 @@ func envList(key, def string) []string { return args } +var filterFlags = os.Getenv("CGO_PEDANTIC") == "" + -+func filterCompilerFlags(flags []string) []string { ++func filterCompilerFlags(flags []string, keepfirst bool) []string { + var newflags []string ++ var realkeepfirst bool = keepfirst + if !filterFlags { + return flags + } + for _, flag := range flags { -+ if strings.HasPrefix(flag, "-m") { ++ if strings.HasPrefix(flag, "-m") || realkeepfirst { + newflags = append(newflags, flag) ++ realkeepfirst = false + } + } + return newflagsReview index c88b315..a06455c 100644 · line 133
Quoted 3 lines
(quoted diff) +0
-0@@ -129,21 +131,21 @@ index c88b315..a06455c 100644 defaults := "-g -O2" if cppflags, err = buildFlags("CPPFLAGS", "", p.CgoCPPFLAGS,checkCompilerFlags); err != nil {
Review index c88b315..a06455c 100644 · line 132
Quoted 1 line
(quoted diff) +0
-1-@@ -2724,6 +2741,13 @@ func (b *Builder) CFlags(p *load.Package)(cppflags, cflags, cxxflags, fflags, l
Review index c88b315..a06455c 100644 · line 134
Quoted 1 line
(quoted diff) +1
-0+@@ -2724,6 +2743,13 @@ func (b *Builder) CFlags(p *load.Package)(cppflags, cflags, cxxflags, fflags, l
Review index c88b315..a06455c 100644 · line 135
Quoted 1 line
(quoted diff) +0
-0if ldflags, err = buildFlags("LDFLAGS", defaults, p.CgoLDFLAGS,checkLinkerFlags); err != nil {
Quoted 11 lines
(quoted diff) +3
-5return } + if filtered { -+ cppflags = filterCompilerFlags(cppflags) -+ cflags = filterCompilerFlags(cflags) -+ cxxflags = filterCompilerFlags(cxxflags) -+ fflags = filterCompilerFlags(fflags) -+ ldflags = filterCompilerFlags(ldflags) ++ cppflags = filterCompilerFlags(cppflags, false) ++ cflags = filterCompilerFlags(cflags, false) ++ cxxflags = filterCompilerFlags(cxxflags, false)Review index c88b315..a06455c 100644 · line 150
Quoted 10 lines
(quoted diff) +3
-1++ fflags = filterCompilerFlags(fflags, false) ++ ldflags = filterCompilerFlags(ldflags, false) + } return } -@@ -2739,7 +2763,7 @@ var cgoRe = lazyregexp.New(`[/\\:]`) +@@ -2739,7 +2765,7 @@ var cgoRe = lazyregexp.New(`[/\\:]`) func (b *Builder) cgo(a *Action, cgoExe, objdir string, pcCFLAGS,pcLDFLAGS, cgofiles, gccfiles, gxxfiles, mfiles, ffiles []string) (outGo, outObj []string, err error) {
Review index c88b315..a06455c 100644 · line 155
Quoted 6 lines
(quoted diff) +0
-1p := a.Package@@ -152,7 +154,7 @@ index c88b315..a06455c 100644 if err != nil { return nil, nil, err } -@@ -3246,7 +3270,7 @@ func (b *Builder) swigIntSize(objdir string)(intsize string, err error) {
Review index c88b315..a06455c 100644 · line 157
Quoted 1 line
(quoted diff) +1
-0+@@ -3246,7 +3272,7 @@ func (b *Builder) swigIntSize(objdir string)(intsize string, err error) {
Review index c88b315..a06455c 100644 · line 159
Quoted 3 lines
(quoted diff) +0
-0// Run SWIG on one SWIG input file. func (b *Builder) swigOne(a *Action, p *load.Package, file, objdirstring, pcCFLAGS []string, cxx bool, intgosize string) (outGo, outC string, err error) {
Quoted 5 lines
--
Ryan Eatmon [email protected]
-----------------------------------------
Texas Instruments, Inc. - LCPD - MGTS-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group. View/Reply Online (#174301):
https://lists.openembedded.org/g/openembedded-core/message/174301
Mute This Topic: https://lists.openembedded.org/mt/95334740/5052612
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [ [email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
Signature · 3 lines
Best regards, José Quaresma