ninja.go 7.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274
  1. // Copyright 2018 Google Inc. All rights reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package status
  15. import (
  16. "bufio"
  17. "fmt"
  18. "io"
  19. "os"
  20. "regexp"
  21. "strings"
  22. "syscall"
  23. "time"
  24. "google.golang.org/protobuf/proto"
  25. "android/soong/ui/logger"
  26. "android/soong/ui/status/ninja_frontend"
  27. )
  28. // NewNinjaReader reads the protobuf frontend format from ninja and translates it
  29. // into calls on the ToolStatus API.
  30. func NewNinjaReader(ctx logger.Logger, status ToolStatus, fifo string) *NinjaReader {
  31. os.Remove(fifo)
  32. if err := syscall.Mkfifo(fifo, 0666); err != nil {
  33. ctx.Fatalf("Failed to mkfifo(%q): %v", fifo, err)
  34. }
  35. n := &NinjaReader{
  36. status: status,
  37. fifo: fifo,
  38. done: make(chan bool),
  39. cancel: make(chan bool),
  40. }
  41. go n.run()
  42. return n
  43. }
  44. type NinjaReader struct {
  45. status ToolStatus
  46. fifo string
  47. done chan bool
  48. cancel chan bool
  49. }
  50. const NINJA_READER_CLOSE_TIMEOUT = 5 * time.Second
  51. // Close waits for NinjaReader to finish reading from the fifo, or 5 seconds.
  52. func (n *NinjaReader) Close() {
  53. // Signal the goroutine to stop if it is blocking opening the fifo.
  54. close(n.cancel)
  55. timeoutCh := time.After(NINJA_READER_CLOSE_TIMEOUT)
  56. select {
  57. case <-n.done:
  58. // Nothing
  59. case <-timeoutCh:
  60. n.status.Error(fmt.Sprintf("ninja fifo didn't finish after %s", NINJA_READER_CLOSE_TIMEOUT.String()))
  61. }
  62. return
  63. }
  64. func (n *NinjaReader) run() {
  65. defer close(n.done)
  66. // Opening the fifo can block forever if ninja never opens the write end, do it in a goroutine so this
  67. // method can exit on cancel.
  68. fileCh := make(chan *os.File)
  69. go func() {
  70. f, err := os.Open(n.fifo)
  71. if err != nil {
  72. n.status.Error(fmt.Sprintf("Failed to open fifo: %v", err))
  73. close(fileCh)
  74. return
  75. }
  76. fileCh <- f
  77. }()
  78. var f *os.File
  79. select {
  80. case f = <-fileCh:
  81. // Nothing
  82. case <-n.cancel:
  83. return
  84. }
  85. defer f.Close()
  86. r := bufio.NewReader(f)
  87. running := map[uint32]*Action{}
  88. for {
  89. size, err := readVarInt(r)
  90. if err != nil {
  91. if err != io.EOF {
  92. n.status.Error(fmt.Sprintf("Got error reading from ninja: %s", err))
  93. }
  94. return
  95. }
  96. buf := make([]byte, size)
  97. _, err = io.ReadFull(r, buf)
  98. if err != nil {
  99. if err == io.EOF {
  100. n.status.Print(fmt.Sprintf("Missing message of size %d from ninja\n", size))
  101. } else {
  102. n.status.Error(fmt.Sprintf("Got error reading from ninja: %s", err))
  103. }
  104. return
  105. }
  106. msg := &ninja_frontend.Status{}
  107. err = proto.Unmarshal(buf, msg)
  108. if err != nil {
  109. n.status.Print(fmt.Sprintf("Error reading message from ninja: %v", err))
  110. continue
  111. }
  112. // Ignore msg.BuildStarted
  113. if msg.TotalEdges != nil {
  114. n.status.SetTotalActions(int(msg.TotalEdges.GetTotalEdges()))
  115. }
  116. if msg.EdgeStarted != nil {
  117. action := &Action{
  118. Description: msg.EdgeStarted.GetDesc(),
  119. Outputs: msg.EdgeStarted.Outputs,
  120. Inputs: msg.EdgeStarted.Inputs,
  121. Command: msg.EdgeStarted.GetCommand(),
  122. }
  123. n.status.StartAction(action)
  124. running[msg.EdgeStarted.GetId()] = action
  125. }
  126. if msg.EdgeFinished != nil {
  127. if started, ok := running[msg.EdgeFinished.GetId()]; ok {
  128. delete(running, msg.EdgeFinished.GetId())
  129. var err error
  130. exitCode := int(msg.EdgeFinished.GetStatus())
  131. if exitCode != 0 {
  132. err = fmt.Errorf("exited with code: %d", exitCode)
  133. }
  134. outputWithErrorHint := errorHintGenerator.GetOutputWithErrorHint(msg.EdgeFinished.GetOutput(), exitCode)
  135. n.status.FinishAction(ActionResult{
  136. Action: started,
  137. Output: outputWithErrorHint,
  138. Error: err,
  139. Stats: ActionResultStats{
  140. UserTime: msg.EdgeFinished.GetUserTime(),
  141. SystemTime: msg.EdgeFinished.GetSystemTime(),
  142. MaxRssKB: msg.EdgeFinished.GetMaxRssKb(),
  143. MinorPageFaults: msg.EdgeFinished.GetMinorPageFaults(),
  144. MajorPageFaults: msg.EdgeFinished.GetMajorPageFaults(),
  145. IOInputKB: msg.EdgeFinished.GetIoInputKb(),
  146. IOOutputKB: msg.EdgeFinished.GetIoOutputKb(),
  147. VoluntaryContextSwitches: msg.EdgeFinished.GetVoluntaryContextSwitches(),
  148. InvoluntaryContextSwitches: msg.EdgeFinished.GetInvoluntaryContextSwitches(),
  149. Tags: msg.EdgeFinished.GetTags(),
  150. },
  151. })
  152. }
  153. }
  154. if msg.Message != nil {
  155. message := "ninja: " + msg.Message.GetMessage()
  156. switch msg.Message.GetLevel() {
  157. case ninja_frontend.Status_Message_INFO:
  158. n.status.Status(message)
  159. case ninja_frontend.Status_Message_WARNING:
  160. n.status.Print("warning: " + message)
  161. case ninja_frontend.Status_Message_ERROR:
  162. n.status.Error(message)
  163. case ninja_frontend.Status_Message_DEBUG:
  164. n.status.Verbose(message)
  165. default:
  166. n.status.Print(message)
  167. }
  168. }
  169. if msg.BuildFinished != nil {
  170. n.status.Finish()
  171. }
  172. }
  173. }
  174. func readVarInt(r *bufio.Reader) (int, error) {
  175. ret := 0
  176. shift := uint(0)
  177. for {
  178. b, err := r.ReadByte()
  179. if err != nil {
  180. return 0, err
  181. }
  182. ret += int(b&0x7f) << (shift * 7)
  183. if b&0x80 == 0 {
  184. break
  185. }
  186. shift += 1
  187. if shift > 4 {
  188. return 0, fmt.Errorf("Expected varint32 length-delimited message")
  189. }
  190. }
  191. return ret, nil
  192. }
  193. // key is pattern in stdout/stderr
  194. // value is error hint
  195. var allErrorHints = map[string]string{
  196. "Read-only file system": `\nWrite to a read-only file system detected. Possible fixes include
  197. 1. Generate file directly to out/ which is ReadWrite, #recommend solution
  198. 2. BUILD_BROKEN_SRC_DIR_RW_ALLOWLIST := <my/path/1> <my/path/2> #discouraged, subset of source tree will be RW
  199. 3. BUILD_BROKEN_SRC_DIR_IS_WRITABLE := true #highly discouraged, entire source tree will be RW
  200. `,
  201. }
  202. var errorHintGenerator = *newErrorHintGenerator(allErrorHints)
  203. type ErrorHintGenerator struct {
  204. allErrorHints map[string]string
  205. allErrorHintPatternsCompiled *regexp.Regexp
  206. }
  207. func newErrorHintGenerator(allErrorHints map[string]string) *ErrorHintGenerator {
  208. var allErrorHintPatterns []string
  209. for errorHintPattern, _ := range allErrorHints {
  210. allErrorHintPatterns = append(allErrorHintPatterns, errorHintPattern)
  211. }
  212. allErrorHintPatternsRegex := strings.Join(allErrorHintPatterns[:], "|")
  213. re := regexp.MustCompile(allErrorHintPatternsRegex)
  214. return &ErrorHintGenerator{
  215. allErrorHints: allErrorHints,
  216. allErrorHintPatternsCompiled: re,
  217. }
  218. }
  219. func (errorHintGenerator *ErrorHintGenerator) GetOutputWithErrorHint(rawOutput string, buildExitCode int) string {
  220. if buildExitCode == 0 {
  221. return rawOutput
  222. }
  223. errorHint := errorHintGenerator.getErrorHint(rawOutput)
  224. if errorHint == nil {
  225. return rawOutput
  226. }
  227. return rawOutput + *errorHint
  228. }
  229. // Returns the error hint corresponding to the FIRST match in raw output
  230. func (errorHintGenerator *ErrorHintGenerator) getErrorHint(rawOutput string) *string {
  231. firstMatch := errorHintGenerator.allErrorHintPatternsCompiled.FindString(rawOutput)
  232. if _, found := errorHintGenerator.allErrorHints[firstMatch]; found {
  233. errorHint := errorHintGenerator.allErrorHints[firstMatch]
  234. return &errorHint
  235. }
  236. return nil
  237. }