stitch_net_log_files.py 3.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128
  1. #!/usr/bin/env python
  2. # Copyright 2016 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. '''
  6. This script "stitches" the NetLog files from a ".inprogress" directory to
  7. create a single NetLog file.
  8. '''
  9. import glob
  10. import os
  11. import re
  12. import sys
  13. USAGE ='''Usage: stitch_net_log_files.py <INPROGRESS_DIR> [<OUTPUT_PATH>]
  14. Will copy all the files in <INPROGRESS_DIR> and write the their content into a
  15. NetLog file at path <OUTPUT_PATH>.
  16. If <OUTPUT_PATH> is not specified, it should end with ".inprogress", and the
  17. completed NetLog file will be written to the location with ".inprogress"
  18. stripped.
  19. '''
  20. def get_event_file_sort_key(path):
  21. '''Returns a tuple (modification timestamp, file number) for a path of the
  22. form event_file_%d.json'''
  23. m = re.match('^event_file_(\d+).json$', path)
  24. file_index = int(m.group(1))
  25. return (os.path.getmtime(path), file_index)
  26. def get_ordered_event_files():
  27. '''Returns a list of file paths to event files. The order of the files is
  28. from oldest to newest. If modification times are the same, files will be
  29. ordered based on the numeral in their file name.'''
  30. paths = glob.glob("event_file_*.json")
  31. paths = sorted(paths, key=get_event_file_sort_key)
  32. sys.stdout.write("Identified %d event files:\n %s\n" %
  33. (len(paths), "\n ".join(paths)))
  34. return paths
  35. def main():
  36. if len(sys.argv) != 2 and len(sys.argv) != 3:
  37. sys.stderr.write(USAGE)
  38. sys.exit(1)
  39. inprogress_dir = sys.argv[1]
  40. output_path = None
  41. # Pick an output path based on command line arguments.
  42. if len(sys.argv) == 3:
  43. output_path = sys.argv[2]
  44. elif len(sys.argv) == 2:
  45. m = re.match("^(.*)\.inprogress/?$", inprogress_dir)
  46. if not m:
  47. sys.stdout.write("Must specify OUTPUT_PATH\n")
  48. sys.exit(1)
  49. output_path = m.group(1)
  50. output_path = os.path.abspath(output_path)
  51. sys.stdout.write("Reading data from: %s\n" % inprogress_dir)
  52. sys.stdout.write("Writing log file to: %s\n" % output_path)
  53. os.chdir(inprogress_dir)
  54. with open(output_path, "w") as stitched_file:
  55. try:
  56. file = open("constants.json")
  57. with file:
  58. for line in file:
  59. stitched_file.write(line)
  60. except IOError:
  61. sys.stderr.write("Failed reading \"constants.json\".\n")
  62. sys.exit(1)
  63. events_written = False;
  64. for event_file_path in get_ordered_event_files():
  65. try:
  66. file = open(event_file_path)
  67. with file:
  68. if not events_written:
  69. line = file.readline();
  70. events_written = True
  71. for next_line in file:
  72. if next_line.strip() == "":
  73. line += next_line
  74. else:
  75. stitched_file.write(line)
  76. line = next_line
  77. except IOError:
  78. sys.stderr.write("Failed reading \"%s\"\n" % event_file_path)
  79. sys.exit(1)
  80. # Remove hanging comma from last event
  81. # TODO(dconnol): Check if the last line is a valid JSON object. If not,
  82. # do not write the line to file. This handles incomplete logs.
  83. line = line.strip()
  84. if line[-1:] == ",":
  85. stitched_file.write(line[:-1])
  86. elif line:
  87. raise ValueError('Last event is not properly formed')
  88. if os.path.exists("end_netlog.json"):
  89. try:
  90. file = open("end_netlog.json")
  91. with file:
  92. for line in file:
  93. stitched_file.write(line)
  94. except IOError:
  95. sys.stderr.write("Failed reading \"end_netlog.json\".\n")
  96. sys.exit(1)
  97. else:
  98. # end_netlog.json won't exist when using this tool to stitch logging
  99. # sessions that didn't shutdown gracefully.
  100. #
  101. # Close the events array and then the log (no polled_data).
  102. stitched_file.write("]}\n")
  103. if __name__ == "__main__":
  104. main()