redaction_tool.cc 36 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933
  1. // Copyright 2015 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "components/feedback/redaction_tool.h"
  5. #include <set>
  6. #include <utility>
  7. #include <vector>
  8. #include "base/files/file_path.h"
  9. #include "base/strings/strcat.h"
  10. #include "base/strings/string_number_conversions.h"
  11. #include "base/strings/string_util.h"
  12. #include "base/strings/stringprintf.h"
  13. #include "base/threading/thread_restrictions.h"
  14. #include "build/chromeos_buildflags.h"
  15. #include "components/feedback/pii_types.h"
  16. #include "net/base/ip_address.h"
  17. #include "third_party/re2/src/re2/re2.h"
  18. using re2::RE2;
  19. namespace feedback {
  20. namespace {
  21. // The |kCustomPatternsWithContext| array defines patterns to match and
  22. // redact. Each pattern needs to define three capturing parentheses groups:
  23. //
  24. // - a group for the pattern before the identifier to be redacted;
  25. // - a group for the identifier to be redacted;
  26. // - a group for the pattern after the identifier to be redacted.
  27. //
  28. // The first and the last capture group are the origin of the "WithContext"
  29. // suffix in the name of this constant.
  30. //
  31. // Every matched identifier (in the context of the whole pattern) is redacted
  32. // by replacing it with an incremental instance identifier. Every different
  33. // pattern defines a separate instance identifier space. See the unit test for
  34. // RedactionToolTest::RedactCustomPatterns for pattern redaction examples.
  35. //
  36. // Useful regular expression syntax:
  37. //
  38. // +? is a non-greedy (lazy) +.
  39. // \b matches a word boundary.
  40. // (?i) turns on case insensitivity for the remainder of the regex.
  41. // (?-s) turns off "dot matches newline" for the remainder of the regex.
  42. // (?:regex) denotes non-capturing parentheses group.
  43. CustomPatternWithAlias kCustomPatternsWithContext[] = {
  44. // ModemManager
  45. {"CellID", "(\\bCell ID: ')([0-9a-fA-F]+)(')", PIIType::kLocationInfo},
  46. {"LocAC", "(\\bLocation area code: ')([0-9a-fA-F]+)(')",
  47. PIIType::kLocationInfo},
  48. // Android. Must run first since this expression matches the replacement.
  49. {"SSID", "(?i-s)(\\SSID: ['\"]??)(.+)(['\"]??)", PIIType::kSSID},
  50. // wpa_supplicant
  51. {"SSID", "(?i-s)(\\bssid[= ]')(.+)(')", PIIType::kSSID},
  52. {"SSID", "(?i-s)(\\bssid[= ]\")(.+)(\")", PIIType::kSSID},
  53. {"SSID", "(\\* SSID=)(.+)($)", PIIType::kSSID},
  54. {"SSIDHex", "(?-s)(\\bSSID - hexdump\\(len=[0-9]+\\): )(.+)()",
  55. PIIType::kSSID},
  56. // shill
  57. {"SSID", "(?-s)(\\[SSID=)(.+?)(\\])", PIIType::kSSID},
  58. // Serial numbers. The actual serial number itself can include any alphanum
  59. // char as well as dashes, periods, colons, slashes and unprintable ASCII
  60. // chars (except newline). The second one is for a special case in
  61. // edid-decode, where if we genericized it further then we would catch too
  62. // many other cases that we don't want to redact.
  63. {"Serial",
  64. "(?i-s)(\\bserial\\s*_?(?:number)?['\"]?\\s*[:=|]\\s*['\"]?)"
  65. "([0-9a-zA-Z\\-.:\\/\\\\\\x00-\\x09\\x0B-\\x1F]+)(\\b)",
  66. PIIType::kSerial},
  67. {"Serial", "( Serial Number )(\\d+)(\\b)", PIIType::kSerial},
  68. // The attested device id, a serial number, that comes from vpd_2.0.txt.
  69. // The pattern was recently clarified as being a case insensitive string of
  70. // ASCII letters and digits, plus the dash/hyphen character. The dash cannot
  71. // appear first or last
  72. {"Serial", "(\"attested_device_id\"=\")([^-][0-9a-zA-Z-]+[^-])(\")",
  73. PIIType::kSerial},
  74. // GAIA IDs
  75. {"GAIA", R"xxx((\"?\bgaia_id\"?[=:]['\"])(\d+)(\b['\"]))xxx",
  76. PIIType::kGaiaID},
  77. {"GAIA", R"xxx((\{id: )(\d+)(, email:))xxx", PIIType::kGaiaID},
  78. // UUIDs given by the 'blkid' tool. These don't necessarily look like
  79. // standard UUIDs, so treat them specially.
  80. {"UUID", R"xxx((UUID=")([0-9a-zA-Z-]+)("))xxx", PIIType::kStableIdentifier},
  81. // Also cover UUIDs given by the 'lvs' and 'pvs' tools, which similarly
  82. // don't necessarily look like standard UUIDs.
  83. {"UUID", R"xxx(("[lp]v_uuid":")([0-9a-zA-Z-]+)("))xxx",
  84. PIIType::kStableIdentifier},
  85. // Volume labels presented in the 'blkid' tool, and as part of removable
  86. // media paths shown in various logs such as cros-disks (in syslog).
  87. // There isn't a well-defined format for these. For labels in blkid,
  88. // capture everything between the open and closing quote.
  89. {"Volume Label", R"xxx((LABEL=")([^"]+)("))xxx", PIIType::kVolumeLabel},
  90. // For paths, this is harder. The only restricted characters are '/' and
  91. // NUL, so use a simple heuristic. cros-disks generally quotes paths using
  92. // single-quotes, so capture everything until a quote character. For lsblk,
  93. // capture everything until the end of the line, since the mount path is the
  94. // last field.
  95. {"Volume Label", R"xxx((/media/removable/)(.+?)(['"/\n]|$))xxx",
  96. PIIType::kVolumeLabel},
  97. // IPP (Internet Printing Protocol) Addresses
  98. {"IPP Address", R"xxx((ipp:\/\/)(.+?)(\/ipp))xxx", PIIType::kIPPAddress},
  99. };
  100. bool MaybeUnmapAddress(net::IPAddress* addr) {
  101. if (!addr->IsIPv4MappedIPv6())
  102. return false;
  103. *addr = net::ConvertIPv4MappedIPv6ToIPv4(*addr);
  104. return true;
  105. }
  106. bool MaybeUntranslateAddress(net::IPAddress* addr) {
  107. if (!addr->IsIPv6())
  108. return false;
  109. static const net::IPAddress kTranslated6To4(0, 0x64, 0xff, 0x9b, 0, 0, 0, 0,
  110. 0, 0, 0, 0, 0, 0, 0, 0);
  111. if (!IPAddressMatchesPrefix(*addr, kTranslated6To4, 96))
  112. return false;
  113. const auto bytes = addr->bytes();
  114. *addr = net::IPAddress(bytes[12], bytes[13], bytes[14], bytes[15]);
  115. return true;
  116. }
  117. // If |addr| points to a valid IPv6 address, this function truncates it at /32.
  118. bool MaybeTruncateIPv6(net::IPAddress* addr) {
  119. if (!addr->IsIPv6())
  120. return false;
  121. const auto bytes = addr->bytes();
  122. *addr = net::IPAddress(bytes[0], bytes[1], bytes[2], bytes[3], 0, 0, 0, 0, 0,
  123. 0, 0, 0, 0, 0, 0, 0);
  124. return true;
  125. }
  126. // Returns an appropriately scrubbed version of |addr| if applicable.
  127. std::string MaybeScrubIPAddress(const std::string& addr) {
  128. struct {
  129. net::IPAddress ip_addr;
  130. int prefix_length;
  131. bool scrub;
  132. } static const kNonIdentifyingIPRanges[] = {
  133. // Private.
  134. {net::IPAddress(10, 0, 0, 0), 8, true},
  135. {net::IPAddress(172, 16, 0, 0), 12, true},
  136. {net::IPAddress(192, 168, 0, 0), 16, true},
  137. // Chrome OS containers and VMs.
  138. {net::IPAddress(100, 115, 92, 0), 24, false},
  139. // Loopback.
  140. {net::IPAddress(127, 0, 0, 0), 8, true},
  141. // Any.
  142. {net::IPAddress(0, 0, 0, 0), 8, true},
  143. // DNS.
  144. {net::IPAddress(8, 8, 8, 8), 32, false},
  145. {net::IPAddress(8, 8, 4, 4), 32, false},
  146. {net::IPAddress(1, 1, 1, 1), 32, false},
  147. // Multicast.
  148. {net::IPAddress(224, 0, 0, 0), 4, true},
  149. // Link local.
  150. {net::IPAddress(169, 254, 0, 0), 16, true},
  151. {net::IPAddress(0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), 10,
  152. true},
  153. // Broadcast.
  154. {net::IPAddress(255, 255, 255, 255), 32, false},
  155. // IPv6 loopback, unspecified and non-address strings.
  156. {net::IPAddress::IPv6AllZeros(), 112, false},
  157. // IPv6 multicast all nodes and routers.
  158. {net::IPAddress(0xff, 0x01, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1),
  159. 128, false},
  160. {net::IPAddress(0xff, 0x01, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2),
  161. 128, false},
  162. {net::IPAddress(0xff, 0x02, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1),
  163. 128, false},
  164. {net::IPAddress(0xff, 0x02, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2),
  165. 128, false},
  166. // IPv6 other multicast (link and interface local).
  167. {net::IPAddress(0xff, 0x01, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), 16,
  168. true},
  169. {net::IPAddress(0xff, 0x02, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), 16,
  170. true},
  171. };
  172. net::IPAddress input_addr;
  173. if (input_addr.AssignFromIPLiteral(addr) && input_addr.IsValid()) {
  174. bool mapped = MaybeUnmapAddress(&input_addr);
  175. bool translated = !mapped ? MaybeUntranslateAddress(&input_addr) : false;
  176. for (const auto& range : kNonIdentifyingIPRanges) {
  177. if (IPAddressMatchesPrefix(input_addr, range.ip_addr,
  178. range.prefix_length)) {
  179. std::string prefix;
  180. std::string out_addr = addr;
  181. if (mapped) {
  182. prefix = "M ";
  183. out_addr = input_addr.ToString();
  184. } else if (translated) {
  185. prefix = "T ";
  186. out_addr = input_addr.ToString();
  187. }
  188. if (range.scrub) {
  189. out_addr = base::StringPrintf(
  190. "%s/%d", range.ip_addr.ToString().c_str(), range.prefix_length);
  191. }
  192. return base::StrCat({prefix, out_addr});
  193. }
  194. }
  195. // |addr| may have been over-aggressively matched as an IPv6 address when
  196. // it's really just an arbitrary part of a sentence. If the string is the
  197. // same as the coarsely truncated address then keep it because even if
  198. // it happens to be a real address, there is no leak.
  199. if (MaybeTruncateIPv6(&input_addr) && input_addr.ToString() == addr)
  200. return addr;
  201. }
  202. return "";
  203. }
  204. // Some strings can contain pieces that match like IPv4 addresses but aren't.
  205. // This function can be used to determine if this was the case by evaluating
  206. // the skipped piece. It returns true, if the matched address was erroneous
  207. // and should be skipped instead.
  208. bool ShouldSkipIPAddress(const re2::StringPiece& skipped) {
  209. // MomdemManager can dump out firmware revision fields that can also
  210. // confuse the IPv4 matcher e.g. "Revision: 81600.0000.00.29.19.16_DO"
  211. // so ignore the replacement if the skipped piece looks like
  212. // "Revision: .*<ipv4>". Note however that if this field contains
  213. // values delimited by multiple spaces, any matches after the first
  214. // will lose the context and be redacted.
  215. static const re2::StringPiece rev("Revision: ");
  216. static const re2::StringPiece space(" ");
  217. const auto pos = skipped.rfind(rev);
  218. if (pos != re2::StringPiece::npos &&
  219. skipped.find(space, pos + rev.length()) == re2::StringPiece::npos) {
  220. return true;
  221. }
  222. // USB paths can be confused with IPv4 Addresses because they can look
  223. // similar: n-n.n.n.n . Ignore replacement if previous char is `-`
  224. static const re2::StringPiece dash("-");
  225. return skipped.ends_with(dash);
  226. }
  227. // Helper macro: Non capturing group
  228. #define NCG(x) "(?:" x ")"
  229. // Helper macro: Optional non capturing group
  230. #define OPT_NCG(x) NCG(x) "?"
  231. //////////////////////////////////////////////////////////////////////////
  232. // Patterns for URLs, or better IRIs, based on RFC 3987 with an artificial
  233. // limitation on the scheme to increase precision. Otherwise anything
  234. // like "ID:" would be considered an IRI.
  235. #define UNRESERVED "[-a-z0-9._~]"
  236. #define RESERVED NGC(GEN_DELIMS "|" SUB_DELIMS)
  237. #define SUB_DELIMS "[!$&'()*+,;=]"
  238. #define GEN_DELIMS "[:/?#[\\]@]"
  239. #define DIGIT "[0-9]"
  240. #define HEXDIG "[0-9a-f]"
  241. #define PCT_ENCODED "%" HEXDIG HEXDIG
  242. #define DEC_OCTET NCG("1[0-9][0-9]|2[0-4][0-9]|25[0-5]|[1-9][0-9]|[0-9]")
  243. #define IPV4ADDRESS DEC_OCTET "\\." DEC_OCTET "\\." DEC_OCTET "\\." DEC_OCTET
  244. #define H16 NCG(HEXDIG) "{1,4}"
  245. #define LS32 NCG(H16 ":" H16 "|" IPV4ADDRESS)
  246. #define WB "\\b"
  247. // clang-format off
  248. #define IPV6ADDRESS NCG( \
  249. WB NCG(H16 ":") "{6}" LS32 WB "|" \
  250. "::" NCG(H16 ":") "{5}" LS32 WB "|" \
  251. OPT_NCG( WB H16) "::" NCG(H16 ":") "{4}" LS32 WB "|" \
  252. OPT_NCG( WB NCG(H16 ":") "{0,1}" H16) "::" NCG(H16 ":") "{3}" LS32 WB "|" \
  253. OPT_NCG( WB NCG(H16 ":") "{0,2}" H16) "::" NCG(H16 ":") "{2}" LS32 WB "|" \
  254. OPT_NCG( WB NCG(H16 ":") "{0,3}" H16) "::" NCG(H16 ":") LS32 WB "|" \
  255. OPT_NCG( WB NCG(H16 ":") "{0,4}" H16) "::" LS32 WB "|" \
  256. OPT_NCG( WB NCG(H16 ":") "{0,5}" H16) "::" H16 WB "|" \
  257. OPT_NCG( WB NCG(H16 ":") "{0,6}" H16) "::")
  258. // clang-format on
  259. #define IPVFUTURE \
  260. "v" HEXDIG \
  261. "+" \
  262. "\\." NCG(UNRESERVED "|" SUB_DELIMS \
  263. "|" \
  264. ":") "+"
  265. #define IP_LITERAL "\\[" NCG(IPV6ADDRESS "|" IPVFUTURE) "\\]"
  266. #define PORT DIGIT "*"
  267. // This is a diversion of RFC 3987
  268. #define SCHEME NCG("http|https|ftp|chrome|chrome-extension|android|rtsp|file")
  269. #define IPRIVATE \
  270. "[" \
  271. "\\x{E000}-\\x{F8FF}" \
  272. "\\x{F0000}-\\x{FFFFD}" \
  273. "\\x{100000}-\\x{10FFFD}" \
  274. "]"
  275. #define UCSCHAR \
  276. "[" \
  277. "\\x{A0}-\\x{D7FF}" \
  278. "\\x{F900}-\\x{FDCF}" \
  279. "\\x{FDF0}-\\x{FFEF}" \
  280. "\\x{10000}-\\x{1FFFD}" \
  281. "\\x{20000}-\\x{2FFFD}" \
  282. "\\x{30000}-\\x{3FFFD}" \
  283. "\\x{40000}-\\x{4FFFD}" \
  284. "\\x{50000}-\\x{5FFFD}" \
  285. "\\x{60000}-\\x{6FFFD}" \
  286. "\\x{70000}-\\x{7FFFD}" \
  287. "\\x{80000}-\\x{8FFFD}" \
  288. "\\x{90000}-\\x{9FFFD}" \
  289. "\\x{A0000}-\\x{AFFFD}" \
  290. "\\x{B0000}-\\x{BFFFD}" \
  291. "\\x{C0000}-\\x{CFFFD}" \
  292. "\\x{D0000}-\\x{DFFFD}" \
  293. "\\x{E1000}-\\x{EFFFD}" \
  294. "]"
  295. #define IUNRESERVED \
  296. NCG("[-a-z0-9._~]" \
  297. "|" UCSCHAR)
  298. #define IPCHAR \
  299. NCG(IUNRESERVED "|" PCT_ENCODED "|" SUB_DELIMS \
  300. "|" \
  301. "[:@]")
  302. #define IFRAGMENT \
  303. NCG(IPCHAR \
  304. "|" \
  305. "[/?]") \
  306. "*"
  307. #define IQUERY \
  308. NCG(IPCHAR "|" IPRIVATE \
  309. "|" \
  310. "[/?]") \
  311. "*"
  312. #define ISEGMENT IPCHAR "*"
  313. #define ISEGMENT_NZ IPCHAR "+"
  314. #define ISEGMENT_NZ_NC \
  315. NCG(IUNRESERVED "|" PCT_ENCODED "|" SUB_DELIMS \
  316. "|" \
  317. "@") \
  318. "+"
  319. #define IPATH_EMPTY ""
  320. #define IPATH_ROOTLESS ISEGMENT_NZ NCG("/" ISEGMENT) "*"
  321. #define IPATH_NOSCHEME ISEGMENT_NZ_NC NCG("/" ISEGMENT) "*"
  322. #define IPATH_ABSOLUTE "/" OPT_NCG(ISEGMENT_NZ NCG("/" ISEGMENT) "*")
  323. #define IPATH_ABEMPTY NCG("/" ISEGMENT) "*"
  324. #define IPATH \
  325. NCG(IPATH_ABEMPTY "|" IPATH_ABSOLUTE "|" IPATH_NOSCHEME "|" IPATH_ROOTLESS \
  326. "|" IPATH_EMPTY)
  327. #define IREG_NAME NCG(IUNRESERVED "|" PCT_ENCODED "|" SUB_DELIMS) "*"
  328. #define IHOST NCG(IP_LITERAL "|" IPV4ADDRESS "|" IREG_NAME)
  329. #define IUSERINFO \
  330. NCG(IUNRESERVED "|" PCT_ENCODED "|" SUB_DELIMS \
  331. "|" \
  332. ":") \
  333. "*"
  334. #define IAUTHORITY OPT_NCG(IUSERINFO "@") IHOST OPT_NCG(":" PORT)
  335. #define IRELATIVE_PART \
  336. NCG("//" IAUTHORITY IPATH_ABEMPTY "|" IPATH_ABSOLUTE "|" IPATH_NOSCHEME \
  337. "|" IPATH_EMPTY)
  338. #define IRELATIVE_REF IRELATIVE_PART OPT_NCG("?" IQUERY) OPT_NCG("#" IFRAGMENT)
  339. // RFC 3987 requires IPATH_EMPTY here but it is omitted so that statements
  340. // that end with "Android:" for example are not considered a URL.
  341. #define IHIER_PART \
  342. NCG("//" IAUTHORITY IPATH_ABEMPTY "|" IPATH_ABSOLUTE "|" IPATH_ROOTLESS)
  343. #define ABSOLUTE_IRI SCHEME ":" IHIER_PART OPT_NCG("?" IQUERY)
  344. #define IRI SCHEME ":" IHIER_PART OPT_NCG("\\?" IQUERY) OPT_NCG("#" IFRAGMENT)
  345. #define IRI_REFERENCE NCG(IRI "|" IRELATIVE_REF)
  346. // TODO(battre): Use http://tools.ietf.org/html/rfc5322 to represent email
  347. // addresses. Capture names as well ("First Lastname" <foo@bar.com>).
  348. // The |kCustomPatternWithoutContext| array defines further patterns to match
  349. // and redact. Each pattern consists of a single capturing group.
  350. CustomPatternWithAlias kCustomPatternsWithoutContext[] = {
  351. {"URL", "(?i)(" IRI ")", PIIType::kURL},
  352. // Email Addresses need to come after URLs because they can be part
  353. // of a query parameter.
  354. {"email", "(?i)([0-9a-z._%+-]+@[a-z0-9.-]+\\.[a-z]{2,6})", PIIType::kEmail},
  355. // IP filter rules need to come after URLs so that they don't disturb the
  356. // URL pattern in case the IP address is part of a URL.
  357. {"IPv4", "(?i)(" IPV4ADDRESS ")", PIIType::kIPAddress},
  358. {"IPv6", "(?i)(" IPV6ADDRESS ")", PIIType::kIPAddress},
  359. // Universal Unique Identifiers (UUIDs).
  360. {"UUID",
  361. "(?i)([0-9a-zA-Z]{8}-[0-9a-zA-Z]{4}-[0-9a-zA-Z]{4}-[0-9a-zA-Z]{4}-"
  362. "[0-9a-zA-Z]{12})",
  363. PIIType::kStableIdentifier},
  364. // Eche UID which is a base64 conversion of a 32 bytes public key.
  365. {"UID",
  366. "((?:[A-Za-z0-9+/]{4}){10}(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=))",
  367. PIIType::kStableIdentifier},
  368. };
  369. // Like RE2's FindAndConsume, searches for the first occurrence of |pattern| in
  370. // |input| and consumes the bytes until the end of the pattern matching. Unlike
  371. // FindAndConsume, the bytes skipped before the match of |pattern| are stored
  372. // in |skipped_input|. |args| needs to contain at least one element.
  373. // Returns whether a match was found.
  374. //
  375. // Example: input = "aaabbbc", pattern = "(b+)" leads to skipped_input = "aaa",
  376. // args[0] = "bbb", and the beginning input is moved to the right so that it
  377. // only contains "c".
  378. // Example: input = "aaabbbc", pattern = "(z+)" leads to input = "aaabbbc",
  379. // the args values are not modified and skipped_input is not modified.
  380. bool FindAndConsumeAndGetSkippedN(re2::StringPiece* input,
  381. const re2::RE2& pattern,
  382. re2::StringPiece* skipped_input,
  383. re2::StringPiece* args[],
  384. int argc) {
  385. re2::StringPiece old_input = *input;
  386. CHECK_GE(argc, 1);
  387. re2::RE2::Arg a0(argc > 0 ? args[0] : nullptr);
  388. re2::RE2::Arg a1(argc > 1 ? args[1] : nullptr);
  389. re2::RE2::Arg a2(argc > 2 ? args[2] : nullptr);
  390. re2::RE2::Arg a3(argc > 3 ? args[3] : nullptr);
  391. const re2::RE2::Arg* const wrapped_args[] = {&a0, &a1, &a2, &a3};
  392. CHECK_LE(argc, 4);
  393. bool result = re2::RE2::FindAndConsumeN(input, pattern, wrapped_args, argc);
  394. if (skipped_input && result) {
  395. size_t bytes_skipped = args[0]->data() - old_input.data();
  396. *skipped_input = re2::StringPiece(old_input.data(), bytes_skipped);
  397. }
  398. return result;
  399. }
  400. // All |match_groups| need to be of type re2::StringPiece*.
  401. template <typename... Arg>
  402. bool FindAndConsumeAndGetSkipped(re2::StringPiece* input,
  403. const re2::RE2& pattern,
  404. re2::StringPiece* skipped_input,
  405. Arg*... match_groups) {
  406. re2::StringPiece* args[] = {match_groups...};
  407. return FindAndConsumeAndGetSkippedN(input, pattern, skipped_input, args,
  408. std::size(args));
  409. }
  410. // The following MAC addresses will not be redacted as they are not specific
  411. // to a device but have general meanings.
  412. const char* const kUnredactedMacAddresses[] = {
  413. "00:00:00:00:00:00", // ARP failure result MAC.
  414. "ff:ff:ff:ff:ff:ff", // Broadcast MAC.
  415. };
  416. constexpr size_t kNumUnredactedMacs = std::size(kUnredactedMacAddresses);
  417. } // namespace
  418. RedactionTool::RedactionTool(const char* const* first_party_extension_ids)
  419. : first_party_extension_ids_(first_party_extension_ids) {
  420. DETACH_FROM_SEQUENCE(sequence_checker_);
  421. // Identity-map these, so we don't mangle them.
  422. for (const char* mac : kUnredactedMacAddresses)
  423. mac_addresses_[mac] = mac;
  424. }
  425. RedactionTool::~RedactionTool() {
  426. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  427. }
  428. std::map<PIIType, std::set<std::string>> RedactionTool::Detect(
  429. const std::string& input) {
  430. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  431. base::AssertLongCPUWorkAllowed();
  432. std::map<PIIType, std::set<std::string>> detected;
  433. RedactMACAddresses(input, &detected);
  434. // This function will add to |detected| only on Chrome OS as Android app
  435. // storage paths are only detected for Chrome OS.
  436. RedactAndroidAppStoragePaths(input, &detected);
  437. DetectWithCustomPatterns(input, &detected);
  438. // Do hashes last since they may appear in URLs and they also prevent us from
  439. // properly recognizing the Android storage paths.
  440. RedactHashes(input, &detected);
  441. return detected;
  442. }
  443. std::string RedactionTool::Redact(const std::string& input) {
  444. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  445. return RedactAndKeepSelected(input, /*pii_types_to_keep=*/{});
  446. }
  447. std::string RedactionTool::RedactAndKeepSelected(
  448. const std::string& input,
  449. const std::set<PIIType>& pii_types_to_keep) {
  450. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  451. base::AssertLongCPUWorkAllowed();
  452. // Copy |input| so we can modify it.
  453. std::string redacted = input;
  454. if (pii_types_to_keep.find(PIIType::kMACAddress) == pii_types_to_keep.end()) {
  455. redacted = RedactMACAddresses(std::move(redacted), nullptr);
  456. }
  457. if (pii_types_to_keep.find(PIIType::kAndroidAppStoragePath) ==
  458. pii_types_to_keep.end()) {
  459. redacted = RedactAndroidAppStoragePaths(std::move(redacted), nullptr);
  460. }
  461. redacted = RedactAndKeepSelectedCustomPatterns(std::move(redacted),
  462. pii_types_to_keep);
  463. // Do hashes last since they may appear in URLs and they also prevent us
  464. // from properly recognizing the Android storage paths.
  465. if (pii_types_to_keep.find(PIIType::kStableIdentifier) ==
  466. pii_types_to_keep.end()) {
  467. // URLs and Android storage paths will be partially redacted (only hashes)
  468. // if |pii_types_to_keep| contains PIIType::kURL or
  469. // PIIType::kAndroidAppStoragePath and not PIIType::kStableIdentifier.
  470. redacted = RedactHashes(std::move(redacted), nullptr);
  471. }
  472. return redacted;
  473. }
  474. RE2* RedactionTool::GetRegExp(const std::string& pattern) {
  475. if (regexp_cache_.find(pattern) == regexp_cache_.end()) {
  476. RE2::Options options;
  477. // set_multiline of pcre is not supported by RE2, yet.
  478. options.set_dot_nl(true); // Dot matches a new line.
  479. std::unique_ptr<RE2> re = std::make_unique<RE2>(pattern, options);
  480. DCHECK_EQ(re2::RE2::NoError, re->error_code()) << "Failed to parse:\n"
  481. << pattern << "\n"
  482. << re->error();
  483. regexp_cache_[pattern] = std::move(re);
  484. }
  485. return regexp_cache_[pattern].get();
  486. }
  487. std::string RedactionTool::RedactMACAddresses(
  488. const std::string& input,
  489. std::map<PIIType, std::set<std::string>>* detected) {
  490. // This regular expression finds the next MAC address. It splits the data into
  491. // an OUI (Organizationally Unique Identifier) part and a NIC (Network
  492. // Interface Controller) specific part. We also match on dash and underscore
  493. // because we have seen instances of both of those occurring.
  494. RE2* mac_re = GetRegExp(
  495. "([0-9a-fA-F][0-9a-fA-F][:\\-_]"
  496. "[0-9a-fA-F][0-9a-fA-F][:\\-_]"
  497. "[0-9a-fA-F][0-9a-fA-F])[:\\-_]("
  498. "[0-9a-fA-F][0-9a-fA-F][:\\-_]"
  499. "[0-9a-fA-F][0-9a-fA-F][:\\-_]"
  500. "[0-9a-fA-F][0-9a-fA-F])");
  501. std::string result;
  502. result.reserve(input.size());
  503. // Keep consuming, building up a result string as we go.
  504. re2::StringPiece text(input);
  505. re2::StringPiece skipped, oui, nic;
  506. static const char kMacSeparatorChars[] = "-_";
  507. while (FindAndConsumeAndGetSkipped(&text, *mac_re, &skipped, &oui, &nic)) {
  508. // Look up the MAC address in the hash. Force the separator to be a colon
  509. // so that the same MAC with a different format will match in all cases.
  510. std::string oui_string = base::ToLowerASCII(std::string(oui));
  511. base::ReplaceChars(oui_string, kMacSeparatorChars, ":", &oui_string);
  512. std::string nic_string = base::ToLowerASCII(std::string(nic));
  513. base::ReplaceChars(nic_string, kMacSeparatorChars, ":", &nic_string);
  514. std::string mac = oui_string + ":" + nic_string;
  515. std::string replacement_mac = mac_addresses_[mac];
  516. if (replacement_mac.empty()) {
  517. // If not found, build up a replacement MAC address by generating a new
  518. // NIC part.
  519. int mac_id = mac_addresses_.size() - kNumUnredactedMacs;
  520. replacement_mac = base::StringPrintf("[MAC OUI=%s IFACE=%d]",
  521. oui_string.c_str(), mac_id);
  522. mac_addresses_[mac] = replacement_mac;
  523. }
  524. if (detected != nullptr) {
  525. (*detected)[PIIType::kMACAddress].insert(mac);
  526. }
  527. skipped.AppendToString(&result);
  528. result += replacement_mac;
  529. }
  530. text.AppendToString(&result);
  531. return result;
  532. }
  533. std::string RedactionTool::RedactHashes(
  534. const std::string& input,
  535. std::map<PIIType, std::set<std::string>>* detected) {
  536. // This will match hexadecimal strings from length 32 to 64 that have a word
  537. // boundary at each end. We then check to make sure they are one of our valid
  538. // hash lengths before replacing.
  539. // NOTE: There are some occurrences in the dump data (specifically modetest)
  540. // where relevant data is formatted with 32 hex chars on a line. In this case,
  541. // it is preceded by at least 3 whitespace chars, so check for that and in
  542. // that case do not redact.
  543. RE2* hash_re = GetRegExp(R"((\s*)\b([0-9a-fA-F]{4})([0-9a-fA-F]{28,60})\b)");
  544. std::string result;
  545. result.reserve(input.size());
  546. // Keep consuming, building up a result string as we go.
  547. re2::StringPiece text(input);
  548. re2::StringPiece skipped, pre_whitespace, hash_prefix, hash_suffix;
  549. while (FindAndConsumeAndGetSkipped(&text, *hash_re, &skipped, &pre_whitespace,
  550. &hash_prefix, &hash_suffix)) {
  551. skipped.AppendToString(&result);
  552. pre_whitespace.AppendToString(&result);
  553. // Check if it's a valid length for our hashes or if we need to skip due to
  554. // the whitespace check.
  555. size_t hash_length = 4 + hash_suffix.length();
  556. if ((hash_length != 32 && hash_length != 40 && hash_length != 64) ||
  557. (hash_length == 32 && pre_whitespace.length() >= 3)) {
  558. // This is not a hash string, skip it.
  559. hash_prefix.AppendToString(&result);
  560. hash_suffix.AppendToString(&result);
  561. continue;
  562. }
  563. // Look up the hash value address in the map of replacements.
  564. std::string hash_prefix_string =
  565. base::ToLowerASCII(std::string(hash_prefix));
  566. std::string hash =
  567. hash_prefix_string + base::ToLowerASCII(std::string(hash_suffix));
  568. std::string replacement_hash = hashes_[hash];
  569. if (replacement_hash.empty()) {
  570. // If not found, build up a replacement value.
  571. replacement_hash = base::StringPrintf(
  572. "<HASH:%s %zd>", hash_prefix_string.c_str(), hashes_.size());
  573. hashes_[hash] = replacement_hash;
  574. }
  575. if (detected != nullptr) {
  576. (*detected)[PIIType::kStableIdentifier].insert(hash);
  577. }
  578. result += replacement_hash;
  579. }
  580. text.AppendToString(&result);
  581. return result;
  582. }
  583. std::string RedactionTool::RedactAndroidAppStoragePaths(
  584. const std::string& input,
  585. std::map<PIIType, std::set<std::string>>* detected) {
  586. // We only use this on Chrome OS and there's differences in the API for
  587. // FilePath on Windows which prevents this from compiling, so only enable this
  588. // code for Chrome OS.
  589. #if BUILDFLAG(IS_CHROMEOS_ASH)
  590. std::string result;
  591. result.reserve(input.size());
  592. // This is for redacting Android data paths included in 'android_app_storage'
  593. // and 'audit_log' output. <app_specific_path> in the following data paths
  594. // will be redacted.
  595. // - /data/data/<package_name>/<app_specific_path>
  596. // - /data/app/<package_name>/<app_specific_path>
  597. // - /data/user_de/<number>/<package_name>/<app_specific_path>
  598. // These data paths are preceded by "/home/root/<user_hash>/android-data" in
  599. // 'android_app_storage' output, and preceded by "path=" or "exe=" in
  600. // 'audit_log' output.
  601. RE2* path_re = GetRegExp(
  602. R"((?m)((path=|exe=|/home/root/[\da-f]+/android-data))"
  603. R"(/data/(data|app|user_de/\d+)/[^/\n]+)(/[^\n\s]+))");
  604. // Keep consuming, building up a result string as we go.
  605. re2::StringPiece text(input);
  606. re2::StringPiece skipped;
  607. re2::StringPiece path_prefix; // path before app_specific;
  608. re2::StringPiece pre_data; // (path=|exe=|/home/root/<hash>/android-data)
  609. re2::StringPiece post_data; // (data|app|user_de/\d+)
  610. re2::StringPiece app_specific; // (/[^\n\s]+)
  611. while (FindAndConsumeAndGetSkipped(&text, *path_re, &skipped, &path_prefix,
  612. &pre_data, &post_data, &app_specific)) {
  613. // We can record these parts as-is.
  614. skipped.AppendToString(&result);
  615. path_prefix.AppendToString(&result);
  616. // |app_specific| has to be redacted. First, convert it into components,
  617. // and then redact each component as follows:
  618. // - If the component has a non-ASCII character, change it to '*'.
  619. // - Otherwise, remove all the characters in the component but the first
  620. // one.
  621. // - If the original component has 2 or more bytes, add '_'.
  622. const base::FilePath path(app_specific.as_string());
  623. std::vector<std::string> components = path.GetComponents();
  624. DCHECK(!components.empty());
  625. auto it = components.begin() + 1; // ignore the leading slash
  626. for (; it != components.end(); ++it) {
  627. const auto& component = *it;
  628. DCHECK(!component.empty());
  629. result += '/';
  630. result += (base::IsStringASCII(component) ? component[0] : '*');
  631. if (component.length() > 1)
  632. result += '_';
  633. }
  634. if (detected != nullptr) {
  635. (*detected)[PIIType::kAndroidAppStoragePath].insert(
  636. app_specific.as_string());
  637. }
  638. }
  639. text.AppendToString(&result);
  640. return result;
  641. #else
  642. return input;
  643. #endif // BUILDFLAG(IS_CHROMEOS_ASH)
  644. }
  645. std::string RedactionTool::RedactAndKeepSelectedCustomPatterns(
  646. std::string input,
  647. const std::set<PIIType>& pii_types_to_keep) {
  648. for (const auto& pattern : kCustomPatternsWithContext) {
  649. if (pii_types_to_keep.find(pattern.pii_type) == pii_types_to_keep.end()) {
  650. input = RedactCustomPatternWithContext(input, pattern, nullptr);
  651. }
  652. }
  653. for (const auto& pattern : kCustomPatternsWithoutContext) {
  654. if (pii_types_to_keep.find(pattern.pii_type) == pii_types_to_keep.end()) {
  655. input = RedactCustomPatternWithoutContext(input, pattern, nullptr);
  656. }
  657. }
  658. return input;
  659. }
  660. void RedactionTool::DetectWithCustomPatterns(
  661. std::string input,
  662. std::map<PIIType, std::set<std::string>>* detected) {
  663. for (const auto& pattern : kCustomPatternsWithContext) {
  664. RedactCustomPatternWithContext(input, pattern, detected);
  665. }
  666. for (const auto& pattern : kCustomPatternsWithoutContext) {
  667. RedactCustomPatternWithoutContext(input, pattern, detected);
  668. }
  669. }
  670. std::string RedactionTool::RedactCustomPatternWithContext(
  671. const std::string& input,
  672. const CustomPatternWithAlias& pattern,
  673. std::map<PIIType, std::set<std::string>>* detected) {
  674. RE2* re = GetRegExp(pattern.pattern);
  675. DCHECK_EQ(3, re->NumberOfCapturingGroups());
  676. std::map<std::string, std::string>* identifier_space =
  677. &custom_patterns_with_context_[pattern.alias];
  678. std::string result;
  679. result.reserve(input.size());
  680. // Keep consuming, building up a result string as we go.
  681. re2::StringPiece text(input);
  682. re2::StringPiece skipped;
  683. re2::StringPiece pre_match, pre_matched_id, matched_id, post_matched_id;
  684. while (FindAndConsumeAndGetSkipped(&text, *re, &skipped, &pre_matched_id,
  685. &matched_id, &post_matched_id)) {
  686. std::string matched_id_as_string(matched_id);
  687. std::string replacement_id;
  688. if (identifier_space->count(matched_id_as_string) == 0) {
  689. // The weird NumberToString trick is because Windows does not like
  690. // to deal with %zu and a size_t in printf, nor does it support %llu.
  691. replacement_id = base::StringPrintf(
  692. "<%s: %s>", pattern.alias,
  693. base::NumberToString(identifier_space->size() + 1).c_str());
  694. (*identifier_space)[matched_id_as_string] = replacement_id;
  695. } else {
  696. replacement_id = (*identifier_space)[matched_id_as_string];
  697. }
  698. if (detected != nullptr) {
  699. (*detected)[pattern.pii_type].insert(matched_id_as_string);
  700. }
  701. skipped.AppendToString(&result);
  702. pre_matched_id.AppendToString(&result);
  703. result += replacement_id;
  704. post_matched_id.AppendToString(&result);
  705. }
  706. text.AppendToString(&result);
  707. return result;
  708. }
  709. // This takes a |url| argument and returns true if the URL is exempt from
  710. // redaction, returns false otherwise.
  711. bool IsUrlExempt(re2::StringPiece url,
  712. const char* const* first_party_extension_ids) {
  713. // We do not exempt anything with a query parameter.
  714. if (url.contains("?"))
  715. return false;
  716. // Last part of an SELinux context is misdetected as a URL.
  717. // e.g. "u:object_r:system_data_file:s0:c512,c768"
  718. if (url.starts_with("file:s0"))
  719. return true;
  720. // Check for chrome:// URLs that are exempt.
  721. if (url.starts_with("chrome://")) {
  722. // We allow everything in chrome://resources/.
  723. if (url.starts_with("chrome://resources/"))
  724. return true;
  725. // We allow chrome://*/crisper.js.
  726. if (url.ends_with("/crisper.js"))
  727. return true;
  728. return false;
  729. }
  730. if (!first_party_extension_ids)
  731. return false;
  732. // Exempt URLs of the format chrome-extension://<first-party-id>/*.js
  733. if (!url.starts_with("chrome-extension://"))
  734. return false;
  735. // These must end with a .js extension.
  736. if (!url.ends_with(".js"))
  737. return false;
  738. int i = 0;
  739. const char* test_id = first_party_extension_ids[i];
  740. const re2::StringPiece url_sub =
  741. url.substr(sizeof("chrome-extension://") - 1);
  742. while (test_id) {
  743. if (url_sub.starts_with(test_id))
  744. return true;
  745. test_id = first_party_extension_ids[++i];
  746. }
  747. return false;
  748. }
  749. std::string RedactionTool::RedactCustomPatternWithoutContext(
  750. const std::string& input,
  751. const CustomPatternWithAlias& pattern,
  752. std::map<PIIType, std::set<std::string>>* detected) {
  753. RE2* re = GetRegExp(pattern.pattern);
  754. DCHECK_EQ(1, re->NumberOfCapturingGroups());
  755. std::map<std::string, std::string>* identifier_space =
  756. &custom_patterns_without_context_[pattern.alias];
  757. std::string result;
  758. result.reserve(input.size());
  759. // Keep consuming, building up a result string as we go.
  760. re2::StringPiece text(input);
  761. re2::StringPiece skipped;
  762. re2::StringPiece matched_id;
  763. while (FindAndConsumeAndGetSkipped(&text, *re, &skipped, &matched_id)) {
  764. if (IsUrlExempt(matched_id, first_party_extension_ids_)) {
  765. skipped.AppendToString(&result);
  766. matched_id.AppendToString(&result);
  767. continue;
  768. }
  769. std::string matched_id_as_string(matched_id);
  770. std::string replacement_id;
  771. if (identifier_space->count(matched_id_as_string) == 0) {
  772. replacement_id = MaybeScrubIPAddress(matched_id_as_string);
  773. if (replacement_id != matched_id_as_string) {
  774. // Double-check overly opportunistic IPv4 address matching.
  775. if ((strcmp("IPv4", pattern.alias) == 0) &&
  776. ShouldSkipIPAddress(skipped)) {
  777. skipped.AppendToString(&result);
  778. matched_id.AppendToString(&result);
  779. continue;
  780. }
  781. // The weird NumberToString trick is because Windows does not like
  782. // to deal with %zu and a size_t in printf, nor does it support %llu.
  783. replacement_id = base::StringPrintf(
  784. "<%s: %s>",
  785. replacement_id.empty() ? pattern.alias : replacement_id.c_str(),
  786. base::NumberToString(identifier_space->size() + 1).c_str());
  787. (*identifier_space)[matched_id_as_string] = replacement_id;
  788. if (detected != nullptr) {
  789. (*detected)[pattern.pii_type].insert(matched_id_as_string);
  790. }
  791. }
  792. } else {
  793. replacement_id = (*identifier_space)[matched_id_as_string];
  794. if (detected != nullptr) {
  795. (*detected)[pattern.pii_type].insert(matched_id_as_string);
  796. }
  797. }
  798. skipped.AppendToString(&result);
  799. result += replacement_id;
  800. }
  801. text.AppendToString(&result);
  802. return result;
  803. }
  804. RedactionToolContainer::RedactionToolContainer(
  805. scoped_refptr<base::SequencedTaskRunner> task_runner,
  806. const char* const* first_party_extension_ids)
  807. : redactor_(new RedactionTool(first_party_extension_ids)),
  808. task_runner_(task_runner) {}
  809. RedactionToolContainer::~RedactionToolContainer() {
  810. task_runner_->DeleteSoon(FROM_HERE, std::move(redactor_));
  811. }
  812. RedactionTool* RedactionToolContainer::Get() {
  813. DCHECK(task_runner_->RunsTasksInCurrentSequence());
  814. return redactor_.get();
  815. }
  816. } // namespace feedback