fetch.py 6.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240
  1. #!/usr/bin/env python
  2. # -*- coding: utf-8 -*-
  3. # Copyright 2017 The Chromium Authors. All rights reserved.
  4. # Use of this source code is governed by a BSD-style license that can be
  5. # found in the LICENSE file.
  6. """Fetches articles from the server.
  7. Examples:
  8. $ fetch.py # unauthenticated, no experiments
  9. $ fetch.py --short # abbreviate instead of dumping JSON
  10. $ fetch.py --signed-in -x3313279 # authenticated, results from Google Now
  11. If getting signed-in results, authenticates with OAuth2 and stores the
  12. credentials at ~/.zineauth.
  13. """
  14. from __future__ import absolute_import, division, print_function, unicode_literals
  15. import argparse
  16. import base64
  17. import datetime
  18. import json
  19. import os
  20. import textwrap
  21. import oauth2client.client
  22. import oauth2client.file
  23. import oauth2client.tools
  24. import requests
  25. import sys
  26. API_KEY_FILE = os.path.join(
  27. os.path.dirname(__file__),
  28. "../../../google_apis/internal/google_chrome_api_keys.h")
  29. API_SCOPE = "https://www.googleapis.com/auth/chrome-content-suggestions"
  30. API_HOSTS = {
  31. "prod": "https://chromecontentsuggestions-pa.googleapis.com",
  32. "staging": "https://staging-chromecontentsuggestions-pa.googleapis.com",
  33. "alpha": "https://alpha-chromecontentsuggestions-pa.sandbox.googleapis.com",
  34. }
  35. API_PATH = "/v1/suggestions/fetch"
  36. def main():
  37. default_lang = os.environ.get("LANG", "en_US").split(".")[0]
  38. parser = argparse.ArgumentParser(
  39. description="fetch articles from server",
  40. parents=[oauth2client.tools.argparser])
  41. parser.add_argument("-c", "--component",
  42. default="prod", choices=["prod", "staging", "alpha"],
  43. help="component to fetch from (default: prod)")
  44. parser.add_argument("-x", "--experiment", action="append", type=int,
  45. help="include an experiment ID")
  46. parser.add_argument("-l", "--ui-language", default=default_lang,
  47. help="language code (default: %s)" % default_lang)
  48. parser.add_argument("--ip", help="fake IP address")
  49. parser.add_argument("--api-key", type=str,
  50. help="API key to use for unauthenticated requests"
  51. " (default: use official key)")
  52. parser.add_argument("-s", "--signed-in", action="store_true",
  53. help="sign in and issue authenticated request")
  54. parser.add_argument("--client", metavar="ID,SECRET", type=str,
  55. help="client project to use for authenticated requests"
  56. " (default: use official project ID")
  57. parser.add_argument("--short", action="store_true",
  58. help="print results in abbreviated form")
  59. args = parser.parse_args()
  60. r = PostRequest(args)
  61. j = {}
  62. try:
  63. j = r.json()
  64. except ValueError:
  65. print(r.text.encode("utf-8"))
  66. sys.exit(1)
  67. if j.get("error"):
  68. print(r.text.encode("utf-8"))
  69. sys.exit(1)
  70. if args.short:
  71. PrintShortResponse(j)
  72. return
  73. print(r.text.encode("utf-8"))
  74. if r.status_code != 200:
  75. sys.exit(1)
  76. def GetApiKeyFile():
  77. return API_KEY_FILE
  78. def GetAPIDefs():
  79. """Parses the internal file with API keys and returns a dict."""
  80. with open(GetApiKeyFile()) as f:
  81. lines = f.readlines()
  82. defs = {}
  83. next_name = None
  84. for line in lines:
  85. if next_name:
  86. defs[next_name] = json.loads(line)
  87. next_name = None
  88. elif line.startswith("#define"):
  89. try:
  90. _, name, value = line.split()
  91. except ValueError:
  92. continue
  93. if value == "\\":
  94. next_name = name
  95. else:
  96. defs[name] = json.loads(value)
  97. return defs
  98. def GetAPIKey():
  99. return GetAPIDefs()["GOOGLE_API_KEY"]
  100. def GetOAuthClient():
  101. defs = GetAPIDefs()
  102. return defs["GOOGLE_CLIENT_ID_MAIN"], defs["GOOGLE_CLIENT_SECRET_MAIN"]
  103. def EncodeExperiments(experiments):
  104. """Turn a list of experiment IDs into an X-Client-Data header value.
  105. Encodes all the IDs as a protobuf (tag 1, varint) and base64 encodes the
  106. result.
  107. """
  108. binary = b""
  109. for exp in experiments:
  110. binary += b"\x08"
  111. while True:
  112. byte = (exp & 0x7f)
  113. exp >>= 7
  114. if exp:
  115. binary += chr(0x80 | byte)
  116. else:
  117. binary += chr(byte)
  118. break
  119. return base64.b64encode(binary)
  120. def AbbreviateDuration(duration):
  121. """Turn a datetime.timedelta into a short string like "10h 14m"."""
  122. w = duration.days // 7
  123. d = duration.days % 7
  124. h = duration.seconds // 3600
  125. m = (duration.seconds % 3600) // 60
  126. s = duration.seconds % 60
  127. us = duration.microseconds
  128. if w:
  129. return "%dw %dd" % (w, d)
  130. elif d:
  131. return "%dd %dh" % (d, h)
  132. elif h:
  133. return "%dh %dm" % (h, m)
  134. elif m:
  135. return "%dm %ds" % (m, s)
  136. elif s:
  137. return "%ds" % s
  138. elif us:
  139. return "<1s"
  140. else:
  141. return "0s"
  142. def PostRequest(args):
  143. url = API_HOSTS[args.component] + API_PATH
  144. headers = {}
  145. if args.experiment:
  146. headers["X-Client-Data"] = EncodeExperiments(args.experiment)
  147. if args.ip is not None:
  148. headers["X-User-IP"] = args.ip
  149. if args.signed_in:
  150. if args.client:
  151. client_id, client_secret = args.client.split(",")
  152. else:
  153. client_id, client_secret = GetOAuthClient()
  154. Authenticate(args, headers, client_id, client_secret)
  155. else:
  156. if args.api_key:
  157. api_key = args.api_key
  158. else:
  159. api_key = GetAPIKey()
  160. url += "?key=" + api_key
  161. data = {
  162. "uiLanguage": args.ui_language,
  163. }
  164. return requests.post(url, headers=headers, data=data)
  165. def Authenticate(args, headers, client_id, client_secret):
  166. storage = oauth2client.file.Storage(os.path.expanduser("~/.zineauth"))
  167. creds = storage.get()
  168. if not creds or creds.invalid or creds.access_token_expired:
  169. flow = oauth2client.client.OAuth2WebServerFlow(
  170. client_id=client_id, client_secret=client_secret,
  171. scope=API_SCOPE)
  172. oauth2client.tools.run_flow(flow, storage, args)
  173. creds = storage.get()
  174. creds.apply(headers)
  175. def PrintShortResponse(j):
  176. now = datetime.datetime.now()
  177. for category in j["categories"]:
  178. print("%s: " % category["localizedTitle"])
  179. for suggestion in category.get("suggestions", []):
  180. attribution = suggestion["attribution"]
  181. title = suggestion["title"]
  182. full_url = suggestion["fullPageUrl"]
  183. amp_url = suggestion.get("ampUrl")
  184. creation_time = suggestion["creationTime"]
  185. if len(title) > 40:
  186. title = textwrap.wrap(title, 40)[0] + "…"
  187. creation_time = ParseDateTime(creation_time)
  188. age = AbbreviateDuration(now - creation_time)
  189. print(" “%s” (%s, %s ago)" % (title, attribution, age))
  190. print(" " + (amp_url or full_url))
  191. if category["allowFetchingMoreResults"]:
  192. print(" [More]")
  193. def ParseDateTime(creation_time):
  194. try:
  195. return datetime.datetime.strptime(creation_time, "%Y-%m-%dT%H:%M:%SZ")
  196. except ValueError:
  197. return datetime.datetime.strptime(creation_time, "%Y-%m-%dT%H:%M:%S.%fZ")
  198. if __name__ == "__main__":
  199. main()