my_request.ts 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452
  1. 'use strict';
  2. import cheerio = require('cheerio');
  3. import request = require('request');
  4. import rp = require('request-promise');
  5. import Promise = require('bluebird');
  6. import uuid = require('uuid');
  7. import path = require('path');
  8. import fs = require('fs-extra');
  9. import languages = require('./languages');
  10. import log = require('./log');
  11. import { RequestPromise } from 'request-promise';
  12. import { Response } from 'request';
  13. // tslint:disable-next-line:no-var-requires
  14. const cookieStore = require('tough-cookie-file-store');
  15. const CR_COOKIE_DOMAIN = 'http://crunchyroll.com';
  16. let isAuthenticated = false;
  17. let isPremium = false;
  18. let j: request.CookieJar;
  19. const defaultHeaders =
  20. {
  21. 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
  22. 'Connection': 'keep-alive',
  23. 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',
  24. 'Referer': 'https://www.crunchyroll.com/login',
  25. 'Cache-Control': 'private',
  26. 'Accept-Language': 'en-US,en;q=0.9'
  27. };
  28. const defaultOptions =
  29. {
  30. followAllRedirects: true,
  31. decodeEmails: true,
  32. challengesToSolve: 3,
  33. gzip: true,
  34. };
  35. // tslint:disable-next-line:no-var-requires
  36. const cloudscraper = require('cloudscraper').defaults(defaultOptions);
  37. function AuthError(msg: string): IAuthError
  38. {
  39. return { name: 'AuthError', message: msg, authError: true };
  40. }
  41. function startSession(config: IConfig): Promise<any>
  42. {
  43. return rp(
  44. {
  45. method: 'GET',
  46. url: config.crSessionUrl,
  47. qs:
  48. {
  49. device_id: config.crDeviceId,
  50. device_type: config.crDeviceType,
  51. access_token: config.crSessionKey,
  52. version: config.crAPIVersion,
  53. locale: config.crLocale,
  54. },
  55. json: true,
  56. })
  57. .then((response: any) =>
  58. {
  59. if ((response.data === undefined) || (response.data.session_id === undefined))
  60. {
  61. throw new Error('Getting session failed: ' + JSON.stringify(response));
  62. }
  63. return response.data.session_id;
  64. });
  65. }
  66. function APIlogin(config: IConfig, sessionId: string, user: string, pass: string): Promise<any>
  67. {
  68. return rp(
  69. {
  70. method: 'POST',
  71. url: config.crLoginUrl,
  72. form:
  73. {
  74. account: user,
  75. password: pass,
  76. session_id: sessionId,
  77. version: config.crAPIVersion,
  78. },
  79. json: true,
  80. jar: j,
  81. })
  82. .then((response) =>
  83. {
  84. if (response.error) throw new Error('Login failed: ' + response.message);
  85. return response.data;
  86. });
  87. }
  88. function checkIfUserIsAuth(config: IConfig, done: (err: Error) => void): void
  89. {
  90. if (j === undefined)
  91. {
  92. loadCookies(config);
  93. }
  94. /**
  95. * The main page give us some information about the user
  96. */
  97. const url = 'http://www.crunchyroll.com/';
  98. cloudscraper.get({gzip: true, uri: url}, (err: Error, rep: string, body: string) =>
  99. {
  100. if (err)
  101. {
  102. return done(err);
  103. }
  104. const $ = cheerio.load(body);
  105. /* As we are here, try to detect which locale CR tell us */
  106. const localeRE = /LOCALE = "([a-zA-Z]+)",/g;
  107. const locale = localeRE.exec($('script').text())[1];
  108. const countryCode = languages.localeToCC(locale);
  109. if (config.crlang === undefined)
  110. {
  111. log.info('No locale set. Setting to the one reported by CR: "' + countryCode + '"');
  112. config.crlang = countryCode;
  113. }
  114. else if (config.crlang !== countryCode)
  115. {
  116. log.warn('Crunchy is configured for locale "' + config.crlang + '" but CR report "' + countryCode + '" (LOCALE = ' + locale + ')');
  117. log.warn('Check if it is correct or rerun (once) with "-l ' + countryCode + '" to correct.');
  118. }
  119. /* Check if auth worked */
  120. const regexps = /ga\('set', 'dimension[5-8]', '([^']*)'\);/g;
  121. const dims = regexps.exec($('script').text());
  122. for (let i = 1; i < 5; i++)
  123. {
  124. if ((dims[i] !== undefined) && (dims[i] !== '') && (dims[i] !== 'not-registered'))
  125. {
  126. isAuthenticated = true;
  127. }
  128. if ((dims[i] === 'premium') || (dims[i] === 'premiumplus'))
  129. {
  130. isPremium = true;
  131. }
  132. }
  133. if (isAuthenticated === false)
  134. {
  135. const error = $('ul.message, li.error').text();
  136. log.warn('Authentication failed: ' + error);
  137. log.dumpToDebug('not auth rep', rep);
  138. log.dumpToDebug('not auth body', body);
  139. return done(AuthError('Authentication failed: ' + error));
  140. }
  141. else
  142. {
  143. if (isPremium === false)
  144. {
  145. log.warn('Do not use this app without a premium account.');
  146. }
  147. else
  148. {
  149. log.info('You have a premium account! Good!');
  150. }
  151. }
  152. done(null);
  153. });
  154. }
  155. function loadCookies(config: IConfig)
  156. {
  157. const cookiePath = path.join(config.output || process.cwd(), '.cookies.json');
  158. if (!fs.existsSync(cookiePath))
  159. {
  160. fs.closeSync(fs.openSync(cookiePath, 'w'));
  161. }
  162. j = request.jar(new cookieStore(cookiePath));
  163. }
  164. export function eatCookies(config: IConfig)
  165. {
  166. const cookiePath = path.join(config.output || process.cwd(), '.cookies.json');
  167. if (fs.existsSync(cookiePath))
  168. {
  169. fs.removeSync(cookiePath);
  170. }
  171. j = undefined;
  172. }
  173. export function getUserAgent(): string
  174. {
  175. return defaultHeaders['User-Agent'];
  176. }
  177. /**
  178. * Performs a GET request for the resource.
  179. */
  180. export function get(config: IConfig, options: string|any, done: (err: any, result?: string) => void)
  181. {
  182. if (j === undefined)
  183. {
  184. loadCookies(config);
  185. }
  186. if (config.userAgent)
  187. {
  188. defaultHeaders['User-Agent'] = config.userAgent;
  189. }
  190. authenticate(config, (err) =>
  191. {
  192. if (err)
  193. {
  194. return done(err);
  195. }
  196. cloudscraper.get(modify(options), (error: any, response: any, body: any) =>
  197. {
  198. if (error) return done(error);
  199. done(null, typeof body === 'string' ? body : String(body));
  200. });
  201. });
  202. }
  203. /**
  204. * Performs a POST request for the resource.
  205. */
  206. export function post(config: IConfig, options: request.Options, done: (err: Error, result?: string) => void)
  207. {
  208. if (j === undefined)
  209. {
  210. loadCookies(config);
  211. }
  212. if (config.userAgent)
  213. {
  214. defaultHeaders['User-Agent'] = config.userAgent;
  215. }
  216. authenticate(config, (err) =>
  217. {
  218. if (err)
  219. {
  220. return done(err);
  221. }
  222. cloudscraper.post(modify(options), (error: Error, response: any, body: any) =>
  223. {
  224. if (error)
  225. {
  226. return done(error);
  227. }
  228. done(null, typeof body === 'string' ? body : String(body));
  229. });
  230. });
  231. }
  232. /**
  233. * Authenticates using the configured pass and user.
  234. */
  235. function authenticate(config: IConfig, done: (err: Error) => void)
  236. {
  237. if (isAuthenticated)
  238. {
  239. return done(null);
  240. }
  241. /* First of all, check if the user is not already logged via the cookies */
  242. checkIfUserIsAuth(config, (errCheckAuth) =>
  243. {
  244. if (isAuthenticated)
  245. {
  246. return done(null);
  247. }
  248. /* So if we are here now, that mean we are not authenticated so do as usual */
  249. if ((!config.logUsingApi && !config.logUsingCookie) && (!config.pass || !config.user))
  250. {
  251. log.error('You need to give login/password to use Crunchy');
  252. process.exit(-1);
  253. }
  254. log.info('Seems we are not currently logged. Let\'s login!');
  255. if (config.logUsingApi)
  256. {
  257. if (config.crDeviceId === undefined)
  258. {
  259. config.crDeviceId = uuid.v4();
  260. }
  261. if (!config.crSessionUrl || !config.crDeviceType || !config.crAPIVersion ||
  262. !config.crLocale || !config.crLoginUrl)
  263. {
  264. return done(AuthError('Invalid API configuration, please check your config file.'));
  265. }
  266. startSession(config)
  267. .then((sessionId: string) =>
  268. {
  269. //defaultHeaders['Cookie'] = `sess_id=${sessionId}; c_locale=enUS`;
  270. return APIlogin(config, sessionId, config.user, config.pass);
  271. })
  272. .then((userData) =>
  273. {
  274. checkIfUserIsAuth(config, (errCheckAuth2) =>
  275. {
  276. if (isAuthenticated)
  277. {
  278. return done(null);
  279. }
  280. else
  281. {
  282. return done(errCheckAuth2);
  283. }
  284. });
  285. })
  286. .catch((errInChk) =>
  287. {
  288. return done(AuthError(errInChk.message));
  289. });
  290. }
  291. else if (config.logUsingCookie)
  292. {
  293. j.setCookie(request.cookie('c_userid=' + config.crUserId + '; Domain=crunchyroll.com; HttpOnly; hostOnly=false;'),
  294. CR_COOKIE_DOMAIN);
  295. j.setCookie(request.cookie('c_userkey=' + config.crUserKey + '; Domain=crunchyroll.com; HttpOnly; hostOnly=false;'),
  296. CR_COOKIE_DOMAIN);
  297. checkIfUserIsAuth(config, (errCheckAuth2) =>
  298. {
  299. if (isAuthenticated)
  300. {
  301. return done(null);
  302. }
  303. else
  304. {
  305. return done(errCheckAuth2);
  306. }
  307. });
  308. }
  309. else
  310. {
  311. /* First get https://www.crunchyroll.com/login to get the login token */
  312. const options =
  313. {
  314. jar: j,
  315. uri: 'https://www.crunchyroll.com/login'
  316. };
  317. cloudscraper.get(options, (err: Error, rep: string, body: string) =>
  318. {
  319. if (err) return done(err);
  320. console.log('------- GET --------');
  321. //console.error(err);
  322. console.log(body);
  323. console.log('------- /GET --------');
  324. const $ = cheerio.load(body);
  325. /* Get the token from the login page */
  326. const token = $('input[name="login_form[_token]"]').attr('value');
  327. if (token === '')
  328. {
  329. return done(AuthError('Can\'t find token!'));
  330. }
  331. console.log("Token: " + token);
  332. /* Now call the page again with the token and credentials */
  333. const options =
  334. {
  335. form:
  336. {
  337. 'login_form[name]': config.user,
  338. 'login_form[password]': config.pass,
  339. 'login_form[redirect_url]': '/',
  340. 'login_form[_token]': token
  341. },
  342. jar: j,
  343. url: 'https://www.crunchyroll.com/login'
  344. };
  345. console.info(options);
  346. cloudscraper.post(options, (err: Error, rep: string, body: string) =>
  347. {
  348. console.log('------- POST --------');
  349. //console.error(err);
  350. console.log(body);
  351. console.log('------- /POST --------');
  352. if (err)
  353. {
  354. console.log('------- ERR --------');
  355. return done(err);
  356. }
  357. /* Now let's check if we are authentificated */
  358. checkIfUserIsAuth(config, (errCheckAuth2) =>
  359. {
  360. if (isAuthenticated)
  361. {
  362. console.log('------- YES --------');
  363. return done(null);
  364. }
  365. else
  366. {
  367. console.error(err);
  368. console.log('------- WTF --------');
  369. return done(errCheckAuth2);
  370. }
  371. });
  372. });
  373. });
  374. }
  375. });
  376. }
  377. /**
  378. * Modifies the options to use the authenticated cookie jar.
  379. */
  380. function modify(options: string|any): any
  381. {
  382. if (typeof options !== 'string')
  383. {
  384. options.jar = j;
  385. return options;
  386. }
  387. return {
  388. jar: j,
  389. url: options.toString(),
  390. };
  391. }