episode.ts 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525
  1. 'use strict';
  2. import cheerio = require('cheerio');
  3. import fs = require('fs');
  4. import mkdirp = require('mkdirp');
  5. import my_request = require('./my_request');
  6. import path = require('path');
  7. import subtitle from './subtitle/index';
  8. import vlos from './vlos';
  9. import video from './video/index';
  10. import xml2js = require('xml2js');
  11. import log = require('./log');
  12. /**
  13. * Streams the episode to disk.
  14. */
  15. export default function(config: IConfig, address: string, done: (err: Error, ign: boolean) => void)
  16. {
  17. scrapePage(config, address, (err, page) =>
  18. {
  19. if (err)
  20. {
  21. return done(err, false);
  22. }
  23. if (page.media != null)
  24. {
  25. /* No player to scrape */
  26. download(config, page, null, done);
  27. }
  28. else
  29. {
  30. /* The old way */
  31. scrapePlayer(config, address, page.id, (errS, player) =>
  32. {
  33. if (errS)
  34. {
  35. return done(errS, false);
  36. }
  37. download(config, page, player, done);
  38. });
  39. }
  40. });
  41. }
  42. /**
  43. * Completes a download and writes the message with an elapsed time.
  44. */
  45. function complete(epName: string, message: string, begin: number, done: (err: Error, ign: boolean) => void)
  46. {
  47. const timeInMs = Date.now() - begin;
  48. const seconds = prefix(Math.floor(timeInMs / 1000) % 60, 2);
  49. const minutes = prefix(Math.floor(timeInMs / 1000 / 60) % 60, 2);
  50. const hours = prefix(Math.floor(timeInMs / 1000 / 60 / 60), 2);
  51. log.dispEpisode(epName, message + ' (' + hours + ':' + minutes + ':' + seconds + ')', true);
  52. done(null, false);
  53. }
  54. /**
  55. * Check if a file exist..
  56. */
  57. function fileExist(path: string)
  58. {
  59. try
  60. {
  61. fs.statSync(path);
  62. return true;
  63. }
  64. catch (e)
  65. {
  66. return false;
  67. }
  68. }
  69. function sanitiseFileName(str: string)
  70. {
  71. const sanitized = str.replace(/[\/':\?\*"<>\\\.\|]/g, '_');
  72. return sanitized.replace(/{DIR_SEPARATOR}/g, '/');
  73. }
  74. /**
  75. * Downloads the subtitle and video.
  76. */
  77. function download(config: IConfig, page: IEpisodePage, player: IEpisodePlayer, done: (err: Error | string, ign: boolean) => void)
  78. {
  79. const serieFolder = sanitiseFileName(config.series || page.series);
  80. let fileName = sanitiseFileName(generateName(config, page));
  81. let filePath = path.join(config.output || process.cwd(), serieFolder, fileName);
  82. if (fileExist(filePath + '.mkv'))
  83. {
  84. let count = 0;
  85. if (config.rebuildcrp)
  86. {
  87. log.warn('Adding \'' + fileName + '\' to the DB...');
  88. return done(null, false);
  89. }
  90. log.warn('File \'' + fileName + '\' already exist...');
  91. do
  92. {
  93. count = count + 1;
  94. fileName = sanitiseFileName(generateName(config, page, '-' + count));
  95. filePath = path.join(config.output || process.cwd(), serieFolder, fileName);
  96. } while (fileExist(filePath + '.mkv'));
  97. log.warn('Renaming to \'' + fileName + '\'...');
  98. page.filename = fileName;
  99. }
  100. if (config.rebuildcrp)
  101. {
  102. log.warn('Ignoring \'' + fileName + '\' as it does not exist...');
  103. return done(null, true);
  104. }
  105. const ret = mkdirp(path.dirname(filePath));
  106. if (ret)
  107. {
  108. log.dispEpisode(fileName, 'Fetching...', false);
  109. downloadSubtitle(config, page, player, filePath, (errDS) =>
  110. {
  111. if (errDS)
  112. {
  113. log.dispEpisode(fileName, 'Error...', true);
  114. return done(errDS, false);
  115. }
  116. const now = Date.now();
  117. if ( ((page.media === null) && (player.video.file !== undefined))
  118. || ((page.media !== null) /* Do they still create page in advance for unreleased episodes? */) )
  119. {
  120. log.dispEpisode(fileName, 'Fetching video...', false);
  121. downloadVideo(config, page, player, filePath, (errDV) =>
  122. {
  123. if (errDV)
  124. {
  125. log.dispEpisode(fileName, 'Error...', true);
  126. return done(errDV, false);
  127. }
  128. if (config.merge)
  129. {
  130. return complete(fileName, 'Finished!', now, done);
  131. }
  132. let isSubtitled = true;
  133. if (page.media === null)
  134. {
  135. isSubtitled = Boolean(player.subtitle);
  136. }
  137. else
  138. {
  139. if (page.media.subtitles.length === 0)
  140. {
  141. isSubtitled = false;
  142. }
  143. }
  144. let videoExt = '.mp4';
  145. if ( (page.media === null) && (player.video.mode === 'RTMP'))
  146. {
  147. videoExt = path.extname(player.video.file);
  148. }
  149. log.dispEpisode(fileName, 'Merging...', false);
  150. video.merge(config, isSubtitled, videoExt, filePath, config.verbose, (errVM) =>
  151. {
  152. if (errVM)
  153. {
  154. log.dispEpisode(fileName, 'Error...', true);
  155. return done(errVM, false);
  156. }
  157. complete(fileName, 'Finished!', now, done);
  158. });
  159. });
  160. }
  161. else
  162. {
  163. log.dispEpisode(fileName, 'Ignoring: not released yet', true);
  164. done(null, true);
  165. }
  166. });
  167. }
  168. else
  169. {
  170. log.dispEpisode(fileName, 'Error creating folder \'' + filePath + '\'...', true);
  171. return done('Cannot create folder', false);
  172. }
  173. }
  174. /**
  175. * Saves the subtitles to disk.
  176. */
  177. function downloadSubtitle(config: IConfig, page: IEpisodePage, player: IEpisodePlayer,
  178. filePath: string, done: (err?: Error | string) => void)
  179. {
  180. if (page.media !== null)
  181. {
  182. const subs = page.media.subtitles;
  183. if (subs.length === 0)
  184. {
  185. /* No downloadable subtitles */
  186. console.warn('Can\'t find subtitle ?!');
  187. return done();
  188. }
  189. let i;
  190. let j;
  191. /* Find a proper subtitles */
  192. for (j = 0; j < config.sublang.length; j++)
  193. {
  194. const reqSubLang = config.sublang[j];
  195. for (i = 0; i < subs.length; i++)
  196. {
  197. const curSub = subs[i];
  198. if (curSub.format === 'ass' && curSub.language === reqSubLang)
  199. {
  200. my_request.get(config, curSub.url, (err, result) =>
  201. {
  202. if (err)
  203. {
  204. log.error('An error occured while fetching subtitles...');
  205. return done(err);
  206. }
  207. fs.writeFile(filePath + '.ass', '\ufeff' + result, done);
  208. });
  209. /* Break from the first loop */
  210. j = config.sublang.length;
  211. break;
  212. }
  213. }
  214. }
  215. if (i >= subs.length)
  216. {
  217. done('Cannot find subtitles with requested language(s)');
  218. }
  219. }
  220. else
  221. {
  222. const enc = player.subtitle;
  223. if (!enc)
  224. {
  225. return done();
  226. }
  227. subtitle.decode(enc.id, enc.iv, enc.data, (errSD, data) =>
  228. {
  229. if (errSD)
  230. {
  231. log.error('An error occured while getting subtitles...');
  232. return done(errSD);
  233. }
  234. if (config.debug)
  235. {
  236. log.dumpToDebug('SubtitlesXML', data);
  237. }
  238. const formats = subtitle.formats;
  239. const format = formats[config.format] ? config.format : 'ass';
  240. formats[format](config, data, (errF: Error, decodedSubtitle: string) =>
  241. {
  242. if (errF)
  243. {
  244. return done(errF);
  245. }
  246. fs.writeFile(filePath + '.' + format, '\ufeff' + decodedSubtitle, done);
  247. });
  248. });
  249. }
  250. }
  251. /**
  252. * Streams the video to disk.
  253. */
  254. function downloadVideo(config: IConfig, page: IEpisodePage, player: IEpisodePlayer,
  255. filePath: string, done: (err: any) => void)
  256. {
  257. if (player == null)
  258. {
  259. /* new way */
  260. const streams = page.media.streams;
  261. let i;
  262. /* Find a proper subtitles */
  263. for (i = 0; i < streams.length; i++)
  264. {
  265. if (streams[i].format === 'vo_adaptive_hls' && streams[i].audio_lang === 'jaJP' &&
  266. streams[i].hardsub_lang === null)
  267. {
  268. video.stream('', streams[i].url, '', filePath,
  269. 'mp4', 'HLS', config.verbose, done);
  270. break;
  271. }
  272. }
  273. if (i >= streams.length)
  274. {
  275. done('Cannot find a valid stream');
  276. }
  277. }
  278. else
  279. {
  280. /* Old way */
  281. video.stream(player.video.host, player.video.file, page.swf, filePath,
  282. path.extname(player.video.file), player.video.mode, config.verbose, done);
  283. }
  284. }
  285. /**
  286. * Names the file based on the config, page, series and tag.
  287. */
  288. function generateName(config: IConfig, page: IEpisodePage, extra = '')
  289. {
  290. const episodeNum = parseInt(page.episode, 10);
  291. const volumeNum = parseInt(page.volume, 10);
  292. const episode = (episodeNum < 10 ? '0' : '') + page.episode;
  293. const volume = (volumeNum < 10 ? '0' : '') + page.volume;
  294. const tag = config.tag || 'CrunchyRoll';
  295. const series = config.series || page.series;
  296. return config.nametmpl
  297. .replace(/{EPISODE_ID}/g, page.id.toString())
  298. .replace(/{EPISODE_NUMBER}/g, episode)
  299. .replace(/{SEASON_NUMBER}/g, volume)
  300. .replace(/{VOLUME_NUMBER}/g, volume)
  301. .replace(/{SEASON_TITLE}/g, page.season)
  302. .replace(/{VOLUME_TITLE}/g, page.season)
  303. .replace(/{SERIES_TITLE}/g, series)
  304. .replace(/{EPISODE_TITLE}/g, page.title)
  305. .replace(/{TAG}/g, tag) + extra;
  306. }
  307. /**
  308. * Prefixes a value.
  309. */
  310. function prefix(value: number|string, length: number)
  311. {
  312. let valueString = (typeof value !== 'string') ? String(value) : value;
  313. while (valueString.length < length)
  314. {
  315. valueString = '0' + valueString;
  316. }
  317. return valueString;
  318. }
  319. /**
  320. * Requests the page data and scrapes the id, episode, series and swf.
  321. */
  322. function scrapePage(config: IConfig, address: string, done: (err: Error, page?: IEpisodePage) => void)
  323. {
  324. const epId = parseInt((address.match(/[0-9]+$/) || ['0'])[0], 10);
  325. if (!epId)
  326. {
  327. return done(new Error('Invalid address.'));
  328. }
  329. my_request.get(config, address, (err, result) =>
  330. {
  331. if (err)
  332. {
  333. return done(err);
  334. }
  335. const $ = cheerio.load(result);
  336. /* First check if we have the new player */
  337. const vlosScript = $('#vilos-iframe-container');
  338. if (vlosScript)
  339. {
  340. const pageMetadata = JSON.parse($('script[type="application/ld+json"]')[0].children[0].data);
  341. const divScript = $('div[id="showmedia_video_box_wide"]');
  342. const scripts = divScript.find('script').toArray();
  343. const script = scripts[2].children[0].data;
  344. let seasonNumber = '1';
  345. let seasonTitle = '';
  346. if (pageMetadata.partOfSeason)
  347. {
  348. seasonNumber = pageMetadata.partOfSeason.seasonNumber;
  349. if (seasonNumber === '0') { seasonNumber = '1'; }
  350. seasonTitle = pageMetadata.partOfSeason.name;
  351. }
  352. done(null, vlos.getMedia(script, seasonTitle, seasonNumber));
  353. }
  354. else
  355. {
  356. /* Use the old way */
  357. const swf = /^([^?]+)/.exec($('link[rel=video_src]').attr('href'));
  358. const regexp = /\s*([^\n\r\t\f]+)\n?\s*[^0-9]*([0-9][\-0-9.]*)?,?\n?\s\s*[^0-9]*((PV )?[S0-9][P0-9.]*[a-fA-F]?)/;
  359. const seasonTitle = $('span[itemprop="title"]').text();
  360. const look = $('#showmedia_about_media').text();
  361. const episodeTitle = $('#showmedia_about_name').text().replace(/[“”]/g, '');
  362. const data = regexp.exec(look);
  363. if (config.debug) {
  364. log.dumpToDebug('episode page', $.html());
  365. }
  366. if (!swf || !data) {
  367. log.warn('Somethig unexpected in the page at ' + address + ' (data are: ' + look + ')');
  368. log.warn('Setting Season to ’0’ and episode to ’0’...');
  369. if (config.debug) {
  370. log.dumpToDebug('episode unexpected', look);
  371. }
  372. done(null, {
  373. episode: '0',
  374. id: epId,
  375. series: seasonTitle,
  376. season: seasonTitle,
  377. title: episodeTitle,
  378. swf: swf[1],
  379. volume: '0',
  380. filename: '',
  381. media: null,
  382. });
  383. } else {
  384. done(null, {
  385. episode: data[3],
  386. id: epId,
  387. series: data[1],
  388. season: seasonTitle,
  389. title: episodeTitle,
  390. swf: swf[1],
  391. volume: data[2] || '1',
  392. filename: '',
  393. media: null,
  394. });
  395. }
  396. }
  397. });
  398. }
  399. /**
  400. * Requests the player data and scrapes the subtitle and video data.
  401. */
  402. function scrapePlayer(config: IConfig, address: string, id: number, done: (err: Error, player?: IEpisodePlayer) => void)
  403. {
  404. const url = address.match(/^(https?:\/\/[^\/]+)/);
  405. if (!url)
  406. {
  407. return done(new Error('Invalid address.'));
  408. }
  409. const postForm = {
  410. current_page: address,
  411. video_format: config.video_format,
  412. video_quality: config.video_quality,
  413. media_id: id
  414. };
  415. my_request.post(config, url[1] + '/xml/?req=RpcApiVideoPlayer_GetStandardConfig&media_id=' + id, postForm,
  416. (err, result) =>
  417. {
  418. if (err)
  419. {
  420. return done(err);
  421. }
  422. xml2js.parseString(result, {
  423. explicitArray: false,
  424. explicitRoot: false,
  425. }, (errPS: Error, player: IEpisodePlayerConfig) =>
  426. {
  427. if (errPS)
  428. {
  429. return done(errPS);
  430. }
  431. try
  432. {
  433. const isSubtitled = Boolean(player['default:preload'].subtitle);
  434. let streamMode = 'RTMP';
  435. if (player['default:preload'].stream_info.host === '')
  436. {
  437. streamMode = 'HLS';
  438. }
  439. done(null, {
  440. subtitle: isSubtitled ? {
  441. data: player['default:preload'].subtitle.data,
  442. id: parseInt(player['default:preload'].subtitle.$.id, 10),
  443. iv: player['default:preload'].subtitle.iv,
  444. } : null,
  445. video: {
  446. file: player['default:preload'].stream_info.file,
  447. host: player['default:preload'].stream_info.host,
  448. mode: streamMode,
  449. },
  450. });
  451. } catch (parseError)
  452. {
  453. if (config.debug)
  454. {
  455. log.dumpToDebug('player scrape', parseError);
  456. }
  457. done(parseError);
  458. }
  459. });
  460. });
  461. }