series.ts 7.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274
  1. 'use strict';
  2. import cheerio = require('cheerio');
  3. import episode from './episode';
  4. import fs = require('fs-extra');
  5. import my_request = require('./my_request');
  6. import path = require('path');
  7. import url = require('url');
  8. import log = require('./log');
  9. import languages = require('./languages');
  10. const persistent = '.crpersistent';
  11. /**
  12. * Check if a file exist..
  13. */
  14. function fileExist(path: string)
  15. {
  16. try
  17. {
  18. fs.statSync(path);
  19. return true;
  20. } catch (e)
  21. {
  22. return false;
  23. }
  24. }
  25. /**
  26. * Streams the series to disk.
  27. */
  28. export default function(config: IConfig, task: IConfigTask, done: (err: any) => void)
  29. {
  30. const persistentPath = path.join(config.output || process.cwd(), persistent);
  31. /* Make a backup of the persistent file in case of */
  32. if (fileExist(persistentPath))
  33. {
  34. fs.copySync(persistentPath, persistentPath + '.backup');
  35. }
  36. fs.readFile(persistentPath, 'utf8', (err: Error, contents: string) =>
  37. {
  38. const cache = config.cache ? {} : JSON.parse(contents || '{}');
  39. pageScrape(config, task, (errP, page) =>
  40. {
  41. if (errP)
  42. {
  43. const reqErr = errP.error;
  44. if ((reqErr !== undefined) && (reqErr.syscall))
  45. {
  46. if ((reqErr.syscall === 'getaddrinfo') && (reqErr.errno === 'ENOTFOUND'))
  47. {
  48. log.error('The URL \'' + task.address + '\' is invalid, please check => I\'m ignoring it.');
  49. }
  50. }
  51. return done(errP);
  52. }
  53. let i = 0;
  54. (function next()
  55. {
  56. if (config.debug)
  57. {
  58. log.dumpToDebug('Episode ' + i, JSON.stringify(page.episodes[i]));
  59. }
  60. if (i >= page.episodes.length) return done(null);
  61. download(cache, config, task, page.episodes[i], (errD, ignored) =>
  62. {
  63. if (errD)
  64. {
  65. /* Check if domain is valid */
  66. const reqErr = errD.error;
  67. if ((reqErr !== undefined) && (reqErr.syscall))
  68. {
  69. if ((reqErr.syscall === 'getaddrinfo') && (reqErr.errno === 'ENOTFOUND'))
  70. {
  71. page.episodes[i].retry = 0;
  72. log.error('The URL \'' + task.address + '\' is invalid, please check => I\'m ignoring it.');
  73. }
  74. }
  75. if (page.episodes[i].retry <= 0)
  76. {
  77. log.error(JSON.stringify(errD));
  78. log.error('Cannot fetch episode "s' + page.episodes[i].volume + 'e' + page.episodes[i].episode +
  79. '", please rerun later');
  80. /* Go to the next on the list */
  81. i += 1;
  82. }
  83. else
  84. {
  85. if ((config.verbose) || (config.debug))
  86. {
  87. if (config.debug)
  88. {
  89. log.dumpToDebug('series address', task.address);
  90. log.dumpToDebug('series error', JSON.stringify(errD));
  91. log.dumpToDebug('series data', JSON.stringify(page));
  92. }
  93. log.error(errD);
  94. }
  95. log.warn('Retrying to fetch episode "s' + page.episodes[i].volume + 'e' + page.episodes[i].episode +
  96. '" - Retry ' + page.episodes[i].retry + ' / ' + config.retry);
  97. page.episodes[i].retry -= 1;
  98. }
  99. setTimeout(next, config.sleepTime);
  100. return;
  101. }
  102. else
  103. {
  104. if ((ignored === false) || (ignored === undefined))
  105. {
  106. const newCache = JSON.stringify(cache, null, ' ');
  107. fs.writeFile(persistentPath, newCache, (errW: Error) =>
  108. {
  109. if (errW)
  110. {
  111. return done(errW);
  112. }
  113. i += 1;
  114. setTimeout(next, config.sleepTime);
  115. return;
  116. });
  117. }
  118. else
  119. {
  120. i += 1;
  121. setTimeout(next, config.sleepTime);
  122. return;
  123. }
  124. }
  125. });
  126. })();
  127. });
  128. });
  129. }
  130. /**
  131. * Downloads the episode.
  132. */
  133. function download(cache: {[address: string]: number}, config: IConfig,
  134. task: IConfigTask, item: ISeriesEpisode,
  135. done: (err: any, ign: boolean) => void)
  136. {
  137. const episodeNumber = parseInt(item.episode, 10);
  138. const seasonNumber = item.volume;
  139. if ( (episodeNumber < task.episode_min.episode) ||
  140. (episodeNumber > task.episode_max.episode) )
  141. {
  142. return done(null, false);
  143. }
  144. const address = url.resolve(task.address, item.address);
  145. if (cache[address])
  146. {
  147. return done(null, false);
  148. }
  149. episode(config, address, (err, ignored) =>
  150. {
  151. if (err)
  152. {
  153. return done(err, false);
  154. }
  155. cache[address] = Date.now();
  156. done(null, ignored);
  157. });
  158. }
  159. /**
  160. * Requests the page and scrapes the episodes and series.
  161. */
  162. function pageScrape(config: IConfig, task: IConfigTask, done: (err: any, result?: ISeries) => void)
  163. {
  164. if (task.address[0] === '@')
  165. {
  166. log.info('Trying to fetch from ' + task.address.substr(1));
  167. const episodes: ISeriesEpisode[] = [];
  168. episodes.push({
  169. address: task.address.substr(1),
  170. episode: '',
  171. seasonName: '',
  172. volume: 0,
  173. retry: config.retry,
  174. });
  175. done(null, {episodes: episodes.reverse(), series: ''});
  176. }
  177. else
  178. {
  179. let episodeCount = 0;
  180. my_request.get(config, task.address, (err, result) => {
  181. if (err)
  182. {
  183. return done(err);
  184. }
  185. const $ = cheerio.load(result);
  186. const title = $('meta[itemprop=name]').attr('content');
  187. if (config.debug)
  188. {
  189. log.dumpToDebug('serie page', $.html());
  190. }
  191. if (!title) {
  192. if (config.debug)
  193. {
  194. log.dumpToDebug('missing title', task.address);
  195. }
  196. return done(new Error('Invalid page.(' + task.address + ')'));
  197. }
  198. log.info('Checking availability for ' + title);
  199. const episodes: ISeriesEpisode[] = [];
  200. if ($('.availability-notes-low').length)
  201. {
  202. log.warn('This serie may have georestriction and some missings episode (like some dubs)' +
  203. ' [Message: ' + $('.availability-notes-low').text() + '].');
  204. }
  205. if ($('.availability-notes-high').length)
  206. {
  207. log.warnMore('This serie probably have georestriction and will miss some episodes' +
  208. ' [Message: ' + $('.availability-notes-high').text() + '].');
  209. }
  210. $('.episode').each((i, el) => {
  211. if ($(el).children('img[src*=coming_soon]').length) {
  212. return;
  213. }
  214. const season_name = $(el).closest('ul').prev('a').text();
  215. const volume = /([0-9]+)\s*$/.exec($(el).closest('ul').prev('a').text());
  216. const regexp = languages.get_epregexp(config);
  217. const episode = regexp.exec($(el).children('.series-title').text());
  218. const url = $(el).attr('href');
  219. const igndub_re = languages.get_diregexp(config);
  220. if (config.ignoredub && (igndub_re.exec(season_name)))
  221. {
  222. return;
  223. }
  224. if ((!url) || (!episode))
  225. {
  226. return;
  227. }
  228. episodeCount += 1;
  229. episodes.push({
  230. address: url,
  231. episode: episode[1],
  232. seasonName: season_name,
  233. volume: volume ? parseInt(volume[0], 10) : 1,
  234. retry: config.retry,
  235. });
  236. });
  237. if (episodeCount === 0)
  238. {
  239. log.warn('No episodes found for ' + title + '. Could it be a movie?');
  240. }
  241. done(null, {episodes: episodes.reverse(), series: title});
  242. });
  243. }
  244. }