series.ts 5.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226
  1. 'use strict';
  2. import cheerio = require('cheerio');
  3. import episode from './episode';
  4. // import fs = require('fs');
  5. import fs = require('fs-extra');
  6. import my_request = require('./my_request');
  7. import path = require('path');
  8. import url = require('url');
  9. import log = require('./log');
  10. const persistent = '.crpersistent';
  11. /**
  12. * Check if a file exist..
  13. */
  14. function fileExist(path: string)
  15. {
  16. try
  17. {
  18. fs.statSync(path);
  19. return true;
  20. } catch (e)
  21. {
  22. return false;
  23. }
  24. }
  25. /**
  26. * Streams the series to disk.
  27. */
  28. export default function(config: IConfig, task: IConfigTask, done: (err: Error) => void)
  29. {
  30. const persistentPath = path.join(config.output || process.cwd(), persistent);
  31. /* Make a backup of the persistent file in case of */
  32. if (fileExist(persistentPath))
  33. {
  34. fs.copySync(persistentPath, persistentPath + '.backup');
  35. }
  36. fs.readFile(persistentPath, 'utf8', (err: Error, contents: string) =>
  37. {
  38. const cache = config.cache ? {} : JSON.parse(contents || '{}');
  39. pageScrape(config, task, (errP, page) =>
  40. {
  41. if (errP)
  42. {
  43. return done(errP);
  44. }
  45. let i = 0;
  46. (function next()
  47. {
  48. if (config.debug)
  49. {
  50. log.dumpToDebug('Episode ' + i, JSON.stringify(page.episodes[i]));
  51. }
  52. if (i >= page.episodes.length) return done(null);
  53. download(cache, config, task, page.episodes[i], (errD, ignored) =>
  54. {
  55. if (errD)
  56. {
  57. if (page.episodes[i].retry <= 0)
  58. {
  59. log.dispEpisode(config.filename, 'Error...', true);
  60. log.error(errD);
  61. log.error('Cannot fetch episode "s' + page.episodes[i].volume + 'e' + page.episodes[i].episode +
  62. '", please rerun later');
  63. /* Go to the next on the list */
  64. i += 1;
  65. }
  66. else
  67. {
  68. log.dispEpisode(config.filename, 'Error...', true);
  69. if ((config.verbose) || (config.debug))
  70. {
  71. if (config.debug)
  72. {
  73. log.dumpToDebug('series address', task.address);
  74. log.dumpToDebug('series error', errD.stack || errD);
  75. log.dumpToDebug('series data', JSON.stringify(page));
  76. }
  77. log.error(errD);
  78. }
  79. log.warn('Retrying to fetch episode "s' + page.episodes[i].volume + 'e' + page.episodes[i].episode +
  80. '" - Retry ' + page.episodes[i].retry + ' / ' + config.retry);
  81. page.episodes[i].retry -= 1;
  82. }
  83. next();
  84. }
  85. else
  86. {
  87. if ((ignored === false) || (ignored === undefined))
  88. {
  89. const newCache = JSON.stringify(cache, null, ' ');
  90. fs.writeFile(persistentPath, newCache, (errW: Error) =>
  91. {
  92. if (errW)
  93. {
  94. return done(errW);
  95. }
  96. i += 1;
  97. next();
  98. });
  99. }
  100. else
  101. {
  102. i += 1;
  103. next();
  104. }
  105. }
  106. });
  107. })();
  108. });
  109. });
  110. }
  111. /**
  112. * Downloads the episode.
  113. */
  114. function download(cache: {[address: string]: number}, config: IConfig,
  115. task: IConfigTask, item: ISeriesEpisode,
  116. done: (err: Error, ign: boolean) => void)
  117. {
  118. const episodeNumber = parseInt(item.episode, 10);
  119. if ( (episodeNumber < task.episode_min) ||
  120. (episodeNumber > task.episode_max) )
  121. {
  122. return done(null, false);
  123. }
  124. const address = url.resolve(task.address, item.address);
  125. if (cache[address])
  126. {
  127. return done(null, false);
  128. }
  129. episode(config, address, (err, ignored) =>
  130. {
  131. if (err)
  132. {
  133. return done(err, false);
  134. }
  135. cache[address] = Date.now();
  136. done(null, ignored);
  137. });
  138. }
  139. /**
  140. * Requests the page and scrapes the episodes and series.
  141. */
  142. function pageScrape(config: IConfig, task: IConfigTask, done: (err: Error, result?: ISeries) => void)
  143. {
  144. if (task.address[0] === '@')
  145. {
  146. log.info('Trying to fetch from ' + task.address.substr(1));
  147. const episodes: ISeriesEpisode[] = [];
  148. episodes.push({
  149. address: task.address.substr(1),
  150. episode: '',
  151. volume: 0,
  152. retry: config.retry,
  153. });
  154. done(null, {episodes: episodes.reverse(), series: ''});
  155. }
  156. else
  157. {
  158. let episodeCount = 0;
  159. my_request.get(config, task.address, (err, result) => {
  160. if (err) {
  161. return done(err);
  162. }
  163. const $ = cheerio.load(result);
  164. const title = $('span[itemprop=name]').text();
  165. if (config.debug)
  166. {
  167. log.dumpToDebug('serie page', $.html());
  168. }
  169. if (!title) {
  170. if (config.debug)
  171. {
  172. log.dumpToDebug('missing title', task.address);
  173. }
  174. return done(new Error('Invalid page.(' + task.address + ')'));
  175. }
  176. log.info('Checking availability for ' + title);
  177. const episodes: ISeriesEpisode[] = [];
  178. $('.episode').each((i, el) => {
  179. if ($(el).children('img[src*=coming_soon]').length) {
  180. return;
  181. }
  182. const volume = /([0-9]+)\s*$/.exec($(el).closest('ul').prev('a').text());
  183. const regexp = /Episode\s+((PV )?[S0-9][\-P0-9.]*[a-fA-F]?)\s*$/i;
  184. const episode = regexp.exec($(el).children('.series-title').text());
  185. const url = $(el).attr('href');
  186. if ((!url) || (!episode)) {
  187. return;
  188. }
  189. episodeCount += 1;
  190. episodes.push({
  191. address: url,
  192. episode: episode[1],
  193. volume: volume ? parseInt(volume[0], 10) : 1,
  194. retry: config.retry,
  195. });
  196. });
  197. if (episodeCount === 0)
  198. {
  199. log.warn('No episodes found for ' + title + '. Could it be a movie?');
  200. }
  201. done(null, {episodes: episodes.reverse(), series: title});
  202. });
  203. }
  204. }