series.ts 5.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223
  1. 'use strict';
  2. import cheerio = require('cheerio');
  3. import episode from './episode';
  4. // import fs = require('fs');
  5. import fs = require('fs-extra');
  6. import my_request = require('./my_request');
  7. import path = require('path');
  8. import url = require('url');
  9. import log = require('./log');
  10. const persistent = '.crpersistent';
  11. /**
  12. * Check if a file exist..
  13. */
  14. function fileExist(path: string)
  15. {
  16. try
  17. {
  18. fs.statSync(path);
  19. return true;
  20. } catch (e)
  21. {
  22. return false;
  23. }
  24. }
  25. /**
  26. * Streams the series to disk.
  27. */
  28. export default function(config: IConfig, address: string, done: (err: Error) => void)
  29. {
  30. const persistentPath = path.join(config.output || process.cwd(), persistent);
  31. /* Make a backup of the persistent file in case of */
  32. if (fileExist(persistentPath))
  33. {
  34. fs.copySync(persistentPath, persistentPath + '.backup');
  35. }
  36. fs.readFile(persistentPath, 'utf8', (err: Error, contents: string) =>
  37. {
  38. const cache = config.cache ? {} : JSON.parse(contents || '{}');
  39. page(config, address, (errP, page) =>
  40. {
  41. if (errP)
  42. {
  43. return done(errP);
  44. }
  45. let i = 0;
  46. (function next()
  47. {
  48. if (i >= page.episodes.length) return done(null);
  49. download(cache, config, address, page.episodes[i], (errD, ignored) =>
  50. {
  51. if (errD)
  52. {
  53. if (page.episodes[i].retry <= 0)
  54. {
  55. console.error(err);
  56. log.error('Cannot fetch episode "s' + page.episodes[i].volume + 'e' + page.episodes[i].episode +
  57. '", please rerun later');
  58. }
  59. else
  60. {
  61. if (config.verbose)
  62. {
  63. console.error(errD);
  64. }
  65. log.warn('Retrying to fetch episode "s' + page.episodes[i].volume + 'e' + page.episodes[i].episode +
  66. '" - Retry ' + page.episodes[i].retry + ' / ' + config.retry);
  67. page.episodes[i].retry -= 1;
  68. }
  69. next();
  70. }
  71. else
  72. {
  73. if ((ignored === false) || (ignored === undefined))
  74. {
  75. const newCache = JSON.stringify(cache, null, ' ');
  76. fs.writeFile(persistentPath, newCache, (errW: Error) =>
  77. {
  78. if (errW)
  79. {
  80. return done(errW);
  81. }
  82. i += 1;
  83. next();
  84. });
  85. }
  86. else
  87. {
  88. i += 1;
  89. next();
  90. }
  91. }
  92. });
  93. })();
  94. });
  95. });
  96. }
  97. /**
  98. * Downloads the episode.
  99. */
  100. function download(cache: {[address: string]: number}, config: IConfig,
  101. baseAddress: string, item: ISeriesEpisode,
  102. done: (err: Error, ign: boolean) => void)
  103. {
  104. if (!filter(config, item))
  105. {
  106. return done(null, false);
  107. }
  108. const address = url.resolve(baseAddress, item.address);
  109. if (cache[address])
  110. {
  111. return done(null, false);
  112. }
  113. episode(config, address, (err, ignored) =>
  114. {
  115. if (err)
  116. {
  117. return done(err, false);
  118. }
  119. cache[address] = Date.now();
  120. done(null, ignored);
  121. });
  122. }
  123. /**
  124. * Filters the item based on the configuration.
  125. */
  126. function filter(config: IConfig, item: ISeriesEpisode)
  127. {
  128. // Filter on chapter.
  129. const episodeFilter = config.episode;
  130. // Filter on volume.
  131. const volumeFilter = config.volume;
  132. const currentEpisode = parseInt(item.episode, 10);
  133. const currentVolume = item.volume;
  134. if ( ( (episodeFilter > 0) && (currentEpisode <= episodeFilter) ) ||
  135. ( (episodeFilter < 0) && (currentEpisode >= -episodeFilter) ) ||
  136. ( (volumeFilter > 0) && (currentVolume <= volumeFilter ) ) ||
  137. ( (volumeFilter < 0) && (currentVolume >= -volumeFilter ) ) )
  138. {
  139. return false;
  140. }
  141. return true;
  142. }
  143. /**
  144. * Requests the page and scrapes the episodes and series.
  145. */
  146. function page(config: IConfig, address: string, done: (err: Error, result?: ISeries) => void)
  147. {
  148. if (address[0] === '@')
  149. {
  150. log.info('Trying to fetch from ' + address.substr(1));
  151. const episodes: ISeriesEpisode[] = [];
  152. episodes.push({
  153. address: address.substr(1),
  154. episode: '',
  155. volume: 0,
  156. retry: config.retry,
  157. });
  158. done(null, {episodes: episodes.reverse(), series: ''});
  159. }
  160. else
  161. {
  162. let episodeCount = 0;
  163. my_request.get(config, address, (err, result) => {
  164. if (err) {
  165. return done(err);
  166. }
  167. const $ = cheerio.load(result);
  168. const title = $('span[itemprop=name]').text();
  169. if (!title) {
  170. return done(new Error('Invalid page.(' + address + ')'));
  171. }
  172. log.info('Checking availability for ' + title);
  173. const episodes: ISeriesEpisode[] = [];
  174. $('.episode').each((i, el) => {
  175. if ($(el).children('img[src*=coming_soon]').length) {
  176. return;
  177. }
  178. const volume = /([0-9]+)\s*$/.exec($(el).closest('ul').prev('a').text());
  179. const regexp = /Episode\s+((PV )?[S0-9][\-P0-9.]*[a-fA-F]?)\s*$/i;
  180. const episode = regexp.exec($(el).children('.series-title').text());
  181. const url = $(el).attr('href');
  182. if ((!url) || (!episode)) {
  183. return;
  184. }
  185. episodeCount += 1;
  186. episodes.push({
  187. address: url,
  188. episode: episode[1],
  189. volume: volume ? parseInt(volume[0], 10) : 1,
  190. retry: config.retry,
  191. });
  192. });
  193. if (episodeCount === 0)
  194. {
  195. log.warn('No episodes found for ' + title + '. Could it be a movie?');
  196. }
  197. done(null, {episodes: episodes.reverse(), series: title});
  198. });
  199. }
  200. }