123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525 |
- 'use strict';
- import cheerio = require('cheerio');
- import fs = require('fs');
- import mkdirp = require('mkdirp');
- import my_request = require('./my_request');
- import path = require('path');
- import subtitle from './subtitle/index';
- import vlos from './vlos';
- import video from './video/index';
- import xml2js = require('xml2js');
- import log = require('./log');
- /**
- * Streams the episode to disk.
- */
- export default function(config: IConfig, address: string, done: (err: Error, ign: boolean) => void)
- {
- scrapePage(config, address, (err, page) =>
- {
- if (err)
- {
- return done(err, false);
- }
- if (page.media != null)
- {
- /* No player to scrape */
- download(config, page, null, done);
- }
- else
- {
- /* The old way */
- scrapePlayer(config, address, page.id, (errS, player) =>
- {
- if (errS)
- {
- return done(errS, false);
- }
- download(config, page, player, done);
- });
- }
- });
- }
- /**
- * Completes a download and writes the message with an elapsed time.
- */
- function complete(epName: string, message: string, begin: number, done: (err: Error, ign: boolean) => void)
- {
- const timeInMs = Date.now() - begin;
- const seconds = prefix(Math.floor(timeInMs / 1000) % 60, 2);
- const minutes = prefix(Math.floor(timeInMs / 1000 / 60) % 60, 2);
- const hours = prefix(Math.floor(timeInMs / 1000 / 60 / 60), 2);
- log.dispEpisode(epName, message + ' (' + hours + ':' + minutes + ':' + seconds + ')', true);
- done(null, false);
- }
- /**
- * Check if a file exist..
- */
- function fileExist(path: string)
- {
- try
- {
- fs.statSync(path);
- return true;
- }
- catch (e)
- {
- return false;
- }
- }
- function sanitiseFileName(str: string)
- {
- const sanitized = str.replace(/[\/':\?\*"<>\\\.\|]/g, '_');
- return sanitized.replace(/{DIR_SEPARATOR}/g, '/');
- }
- /**
- * Downloads the subtitle and video.
- */
- function download(config: IConfig, page: IEpisodePage, player: IEpisodePlayer, done: (err: Error | string, ign: boolean) => void)
- {
- const serieFolder = sanitiseFileName(config.series || page.series);
- let fileName = sanitiseFileName(generateName(config, page));
- let filePath = path.join(config.output || process.cwd(), serieFolder, fileName);
- if (fileExist(filePath + '.mkv'))
- {
- let count = 0;
- if (config.rebuildcrp)
- {
- log.warn('Adding \'' + fileName + '\' to the DB...');
- return done(null, false);
- }
- log.warn('File \'' + fileName + '\' already exist...');
- do
- {
- count = count + 1;
- fileName = sanitiseFileName(generateName(config, page, '-' + count));
- filePath = path.join(config.output || process.cwd(), serieFolder, fileName);
- } while (fileExist(filePath + '.mkv'));
- log.warn('Renaming to \'' + fileName + '\'...');
- page.filename = fileName;
- }
- if (config.rebuildcrp)
- {
- log.warn('Ignoring \'' + fileName + '\' as it does not exist...');
- return done(null, true);
- }
- const ret = mkdirp(path.dirname(filePath));
- if (ret)
- {
- log.dispEpisode(fileName, 'Fetching...', false);
- downloadSubtitle(config, page, player, filePath, (errDS) =>
- {
- if (errDS)
- {
- log.dispEpisode(fileName, 'Error...', true);
- return done(errDS, false);
- }
- const now = Date.now();
- if ( ((page.media === null) && (player.video.file !== undefined))
- || ((page.media !== null) /* Do they still create page in advance for unreleased episodes? */) )
- {
- log.dispEpisode(fileName, 'Fetching video...', false);
- downloadVideo(config, page, player, filePath, (errDV) =>
- {
- if (errDV)
- {
- log.dispEpisode(fileName, 'Error...', true);
- return done(errDV, false);
- }
- if (config.merge)
- {
- return complete(fileName, 'Finished!', now, done);
- }
- let isSubtitled = true;
- if (page.media === null)
- {
- isSubtitled = Boolean(player.subtitle);
- }
- else
- {
- if (page.media.subtitles.length === 0)
- {
- isSubtitled = false;
- }
- }
- let videoExt = '.mp4';
- if ( (page.media === null) && (player.video.mode === 'RTMP'))
- {
- videoExt = path.extname(player.video.file);
- }
- log.dispEpisode(fileName, 'Merging...', false);
- video.merge(config, isSubtitled, videoExt, filePath, config.verbose, (errVM) =>
- {
- if (errVM)
- {
- log.dispEpisode(fileName, 'Error...', true);
- return done(errVM, false);
- }
- complete(fileName, 'Finished!', now, done);
- });
- });
- }
- else
- {
- log.dispEpisode(fileName, 'Ignoring: not released yet', true);
- done(null, true);
- }
- });
- }
- else
- {
- log.dispEpisode(fileName, 'Error creating folder \'' + filePath + '\'...', true);
- return done('Cannot create folder', false);
- }
- }
- /**
- * Saves the subtitles to disk.
- */
- function downloadSubtitle(config: IConfig, page: IEpisodePage, player: IEpisodePlayer,
- filePath: string, done: (err?: Error | string) => void)
- {
- if (page.media !== null)
- {
- const subs = page.media.subtitles;
- if (subs.length === 0)
- {
- /* No downloadable subtitles */
- console.warn('Can\'t find subtitle ?!');
- return done();
- }
- let i;
- let j;
- /* Find a proper subtitles */
- for (j = 0; j < config.sublang.length; j++)
- {
- const reqSubLang = config.sublang[j];
- for (i = 0; i < subs.length; i++)
- {
- const curSub = subs[i];
- if (curSub.format === 'ass' && curSub.language === reqSubLang)
- {
- my_request.get(config, curSub.url, (err, result) =>
- {
- if (err)
- {
- log.error('An error occured while fetching subtitles...');
- return done(err);
- }
- fs.writeFile(filePath + '.ass', '\ufeff' + result, done);
- });
- /* Break from the first loop */
- j = config.sublang.length;
- break;
- }
- }
- }
- if (i >= subs.length)
- {
- done('Cannot find subtitles with requested language(s)');
- }
- }
- else
- {
- const enc = player.subtitle;
- if (!enc)
- {
- return done();
- }
- subtitle.decode(enc.id, enc.iv, enc.data, (errSD, data) =>
- {
- if (errSD)
- {
- log.error('An error occured while getting subtitles...');
- return done(errSD);
- }
- if (config.debug)
- {
- log.dumpToDebug('SubtitlesXML', data);
- }
- const formats = subtitle.formats;
- const format = formats[config.format] ? config.format : 'ass';
- formats[format](config, data, (errF: Error, decodedSubtitle: string) =>
- {
- if (errF)
- {
- return done(errF);
- }
- fs.writeFile(filePath + '.' + format, '\ufeff' + decodedSubtitle, done);
- });
- });
- }
- }
- /**
- * Streams the video to disk.
- */
- function downloadVideo(config: IConfig, page: IEpisodePage, player: IEpisodePlayer,
- filePath: string, done: (err: any) => void)
- {
- if (player == null)
- {
- /* new way */
- const streams = page.media.streams;
- let i;
- /* Find a proper subtitles */
- for (i = 0; i < streams.length; i++)
- {
- if (streams[i].format === 'vo_adaptive_hls' && streams[i].audio_lang === 'jaJP' &&
- streams[i].hardsub_lang === null)
- {
- video.stream('', streams[i].url, '', filePath,
- 'mp4', 'HLS', config.verbose, done);
- break;
- }
- }
- if (i >= streams.length)
- {
- done('Cannot find a valid stream');
- }
- }
- else
- {
- /* Old way */
- video.stream(player.video.host, player.video.file, page.swf, filePath,
- path.extname(player.video.file), player.video.mode, config.verbose, done);
- }
- }
- /**
- * Names the file based on the config, page, series and tag.
- */
- function generateName(config: IConfig, page: IEpisodePage, extra = '')
- {
- const episodeNum = parseInt(page.episode, 10);
- const volumeNum = parseInt(page.volume, 10);
- const episode = (episodeNum < 10 ? '0' : '') + page.episode;
- const volume = (volumeNum < 10 ? '0' : '') + page.volume;
- const tag = config.tag || 'CrunchyRoll';
- const series = config.series || page.series;
- return config.nametmpl
- .replace(/{EPISODE_ID}/g, page.id.toString())
- .replace(/{EPISODE_NUMBER}/g, episode)
- .replace(/{SEASON_NUMBER}/g, volume)
- .replace(/{VOLUME_NUMBER}/g, volume)
- .replace(/{SEASON_TITLE}/g, page.season)
- .replace(/{VOLUME_TITLE}/g, page.season)
- .replace(/{SERIES_TITLE}/g, series)
- .replace(/{EPISODE_TITLE}/g, page.title)
- .replace(/{TAG}/g, tag) + extra;
- }
- /**
- * Prefixes a value.
- */
- function prefix(value: number|string, length: number)
- {
- let valueString = (typeof value !== 'string') ? String(value) : value;
- while (valueString.length < length)
- {
- valueString = '0' + valueString;
- }
- return valueString;
- }
- /**
- * Requests the page data and scrapes the id, episode, series and swf.
- */
- function scrapePage(config: IConfig, address: string, done: (err: Error, page?: IEpisodePage) => void)
- {
- const epId = parseInt((address.match(/[0-9]+$/) || ['0'])[0], 10);
- if (!epId)
- {
- return done(new Error('Invalid address.'));
- }
- my_request.get(config, address, (err, result) =>
- {
- if (err)
- {
- return done(err);
- }
- const $ = cheerio.load(result);
- /* First check if we have the new player */
- const vlosScript = $('#vilos-iframe-container');
- if (vlosScript)
- {
- const pageMetadata = JSON.parse($('script[type="application/ld+json"]')[0].children[0].data);
- const divScript = $('div[id="showmedia_video_box_wide"]');
- const scripts = divScript.find('script').toArray();
- const script = scripts[2].children[0].data;
- let seasonNumber = '1';
- let seasonTitle = '';
- if (pageMetadata.partOfSeason)
- {
- seasonNumber = pageMetadata.partOfSeason.seasonNumber;
- if (seasonNumber === '0') { seasonNumber = '1'; }
- seasonTitle = pageMetadata.partOfSeason.name;
- }
- done(null, vlos.getMedia(script, seasonTitle, seasonNumber));
- }
- else
- {
- /* Use the old way */
- const swf = /^([^?]+)/.exec($('link[rel=video_src]').attr('href'));
- const regexp = /\s*([^\n\r\t\f]+)\n?\s*[^0-9]*([0-9][\-0-9.]*)?,?\n?\s\s*[^0-9]*((PV )?[S0-9][P0-9.]*[a-fA-F]?)/;
- const seasonTitle = $('span[itemprop="title"]').text();
- const look = $('#showmedia_about_media').text();
- const episodeTitle = $('#showmedia_about_name').text().replace(/[“”]/g, '');
- const data = regexp.exec(look);
- if (config.debug) {
- log.dumpToDebug('episode page', $.html());
- }
- if (!swf || !data) {
- log.warn('Somethig unexpected in the page at ' + address + ' (data are: ' + look + ')');
- log.warn('Setting Season to ’0’ and episode to ’0’...');
- if (config.debug) {
- log.dumpToDebug('episode unexpected', look);
- }
- done(null, {
- episode: '0',
- id: epId,
- series: seasonTitle,
- season: seasonTitle,
- title: episodeTitle,
- swf: swf[1],
- volume: '0',
- filename: '',
- media: null,
- });
- } else {
- done(null, {
- episode: data[3],
- id: epId,
- series: data[1],
- season: seasonTitle,
- title: episodeTitle,
- swf: swf[1],
- volume: data[2] || '1',
- filename: '',
- media: null,
- });
- }
- }
- });
- }
- /**
- * Requests the player data and scrapes the subtitle and video data.
- */
- function scrapePlayer(config: IConfig, address: string, id: number, done: (err: Error, player?: IEpisodePlayer) => void)
- {
- const url = address.match(/^(https?:\/\/[^\/]+)/);
- if (!url)
- {
- return done(new Error('Invalid address.'));
- }
- const postForm = {
- current_page: address,
- video_format: config.video_format,
- video_quality: config.video_quality,
- media_id: id
- };
- my_request.post(config, url[1] + '/xml/?req=RpcApiVideoPlayer_GetStandardConfig&media_id=' + id, postForm,
- (err, result) =>
- {
- if (err)
- {
- return done(err);
- }
- xml2js.parseString(result, {
- explicitArray: false,
- explicitRoot: false,
- }, (errPS: Error, player: IEpisodePlayerConfig) =>
- {
- if (errPS)
- {
- return done(errPS);
- }
- try
- {
- const isSubtitled = Boolean(player['default:preload'].subtitle);
- let streamMode = 'RTMP';
- if (player['default:preload'].stream_info.host === '')
- {
- streamMode = 'HLS';
- }
- done(null, {
- subtitle: isSubtitled ? {
- data: player['default:preload'].subtitle.data,
- id: parseInt(player['default:preload'].subtitle.$.id, 10),
- iv: player['default:preload'].subtitle.iv,
- } : null,
- video: {
- file: player['default:preload'].stream_info.file,
- host: player['default:preload'].stream_info.host,
- mode: streamMode,
- },
- });
- } catch (parseError)
- {
- if (config.debug)
- {
- log.dumpToDebug('player scrape', parseError);
- }
- done(parseError);
- }
- });
- });
- }
|