From 25d254d71614b861f64da73f44b5c12bba063464 Mon Sep 17 00:00:00 2001 From: gujal Date: Fri, 13 Mar 2020 12:01:55 +1300 Subject: [PATCH 001/234] fembed additional domain (#292) --- lib/resolveurl/plugins/fembed.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/resolveurl/plugins/fembed.py b/lib/resolveurl/plugins/fembed.py index 83ef0803..25c4e2c9 100644 --- a/lib/resolveurl/plugins/fembed.py +++ b/lib/resolveurl/plugins/fembed.py @@ -26,8 +26,8 @@ class FembedResolver(ResolveUrl): name = "fembed" domains = ["fembed.com", "24hd.club", "vcdn.io", "sharinglink.club", "votrefiles.club", "there.to", "femoload.xyz", "feurl.com", "dailyplanet.pw", "jplayer.net", "xstreamcdn.com", "gcloud.live", - "vcdnplay.com", "vidohd.com", "vidsource.me", "zidiplay.com"] - pattern = r'(?://|\.)((?:fembed|feurl|24hd|vcdn|sharinglink|votrefiles|femoload|dailyplanet|jplayer|there|gcloud|xstreamcdn|vcdnplay|vidohd|vidsource|zidiplay)\.(?:com|club|io|xyz|pw|net|to|live|me))/v/([a-zA-Z0-9-]+)' + "vcdnplay.com", "vidohd.com", "vidsource.me", "votrefile.xyz", "zidiplay.com"] + pattern = r'(?://|\.)((?:fembed|feurl|24hd|vcdn|sharinglink|votrefiles?|femoload|dailyplanet|jplayer|there|gcloud|xstreamcdn|vcdnplay|vidohd|vidsource|zidiplay)\.(?:com|club|io|xyz|pw|net|to|live|me))/v/([a-zA-Z0-9-]+)' def __init__(self): self.net = common.Net() From 37e1814e9103b4e61a9f376da7de4c6b81e401d2 Mon Sep 17 00:00:00 2001 From: Paradigm Shifter Date: Wed, 18 Mar 2020 05:50:04 +0200 Subject: [PATCH 002/234] Fix streamz (#295) * Added the web_url as Referer to explicitly get final media url --- lib/resolveurl/plugins/streamz.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/lib/resolveurl/plugins/streamz.py b/lib/resolveurl/plugins/streamz.py index e9c1553d..b678c465 100644 --- a/lib/resolveurl/plugins/streamz.py +++ b/lib/resolveurl/plugins/streamz.py @@ -29,15 +29,26 @@ def __init__(self): self.net = common.Net() def get_media_url(self, host, media_id): + web_url = self.get_url(host, media_id) headers = {'User-Agent': common.RAND_UA} html = self.net.http_GET(web_url, headers=headers).content - if 'File not found, sorry!' not in html: - sources = helpers.scrape_sources(html) - if sources: - return helpers.pick_source(sources) + helpers.append_headers(headers) - raise ResolverError("Video not found") + html += helpers.get_packed_data(html) + sources = helpers.scrape_sources(html) + + sources = [('mp4', self.net.http_HEAD(sources[0][1]).get_url())] + + if sources: + + headers.update({'Referer': web_url}) + + return helpers.pick_source(sources) + helpers.append_headers(headers) + + else: + + raise ResolverError("Video not found") def get_url(self, host, media_id): + return self._default_get_url(host, media_id, template='https://{host}/{media_id}') From b2016ddcecdd800964591aa96669dc318c49cfdd Mon Sep 17 00:00:00 2001 From: lukkie112 <49287495+lukkie112@users.noreply.github.com> Date: Wed, 18 Mar 2020 22:05:36 +0100 Subject: [PATCH 003/234] Create streamwire.py (#297) --- lib/resolveurl/plugins/streamwire.py | 34 ++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 lib/resolveurl/plugins/streamwire.py diff --git a/lib/resolveurl/plugins/streamwire.py b/lib/resolveurl/plugins/streamwire.py new file mode 100644 index 00000000..c1130590 --- /dev/null +++ b/lib/resolveurl/plugins/streamwire.py @@ -0,0 +1,34 @@ +""" + Plugin for ResolveUrl + Copyright (C) 2020 + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +""" + +from __resolve_generic__ import ResolveGeneric +from lib import helpers + + +class StreamWireResolver(ResolveGeneric): + name = "streamwire" + domains = ['streamwire.net'] + pattern = r'(?://|\.)(streamwire\.net)/(?:embed-)?([0-9a-zA-Z]+)' + + def get_media_url(self, host, media_id): + return helpers.get_media_url(self.get_url(host, media_id), + patterns=[r'''src:\s*"(?P[^"]+)'''], + generic_patterns=False) + + def get_url(self, host, media_id): + return self._default_get_url(host, media_id, template='https://{host}/embed-{media_id}.html') From b8fbdaff9f95d69096da02bfb6b758f960f750f8 Mon Sep 17 00:00:00 2001 From: Paradigm Shifter Date: Wed, 18 Mar 2020 23:07:31 +0200 Subject: [PATCH 004/234] Move head request in the conditional (#296) Streamz.cc In order to properly result in ResolverError in case the scrape sources function does not end up in a result. --- lib/resolveurl/plugins/streamz.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/resolveurl/plugins/streamz.py b/lib/resolveurl/plugins/streamz.py index b678c465..fdb2ffdc 100644 --- a/lib/resolveurl/plugins/streamz.py +++ b/lib/resolveurl/plugins/streamz.py @@ -37,10 +37,10 @@ def get_media_url(self, host, media_id): html += helpers.get_packed_data(html) sources = helpers.scrape_sources(html) - sources = [('mp4', self.net.http_HEAD(sources[0][1]).get_url())] - if sources: + sources = [('mp4', self.net.http_HEAD(sources[0][1]).get_url())] + headers.update({'Referer': web_url}) return helpers.pick_source(sources) + helpers.append_headers(headers) From 4a4170a2667a461fc31b0f5e506d0191ea1b5608 Mon Sep 17 00:00:00 2001 From: mbebe Date: Wed, 18 Mar 2020 22:09:07 +0100 Subject: [PATCH 005/234] Update waaw (#294) * Update waaw * upd waaw * Update waaw.py * Update waaw.py --- lib/resolveurl/plugins/waaw.py | 160 +++++---------------------------- 1 file changed, 21 insertions(+), 139 deletions(-) diff --git a/lib/resolveurl/plugins/waaw.py b/lib/resolveurl/plugins/waaw.py index 8f1a974b..11e70c9f 100644 --- a/lib/resolveurl/plugins/waaw.py +++ b/lib/resolveurl/plugins/waaw.py @@ -12,8 +12,10 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . ''' -import re, urllib, json -from lib import helpers + +import re, urllib + +from . lib import helpers from resolveurl import common from resolveurl.resolver import ResolveUrl, ResolverError @@ -22,146 +24,26 @@ class WaawResolver(ResolveUrl): name = "waaw" domains = ["waaw.tv", "hqq.watch", "netu.tv", "hqq.tv", "waaw1.tv"] pattern = "(?://|\.)((?:waaw1?|netu|hqq)\.(?:tv|watch))/(?:watch_video\.php\?v|.+?vid)=([a-zA-Z0-9]+)" - + def __init__(self): self.net = common.Net() - + def get_media_url(self, host, media_id): - web_url = self.get_url(host, media_id) headers = {'User-Agent': common.FF_USER_AGENT, - 'Referer': 'https://waaw.tv/watch_video.php?v=%s&post=1' % media_id} - html = self.net.http_GET(web_url, headers=headers).content - - if html: - try: - wise = re.search('''''', html, - re.DOTALL | re.I).groups()[0] - data_unwise = self.jswise(wise).replace("\\", "") - try: - at = re.search('at=(\w+)', data_unwise, re.I).groups()[0] - except: - at = "" - try: - http_referer = re.search('http_referer=(.*?)&', data_unwise, re.I).groups()[0] - except: - http_referer = "" - player_url = "http://hqq.watch/sec/player/embed_player.php?iss=&vid=%s&at=%s&autoplayed=yes&referer=on&http_referer=%s&pass=&embed_from=&need_captcha=0&hash_from=&secured=0" % ( - media_id, at, http_referer) - headers.update({'Referer': web_url}) - data_player = self.net.http_GET(player_url, headers=headers).content - data_unescape = re.findall('document.write\(unescape\("([^"]+)"', data_player) - data = "" - for d in data_unescape: - data += urllib.unquote(d) - - data_unwise_player = "" - wise = "" - wise = re.search('''''', data_player, - re.DOTALL | re.I) - if wise: - data_unwise_player = self.jswise(wise.group(1)).replace("\\", "") - - try: - vars_data = re.search('/player/get_md5.php",\s*\{(.*?)\}', data, re.DOTALL | re.I).groups()[0] - except: - vars_data = "" - matches = re.findall('\s*([^:]+):\s*([^,]*)[,"]', vars_data) - params = {} - for key, value in matches: - if key == "adb": - params[key] = "0/" - elif '"' in value: - params[key] = value.replace('"', '') - else: - try: - value_var = re.search('var\s*%s\s*=\s*"([^"]+)"' % value, data, re.I).groups()[0] - except: - value_var = "" - if not value_var and data_unwise_player: - try: - value_var = \ - re.search('var\s*%s\s*=\s*"([^"]+)"' % value, data_unwise_player, re.I).groups()[0] - except: - value_var = "" - params[key] = value_var - - params = urllib.urlencode(params) - headers.update({'X-Requested-With': 'XMLHttpRequest', 'Referer': player_url}) - data = "" - data = self.net.http_GET("http://hqq.watch/player/get_md5.php?" + params, headers=headers).content - url_data = json.loads(data) - media_url = "https:" + self.tb(url_data["obf_link"].replace("#", "")) + ".mp4.m3u8" - - if media_url: - del headers['X-Requested-With'] - headers.update({'Origin': 'https://hqq.watch'}) - return media_url + helpers.append_headers(headers) - - except Exception as e: - raise ResolverError(e) - - raise ResolverError('Video not found') - - def tb(self, b_m3u8_2): - j = 0 - s2 = "" - while j < len(b_m3u8_2): - s2 += "\\u0" + b_m3u8_2[j:(j + 3)] - j += 3 - - return s2.decode('unicode-escape').encode('ASCII', 'ignore') - - ## loop2unobfuscated - def jswise(self, wise): - while True: - wise = re.search("var\s.+?\('([^']+)','([^']+)','([^']+)','([^']+)'\)", wise, re.DOTALL) - if not wise: break - ret = wise = self.js_wise(wise.groups()) - - return ret - - ## js2python - def js_wise(self, wise): - w, i, s, e = wise - - v0 = 0; - v1 = 0; - v2 = 0 - v3 = []; - v4 = [] - - while True: - if v0 < 5: - v4.append(w[v0]) - elif v0 < len(w): - v3.append(w[v0]) - v0 += 1 - if v1 < 5: - v4.append(i[v1]) - elif v1 < len(i): - v3.append(i[v1]) - v1 += 1 - if v2 < 5: - v4.append(s[v2]) - elif v2 < len(s): - v3.append(s[v2]) - v2 += 1 - if len(w) + len(i) + len(s) + len(e) == len(v3) + len(v4) + len(e): break - - v5 = "".join(v3); - v6 = "".join(v4) - v1 = 0 - v7 = [] - - for v0 in range(0, len(v3), 2): - v8 = -1 - if ord(v6[v1]) % 2: v8 = 1 - v7.append(chr(int(v5[v0:v0 + 2], 36) - v8)) - v1 += 1 - if v1 >= len(v4): v1 = 0 - - return "".join(v7) - + 'x-requested-with': 'XMLHttpRequest'} + params = ( + ('ver', '0'), + ('secure', '0'), + ('adb', '0/'), + ('v', media_id), + ('token', ''), + ('gt', '')) + params = urllib.urlencode(params) + rurl = "https://hqq.tv/player/get_md5.php?" + params + str_url = self.net.http_HEAD(rurl, headers=headers).get_url() + return str_url+'.mp4.m3u8' + helpers.append_headers(headers) + def get_url(self, host, media_id): return self._default_get_url(host, media_id, - template='http://hqq.watch/player/embed_player.php?vid={media_id}&autoplay=no') + template='http://hqq.tv/player/embed_player.php?vid={media_id}&autoplay=no') + From 24105a3634e3dca6e5a2602eefd42b09cad8d0d9 Mon Sep 17 00:00:00 2001 From: gujal Date: Thu, 19 Mar 2020 23:48:13 +1300 Subject: [PATCH 006/234] Fix mystream.to (#298) --- lib/resolveurl/plugins/mystream.py | 51 +++--------------------------- 1 file changed, 4 insertions(+), 47 deletions(-) diff --git a/lib/resolveurl/plugins/mystream.py b/lib/resolveurl/plugins/mystream.py index d8a635a8..1a4bf2d8 100644 --- a/lib/resolveurl/plugins/mystream.py +++ b/lib/resolveurl/plugins/mystream.py @@ -26,7 +26,7 @@ class MystreamResolver(ResolveUrl): name = "mystream" - domains = ['mystream.la', 'mystream.to'] + domains = ['mystream.la', 'mystream.to', 'mstream.xyz', 'mstream.cloud'] pattern = r'(?://|\.)(my?stream\.(?:la|to|cloud|xyz))/(?:external|watch/)?([0-9a-zA-Z_]+)' def __init__(self): @@ -40,55 +40,12 @@ def get_media_url(self, host, media_id): if "unable to find the video" in html: raise ResolverError('The requested video was not found or may have been removed.') - match = re.search(r'type="file".+?script>\s*([^<]+)', html, re.DOTALL) + match = re.search(r'og:image".+?"(.+)/', html) if match: - aa_text = aadecode.decode(match.group(1).encode('utf8')) - match = re.search(r"atob\('([^']+)", aa_text) - if match: - etext = match.group(1).decode("base64") - secret = self.vv_secret(html) - stream_url = self.vv_decrypt(secret, etext) - return stream_url + helpers.append_headers(headers) + stream_url = '{0}.mp4'.format(match.group(1)) + return stream_url + helpers.append_headers(headers) raise ResolverError('Video Link Not Found') def get_url(self, host, media_id): return self._default_get_url(host, media_id, 'https://embed.mystream.to/{media_id}') - - def vv_secret(self, page): - jjtext = re.search("', html, re.DOTALL | re.I): + for match in re.finditer(r'(eval\s*\(function.*?)', html, re.DOTALL | re.I): try: js_data = jsunpack.unpack(match.group(1)) js_data = js_data.replace('\\', '') packed_data += js_data except: pass - + return packed_data + def sort_sources_list(sources): if len(sources) > 1: try: - sources.sort(key=lambda x: int(re.sub("\D", "", x[0])), reverse=True) + sources.sort(key=lambda x: int(re.sub(r"\D", "", x[0])), reverse=True) except: - common.logger.log_debug('Scrape sources sort failed |int(re.sub("\D", "", x[0])|') + common.logger.log_debug(r'Scrape sources sort failed |int(re.sub("\D", "", x[0])|') try: sources.sort(key=lambda x: re.sub("[^a-zA-Z]", "", x[0].lower())) except: common.logger.log_debug('Scrape sources sort failed |re.sub("[^a-zA-Z]", "", x[0].lower())|') return sources + def parse_sources_list(html): sources = [] - match = re.search('''['"]?sources['"]?\s*:\s*\[(.*?)\]''', html, re.DOTALL) - if match: - sources = [(match[1], match[0].replace('\/', '/')) for match in re.findall('''['"]?file['"]?\s*:\s*['"]([^'"]+)['"][^}]*['"]?label['"]?\s*:\s*['"]([^'"]*)''', match.group(1), re.DOTALL)] + r = re.search(r'''['"]?sources['"]?\s*:\s*\[(.*?)\]''', html, re.DOTALL) + if r: + sources = [(match[1], match[0].replace(r'\/', '/')) for match in re.findall(r'''['"]?file['"]?\s*:\s*['"]([^'"]+)['"][^}]*['"]?label['"]?\s*:\s*['"]([^'"]*)''', r.group(1), re.DOTALL)] return sources + def parse_html5_source_list(html): - label_attrib = 'type' if not re.search('''''', html) else 'data-res' - sources = [(match[1], match[0].replace('\/', '/')) for match in re.findall('''''', html) else 'data-res' + sources = [(match[1], match[0].replace(r'\/', '/')) for match in re.findall(r''' 1) and (i.group(2) is not None): label = i.group(2) sources += [(label, '%s playpath=%s' % (base, i.group(1)))] return sources + def scrape_sources(html, result_blacklist=None, scheme='http', patterns=None, generic_patterns=True): - if patterns is None: patterns = [] - + if patterns is None: + patterns = [] + def __parse_to_list(_html, regex): _blacklist = ['.jpg', '.jpeg', '.gif', '.png', '.js', '.css', '.htm', '.html', '.php', '.srt', '.sub', '.xml', '.swf', '.vtt', '.mpd'] _blacklist = set(_blacklist + result_blacklist) @@ -130,18 +140,20 @@ def __parse_to_list(_html, regex): for r in re.finditer(regex, _html, re.DOTALL): match = r.groupdict() stream_url = match['url'].replace('&', '&') - file_name = urlparse(stream_url[:-1]).path.split('/')[-1] if stream_url.endswith("/") else urlparse(stream_url).path.split('/')[-1] + file_name = urllib_parse.urlparse(stream_url[:-1]).path.split('/')[-1] if stream_url.endswith("/") else urllib_parse.urlparse(stream_url).path.split('/')[-1] blocked = not file_name or any(item in file_name.lower() for item in _blacklist) - if stream_url.startswith('//'): stream_url = scheme + ':' + stream_url + if stream_url.startswith('//'): + stream_url = scheme + ':' + stream_url if '://' not in stream_url or blocked or (stream_url in streams) or any(stream_url == t[1] for t in source_list): continue - + label = match.get('label', file_name) - if label is None: label = file_name + if label is None: + label = file_name labels.append(label) streams.append(stream_url) - - matches = zip(labels, streams) + + matches = zip(labels, streams) if six.PY2 else list(zip(labels, streams)) if matches: common.logger.log_debug('Scrape sources |%s| found |%s|' % (regex, matches)) return matches @@ -150,23 +162,23 @@ def __parse_to_list(_html, regex): result_blacklist = [] elif isinstance(result_blacklist, str): result_blacklist = [result_blacklist] - - html = html.replace("\/", "/") + + html = html.replace(r"\/", "/") html += get_packed_data(html) source_list = [] if generic_patterns or not patterns: - source_list += __parse_to_list(html, '''["']?label\s*["']?\s*[:=]\s*["']?(?P