From 31122774ad887b28ee2517d1c89b1c62c18d2671 Mon Sep 17 00:00:00 2001 From: jan Date: Wed, 28 Aug 2019 22:00:45 +0200 Subject: [PATCH 1/9] support of RX-577 and hierarchical playlists --- README.md | 21 ++- examples/stations.yml.example | 103 +++++++++++--- ycast.py | 251 +++++++++++++++++++++++++--------- 3 files changed, 285 insertions(+), 90 deletions(-) diff --git a/README.md b/README.md index 40c03ba..0bee6e3 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,13 @@ YCast is a self hosted replacement for the vTuner internet radio service which some Yamaha AVRs use. -It was developed for and tested with the __RX-Vx73__ series. - +It was initially developed for and tested with the __RX-Vx73__ series. It _should_ also work for the following Yamaha AVR models: * RX-Vx75 * RX-Vx77 * RX-Vx79 * RX-Vx81 +This version has been adapted and tested with the __RX-V577__ series, but not retested for the above models YCast is for you if: * You do not want to use a proprietary streaming service @@ -18,7 +18,7 @@ YCast is for you if: * You are unsure about the continuation of the service from Yamaha/vTuner ## Dependencies: -Python version: `3` +Python version: `3.6` Python packages: * `PyYAML` @@ -31,7 +31,7 @@ itself gets handled by the AVR directly, i.e. you can run it on a low-spec RISC * Create your initial `stations.yml`. The config follows a basic YAML structure (see below) * Create a manual entry in your DNS server (read 'Router' for most home users) for: - `radioyamaha.vtuner.com` + `radioyamaha.vtuner.com` and/or `radioyamaha2.vtuner.com` to point to the local machine running YCast. @@ -42,7 +42,13 @@ itself gets handled by the AVR directly, i.e. you can run it on a low-spec RISC Category one name: First awesome station name: first.awesome/station/URL Second awesome station name: second.awesome/station/URL - + Subcategory one name: + First station in subcategory one: first.sub/station/URL + Second station in subcategory one: second.sub/station/URL + Subcategory two name: + First station in subcategory two: first.sub2/station/URL + Second station in subcategory two: second.sub2/station/URL + ... Category two name: Third awesome station name: third.awesome/station/URL Fourth awesome station name: fourth.awesome/station/URL @@ -50,6 +56,7 @@ Category two name: You can also have a look at the provided [example](examples/stations.yml.example) to better understand the configuration. +The location of the stations.yml file can be specified on the command line using `-s filepath` ## Web server configuration @@ -83,3 +90,7 @@ YCast was a quick and dirty project to lay the foundation for having a self host It is a barebone service at the moment. It provides your AVR with the basic info it needs to play internet radio stations. Maybe this will change in the future, maybe not. For now just station names and URLs; no web-based management interface, no coverart, no cute kittens, no fancy stuff. + +The RX-577 shows only the first 8 entries of the top categories from stations.yml (no such limit for the subcategories) +The initial response should probably contain a tag with the amount of entries. +To know what tag,someone should tcpdump the full http conversation with the real server. diff --git a/examples/stations.yml.example b/examples/stations.yml.example index ee100e7..623e8e3 100644 --- a/examples/stations.yml.example +++ b/examples/stations.yml.example @@ -1,22 +1,81 @@ -Electronic: - Deep House Lounge: http://198.15.94.34:8006 - Ibiza Sonica: http://s1.sonicabroadcast.com:7005/stream - Bassdrive: http://50.7.98.106:8200 - SomaFM Fluid: http://ice1.somafm.com/fluid-128-mp3 - -Chillout: - Joint Radio: http://radio.jointil.net:9998 - SomaFM DEF CON Radio: http://ice1.somafm.com/defcon-256-mp3 - SomaFM Drone Zone: http://ice1.somafm.com/dronezone-256-mp3 - SomaFM Mission Control: http://ice1.somafm.com/missioncontrol-128-mp3 - The Jazz Groove: http://west-mp3-128.streamthejazzgroove.com - Radionomy Downbeat: http://streaming.radionomy.com/TempoOfTheDownbeat1 - -Casual: - 76Radio: http://192.240.102.133:9566/stream - SomaFM Beat Blender: http://ice1.somafm.com/beatblender-128-mp3 - Jazz Radio Electro Swing: http://jazz-wr04.ice.infomaniak.ch/jazz-wr04-128.mp3 - SomaFM Groove Salad: http://ice1.somafm.com/groovesalad-256-mp3 - SomaFM Lush: http://ice1.somafm.com/lush-128-mp3 - Allzic Radio R&B: http://allzic10.ice.infomaniak.ch/allzic10.mp3 - The UK 1940s Radio Station: http://91.121.134.23:8100/1 +AAA_Bookmarks: + Radio Paradise: http://stream.radioparadise.com/mp3-192 + +By Location: + Belgie: + VRT Radio 1: http://icecast.vrtcdn.be/radio1-high.mp3 + VRT Radio 2: + VRT Radio 2 Antwerpen: http://icecast.vrtcdn.be/ra2ant-high.mp3 + VRT Radio 2 Limburg: http://icecast.vrtcdn.be/ra2lim-high.mp3 + VRT Radio 2 Oost-Vl: http://icecast.vrtcdn.be/ra2ovl-high.mp3 + VRT Radio 2 Vl-Brabant: http://icecast.vrtcdn.be/ra2vlb-high.mp3 + VRT Radio 2 West-Vl: http://icecast.vrtcdn.be/ra2wvl-high.mp3 + VRT Klara: http://icecast.vrtcdn.be/klara-high.mp3 + VRT Studio Brussel: http://icecast.vrtcdn.be/stubru-high.mp3 + VRT StuBru De Tijdloze: http://icecast.vrtcdn.be/stubru_tijdloze-high.mp3 + VRT MNM: http://icecast.vrtcdn.be/mnm-high.mp3 + VRT MNM UrbaNice: http://icecast.vrtcdn.be/mnm_urb-high.mp3 + VRT MNM Hits: http://icecast.vrtcdn.be/mnm_hits-high.mp3 + VRT Ketnet Hits: http://icecast.vrtcdn.be/ketnetradio-high.mp3 + Radio Nostalgie: http://nostalgiewhatafeeling.ice.infomaniak.ch/nostalgiewhatafeeling-128.mp3 + Joe fm: http://icecast-qmusic.cdp.triple-it.nl/JOEfm_be_live_128.mp3 + Q-music: http://icecast-qmusic.cdp.triple-it.nl/Qmusic_be_live_64.aac + Q-music 2: http://21283.live.streamtheworld.com/QMUSIC.mp3 + RTB Classic 21: https://radios.rtbf.be/classic21-128.mp3 + + +News + Talk: + VRTNWS: http://progressive-audio.lwc.vrtcdn.be/content/fixed/11_11niws-snip_hi.mp3 + RTBF La Première: https://radios.rtbf.be/laprem1ere-128.mp3 + +By Genre: + Rock: + Nostalgie.be Rock Classics: http://streamingp.shoutcast.com/NostalgieRockClassics + Rock FM: http://streams.movemedia.eu:8440/;stream + + Easy Listening: + Positivity Radio: http://ample-07.radiojar.com/d90sb07byuquv + Nostalgie Pop: http://streamingp.shoutcast.com/NostalgieLove + + Classical: + Klara: http://icecast.vrtcdn.be/klara-high.mp3 + VRT Klara Continuo: http://icecast.vrtcdn.be/klaracontinuo-high.mp3 + La Classica: http://stream.laclassica.be:8023/stream + RTBF Musiq 3: https://radios.rtbf.be/musiq3-128.mp3 + + Jazz: + Jazz Crooze: http://streams.crooze.fm:8002/listen.pls?sid=1 + Nostalgie Jazz: http://streamingp.shoutcast.com/NostalgieJazz + +By Language: + French: + Nostalgie.be Chansons Françaises: http://streamingp.shoutcast.com/NostalgieChansonsFrancaises + +Various: + Electronic: + Deep House Lounge: http://198.15.94.34:8006 + Ibiza Sonica: http://s1.sonicabroadcast.com:7005/stream + Bassdrive: http://50.7.98.106:8200 + SomaFM Fluid: http://ice1.somafm.com/fluid-128-mp3 + + Chillout: + Joint Radio: http://radio.jointil.net:9998 + SomaFM DEF CON Radio: http://ice1.somafm.com/defcon-256-mp3 + SomaFM Drone Zone: http://ice1.somafm.com/dronezone-256-mp3 + SomaFM Mission Control: http://ice1.somafm.com/missioncontrol-128-mp3 + The Jazz Groove: http://west-mp3-128.streamthejazzgroove.com + Radionomy Downbeat: http://streaming.radionomy.com/TempoOfTheDownbeat1 + + Casual: + 76Radio: http://192.240.102.133:9566/stream + SomaFM Beat Blender: http://ice1.somafm.com/beatblender-128-mp3 + Jazz Radio Electro Swing: http://jazz-wr04.ice.infomaniak.ch/jazz-wr04-128.mp3 + SomaFM Groove Salad: http://ice1.somafm.com/groovesalad-256-mp3 + SomaFM Lush: http://ice1.somafm.com/lush-128-mp3 + Allzic Radio R&B: http://allzic10.ice.infomaniak.ch/allzic10.mp3 + The UK 1940s Radio Station: http://91.121.134.23:8100/1 + + +Xmas: + Kerstmuziek radio: https://20103.live.streamtheworld.com/SRGSTR08.mp3 + diff --git a/ycast.py b/ycast.py index 50b42e5..d4a25b1 100755 --- a/ycast.py +++ b/ycast.py @@ -4,110 +4,235 @@ import sys import argparse from http.server import BaseHTTPRequestHandler, HTTPServer +import urllib.parse as parse import xml.etree.cElementTree as etree - +import logging +import logging.handlers import yaml VTUNER_DNS = 'http://radioyamaha.vtuner.com' VTUNER_INITURL = '/setupapp/Yamaha/asp/BrowseXML/loginXML.asp' +VTUNER_STATURL = '/setupapp/Yamaha/asp/BrowseXML/statxml.asp' XMLHEADER = '' YCAST_LOCATION = 'ycast' +DEFAULTSTATION = 'Radio Paradise - auto:http://stream.radioparadise.com/mp3-192' + +logger = logging.getLogger(__name__) +logger.addHandler(logging.StreamHandler()) +facility = logging.handlers.SysLogHandler.LOG_LOCAL0 +logger.addHandler(logging.handlers.SysLogHandler('/dev/log', facility)) +logger.setLevel(logging.INFO) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + +class StationSource(): + def __init__(self, source): + self.stations = {} + self.stations_by_id = {} + if os.path.isfile(source): + self.source = source + else: + ycast_dir = os.path.dirname(os.path.realpath(__file__)) + self.source = ycast_dir + '/stations.yml' + + def get_stations(self): + try: + with open(self.source, 'r') as sourcefile: + self.stations = yaml.load(sourcefile, Loader=yaml.FullLoader) + except FileNotFoundError: + logger.error("ERROR: Station configuration not found. Please supply a proper stations.yml.") + sys.exit(1) + self.set_station_by_id() + return self.stations + + + def set_station_by_id(self, station_id=1, long_category=None): + ''' Associate each station with a unique id ''' + def walktree(directory, station_id=1, category=None): + for key, data in directory.items(): + if isinstance(data, dict): + station_id = walktree(data, station_id) + elif isinstance(data, str): + directory[key] = (station_id, data) + self.stations_by_id[station_id] = (key, data) + station_id += 1 + return station_id + walktree(self.stations) + + + def by_hierarchy(self, long_category): + ''' Return a dictionary of stations/dirs based on a long category name + a long category is a string with the hierarchy of categories: 'category|subcategory|subcategor|..' + ''' + hierarchy = long_category.split('|') + current_dir = self.stations + for category in hierarchy: + current_dir = current_dir[category] + return current_dir + + def by_id(self, station_id): + return self.stations_by_id[station_id] + + +class YCastHandler(BaseHTTPRequestHandler): + ''' YCastServer creates an instance of this class for each received message. __init__ passes the message to do_GET ''' -stations = {} - - -def get_stations(): - global stations - ycast_dir = os.path.dirname(os.path.realpath(__file__)) - try: - with open(ycast_dir + '/stations.yml', 'r') as f: - stations = yaml.load(f) - except FileNotFoundError: - print("ERROR: Station configuration not found. Please supply a proper stations.yml.") - sys.exit(1) - - -def text_to_url(text): - return text.replace(' ', '%20') - - -def url_to_text(url): - return url.replace('%20', ' ') - - -class YCastServer(BaseHTTPRequestHandler): def do_GET(self): - get_stations() - if self.path.startswith(VTUNER_INITURL + "?token="): - self.send_response(200) - self.send_header('Content-type', 'text/html') - self.end_headers() - self.wfile.write(bytes(XMLHEADER, 'utf-8')) - # return arbitrary token - self.wfile.write( - bytes("aaaaaaaaaaaaaaaa", 'utf-8') - ) + ''' Handle the GET request and send reply to the client ''' + stations = self.server.source.get_stations() + url_split = parse.urlsplit(self.path) + url_query_split = parse.parse_qs(url_split.query) + if url_split.path == VTUNER_INITURL: + if 'token' in url_query_split: + # First request on start of the Amplifier + xml = etree.Element('EncryptedToken') + xml.text = '85d6fa40a9dcc906' # any arbitrarytoken + self.write_message(xml, add_xml_header=False) + else: + # A root directory request + start = int(url_query_split['start'][0]) + size = int(url_query_split['howmany'][0]) + self.reply_with_dir(stations, start - 1, size) + + elif self.path.startswith(VTUNER_STATURL): + station_id = int(url_query_split['id'][0]) + try: + station_name, station_url = self.server.source.by_id(station_id) + except KeyError: + station_id = 999999 + station_name, _, station_url = DEFAULTSTATION.partition('&') + xml = self.create_root() + self.add_station(xml, station_name, station_url, station_id) + self.write_message(xml) + elif self.path == '/' \ or self.path == '/' + YCAST_LOCATION \ or self.path == '/' + YCAST_LOCATION + '/'\ or self.path.startswith(VTUNER_INITURL): - self.send_response(200) - self.send_header('Content-type', 'text/html') - self.end_headers() - self.wfile.write(bytes(XMLHEADER, 'utf-8')) - xml = self.create_root() - for category in sorted(stations, key=str.lower): - self.add_dir(xml, category, - VTUNER_DNS + '/' + YCAST_LOCATION + '/' + text_to_url(category)) - self.wfile.write(bytes(etree.tostring(xml).decode('utf-8'), 'utf-8')) - elif self.path.startswith('/' + YCAST_LOCATION + '/'): - category = url_to_text(self.path[len(YCAST_LOCATION) + 2:].partition('?')[0]) - if category not in stations: + self.reply_with_dir(stations) + + elif self.path.startswith('/' + YCAST_LOCATION + '?'): + hierarchy = parse.unquote(url_query_split['category'][0]) + try: + start = int(url_query_split['start'][0]) + size = int(url_query_split['howmany'][0]) + except KeyError: + start = 1 + size = 8 + try: + self.reply_with_mixed_list(hierarchy, start - 1, size) + except KeyError: self.send_error(404) - return - xml = self.create_root() - for station in sorted(stations[category], key=str.lower): - self.add_station(xml, station, stations[category][station]) - self.send_response(200) - self.send_header('Content-type', 'text/html') - self.end_headers() - self.wfile.write(bytes(XMLHEADER, 'utf-8')) - self.wfile.write(bytes(etree.tostring(xml).decode('utf-8'), 'utf-8')) else: self.send_error(404) + + def reply_with_dir(self, stations, start=0, max_size=8): + ''' Build an xml reply that represents a list of all directories ''' + xml = self.create_root() + count = etree.SubElement(xml,'DirCount').text = '9' + for category in sorted(stations, key=str.lower)[start:start+max_size]: + self.add_dir(xml, category, + VTUNER_DNS + '/' + YCAST_LOCATION + '?category=' + parse.quote(category), + str(len(stations[category]))) + self.write_message(xml) + + + def reply_with_station_list(self, station_list, start=0, max_size=8): + ''' Build an xml reply that represents a list of all stations ''' + xml = self.create_root() + for station in sorted(station_list, key=str.lower)[start:start+max_size]: + station_id, station_url = station_list[station] + self.add_station(xml, station, station_url, station_id) + self.write_message(xml) + + def reply_with_mixed_list(self, hierarchy, start=0, max_size=8): + ''' Build an xml reply that represents a list of mixed stations/directories ''' + station_list = self.server.source.by_hierarchy(hierarchy) + xml = self.create_root() + for item in sorted(station_list, key=str.lower)[start:start+max_size]: + if isinstance(station_list[item], dict): + category = hierarchy + '|' + item + self.add_dir(xml, item, + VTUNER_DNS + '/' + YCAST_LOCATION + '?category=' + parse.quote(category), + str(len(station_list[item]))) + elif isinstance(station_list[item], tuple): + station = item + station_id, station_url = station_list[station] + self.add_station(xml, station, station_url, station_id) + self.write_message(xml) + + + def write_message(self, xml, add_xml_header=True): + ''' Write a message containing the given xml to send to the ycast client ''' + self.send_response(200) + self.send_header('Content-type', 'text/html') + self.end_headers() + if add_xml_header: + self.wfile.write(bytes(XMLHEADER, 'utf-8')) + self.wfile.write(bytes(etree.tostring(xml).decode('utf-8'), 'utf-8')) + + def create_root(self): + ''' Create the root of an xml tree ''' return etree.Element('ListOfItems') - def add_dir(self, root, name, dest): + + def add_dir(self, root, name, dest, dircount): + ''' Add a directory entry to the xml node + root: the node to add the directory information to + name: the name of the directory + dest: the url to visit to get the contents of the directory + dircount: the number of items in the directory (allows the client to say it is showing page 1 of 3) + ''' item = etree.SubElement(root, 'Item') etree.SubElement(item, 'ItemType').text = 'Dir' etree.SubElement(item, 'Title').text = name etree.SubElement(item, 'UrlDir').text = dest + etree.SubElement(item, 'DirCount').text = dircount return item - def add_station(self, root, name, url): + + def add_station(self, root, name, url, station_id): + ''' Add a station entry to the xml node + root: the node to add the directory information to + name: the name of the station + url: the url to visit to listen to the station + station_id: the unique id of the station (that will be sent to the server if the client wants it) + ''' item = etree.SubElement(root, 'Item') etree.SubElement(item, 'ItemType').text = 'Station' etree.SubElement(item, 'StationName').text = name + etree.SubElement(item, 'StationId').text = str(station_id) etree.SubElement(item, 'StationUrl').text = url return item -get_stations() +class YCastServer(HTTPServer): + ''' A HTTPServer that retains a pointer to the source to be used by the BaseHTTPRequestHandler + ''' + def __init__(self, source, *args, **kwargs): + self.source = StationSource(source) + super().__init__(*args, **kwargs) + + parser = argparse.ArgumentParser(description='vTuner API emulation') parser.add_argument('-l', action='store', dest='address', help='Listen address', default='0.0.0.0') parser.add_argument('-p', action='store', dest='port', type=int, help='Listen port', default=80) +parser.add_argument('-s', action='store', dest='station_list', type=str, help='station list file', default='stations.yml') arguments = parser.parse_args() try: - server = HTTPServer((arguments.address, arguments.port), YCastServer) + server = YCastServer(arguments.station_list, (arguments.address, arguments.port), YCastHandler) +except OSError as err: + logger.error(f'OS reports: \"{err.strerror}\"') + sys.exit(2) except PermissionError: - print("ERROR: No permission to create socket. Are you trying to use ports below 1024 without elevated rights?") + logger.error("No permission to create socket. Are you trying to use ports below 1024 without elevated rights?") sys.exit(1) -print('YCast server listening on %s:%s' % (arguments.address, arguments.port)) +logger.info('YCast server listening on {arguments.address}:{arguments.port}') try: server.serve_forever() except KeyboardInterrupt: pass -print('YCast server shutting down') -server.server_close() +finally: + logger.info('YCast server shutting down') + server.server_close() From c1f81dbfca66d98e70a278577ca6119ff376d18f Mon Sep 17 00:00:00 2001 From: jan Date: Sun, 13 Oct 2019 12:06:39 +0200 Subject: [PATCH 2/9] add gentoo openrc service files\nupdate stations examples file --- examples/stations.yml.example | 8 +++++++- gentoo/etc/conf.d/ycast/ycast | 14 ++++++++++++++ gentoo/etc/init.d/ycast/ycast | 14 ++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 gentoo/etc/conf.d/ycast/ycast create mode 100755 gentoo/etc/init.d/ycast/ycast diff --git a/examples/stations.yml.example b/examples/stations.yml.example index 623e8e3..3f3bebf 100644 --- a/examples/stations.yml.example +++ b/examples/stations.yml.example @@ -1,8 +1,11 @@ AAA_Bookmarks: Radio Paradise: http://stream.radioparadise.com/mp3-192 - + RP Mellow: http://icy-7.radioparadise.com/mellow-192 + Willy: https://20723.live.streamtheworld.com/WILLY.mp3 + Zenith Rock: http://148.163.81.10:8006/stream By Location: Belgie: + Willy: https://20723.live.streamtheworld.com/WILLY.mp3 VRT Radio 1: http://icecast.vrtcdn.be/radio1-high.mp3 VRT Radio 2: VRT Radio 2 Antwerpen: http://icecast.vrtcdn.be/ra2ant-high.mp3 @@ -30,12 +33,15 @@ News + Talk: By Genre: Rock: + Willy: https://20723.live.streamtheworld.com/WILLY.mp3 Nostalgie.be Rock Classics: http://streamingp.shoutcast.com/NostalgieRockClassics Rock FM: http://streams.movemedia.eu:8440/;stream + Zenith Rock: http://148.163.81.10:8006/stream Easy Listening: Positivity Radio: http://ample-07.radiojar.com/d90sb07byuquv Nostalgie Pop: http://streamingp.shoutcast.com/NostalgieLove + RP Mellow: http://icy-7.radioparadise.com/mellow-192 Classical: Klara: http://icecast.vrtcdn.be/klara-high.mp3 diff --git a/gentoo/etc/conf.d/ycast/ycast b/gentoo/etc/conf.d/ycast/ycast new file mode 100644 index 0000000..43cbd34 --- /dev/null +++ b/gentoo/etc/conf.d/ycast/ycast @@ -0,0 +1,14 @@ +# /etc/conf.d/ycast + +supervisor="supervise-daemon" +#rc_need="nginx" # when running on a port different than 80, I use nginx on this PC as a reverse proxy + +# ycast should listen on an IP address that is provided by dnsmasq (from name ycast). +# It should be different from the addresses where e.g. nginx listens for port 80 (so no 'listen 80' in nginx.conf, but 192.168.4.1:80) +# no other server should be listening on this IP address & port 80 +# I do this because simply using proxy_pass in nginx doesn't work. It looks like Yamaha doesn't like the proxied reply +# check /etc/dnsmasq.d for the matching configuration + +YCAST_ARGS="-l ycast -p 80" +YCAST_ARGS="$YCAST_ARGS -s /vetc/resources/net-radio/stations.yml" + diff --git a/gentoo/etc/init.d/ycast/ycast b/gentoo/etc/init.d/ycast/ycast new file mode 100755 index 0000000..5d79179 --- /dev/null +++ b/gentoo/etc/init.d/ycast/ycast @@ -0,0 +1,14 @@ +#!/sbin/openrc-run +# Copyright 1999-2016 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +#extra_started_commands="reload" +command="/usr/local/bin/ycast.py" +command_args="${YCAST_ARGS}" +description="Fake vtuner Radio Directory Service" + +depend() { + need net + use localmount +} + From 02c02ec3f5e3e38e38be0d90db6b248b1c875cda Mon Sep 17 00:00:00 2001 From: Jan Date: Sat, 4 Apr 2020 23:13:34 +0200 Subject: [PATCH 3/9] add verbosity; filter streamtheworld.com readirects --- ycast.py | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/ycast.py b/ycast.py index d4a25b1..1eb5649 100755 --- a/ycast.py +++ b/ycast.py @@ -24,6 +24,16 @@ logger.setLevel(logging.INFO) formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +def filter_url(url): + ''' Check the url and translate it if needed into a direct url ''' + if 'streamtheworld.com/' in url: + http = urllib3.PoolManager() + resp = http.urlopen('GET', url, redirect=False) + newurl = resp.get_redirect_location() + return newurl + return url + + class StationSource(): def __init__(self, source): self.stations = {} @@ -203,7 +213,7 @@ def add_station(self, root, name, url, station_id): etree.SubElement(item, 'ItemType').text = 'Station' etree.SubElement(item, 'StationName').text = name etree.SubElement(item, 'StationId').text = str(station_id) - etree.SubElement(item, 'StationUrl').text = url + etree.SubElement(item, 'StationUrl').text = filter_url(url) return item @@ -212,27 +222,36 @@ class YCastServer(HTTPServer): ''' def __init__(self, source, *args, **kwargs): self.source = StationSource(source) + address,port = args[0] + logger.info(f'YCast server listening on {address}:{port}') super().__init__(*args, **kwargs) + def __enter__(self): + print('entering') + return self + + + def __exit__(self, *args): + print('exiting') + logger.info('YCast server shutting down') + self.server_close() + + parser = argparse.ArgumentParser(description='vTuner API emulation') parser.add_argument('-l', action='store', dest='address', help='Listen address', default='0.0.0.0') parser.add_argument('-p', action='store', dest='port', type=int, help='Listen port', default=80) parser.add_argument('-s', action='store', dest='station_list', type=str, help='station list file', default='stations.yml') arguments = parser.parse_args() try: - server = YCastServer(arguments.station_list, (arguments.address, arguments.port), YCastHandler) + with YCastServer(arguments.station_list, (arguments.address, arguments.port), YCastHandler) as server: + print('listening', server) + server.serve_forever() except OSError as err: logger.error(f'OS reports: \"{err.strerror}\"') sys.exit(2) except PermissionError: logger.error("No permission to create socket. Are you trying to use ports below 1024 without elevated rights?") sys.exit(1) -logger.info('YCast server listening on {arguments.address}:{arguments.port}') -try: - server.serve_forever() except KeyboardInterrupt: pass -finally: - logger.info('YCast server shutting down') - server.server_close() From 6b0e432e823be0ff2965fcd661571aeedc588586 Mon Sep 17 00:00:00 2001 From: jan Date: Sat, 13 Jun 2020 23:06:12 +0200 Subject: [PATCH 4/9] missing import, add some comments and logging --- ycast.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/ycast.py b/ycast.py index 1eb5649..1bbb017 100755 --- a/ycast.py +++ b/ycast.py @@ -9,6 +9,7 @@ import logging import logging.handlers import yaml +import urllib3 VTUNER_DNS = 'http://radioyamaha.vtuner.com' VTUNER_INITURL = '/setupapp/Yamaha/asp/BrowseXML/loginXML.asp' @@ -88,6 +89,7 @@ class YCastHandler(BaseHTTPRequestHandler): def do_GET(self): ''' Handle the GET request and send reply to the client ''' + logger.info(f'message received: {self.path}') stations = self.server.source.get_stations() url_split = parse.urlsplit(self.path) url_query_split = parse.parse_qs(url_split.query) @@ -137,7 +139,11 @@ def do_GET(self): def reply_with_dir(self, stations, start=0, max_size=8): - ''' Build an xml reply that represents a list of all directories ''' + ''' Build an xml reply that represents a list of all directories + stations: the list of items to display + start: the first element of the list to display + max_size: the max number of elements to display + ''' xml = self.create_root() count = etree.SubElement(xml,'DirCount').text = '9' for category in sorted(stations, key=str.lower)[start:start+max_size]: @@ -148,7 +154,11 @@ def reply_with_dir(self, stations, start=0, max_size=8): def reply_with_station_list(self, station_list, start=0, max_size=8): - ''' Build an xml reply that represents a list of all stations ''' + ''' Build an xml reply that represents a list of all stations + station_list: the list of items to display + start: the first element of the list to display + max_size: the max number of elements to display + ''' xml = self.create_root() for station in sorted(station_list, key=str.lower)[start:start+max_size]: station_id, station_url = station_list[station] @@ -156,7 +166,11 @@ def reply_with_station_list(self, station_list, start=0, max_size=8): self.write_message(xml) def reply_with_mixed_list(self, hierarchy, start=0, max_size=8): - ''' Build an xml reply that represents a list of mixed stations/directories ''' + ''' Build an xml reply that represents a list of mixed stations/directories + hierarchy: the list of items to display + start: the first element of the list to display + max_size: the max number of elements to display + ''' station_list = self.server.source.by_hierarchy(hierarchy) xml = self.create_root() for item in sorted(station_list, key=str.lower)[start:start+max_size]: @@ -179,7 +193,9 @@ def write_message(self, xml, add_xml_header=True): self.end_headers() if add_xml_header: self.wfile.write(bytes(XMLHEADER, 'utf-8')) - self.wfile.write(bytes(etree.tostring(xml).decode('utf-8'), 'utf-8')) + reply=etree.tostring(xml).decode('utf-8') + logger.info(f'Sending reply: {reply}') + self.wfile.write(bytes(reply, 'utf-8')) def create_root(self): @@ -228,12 +244,10 @@ def __init__(self, source, *args, **kwargs): def __enter__(self): - print('entering') return self def __exit__(self, *args): - print('exiting') logger.info('YCast server shutting down') self.server_close() From 749c0dac445487c3ac62950b6530de6129b22d0e Mon Sep 17 00:00:00 2001 From: jan Date: Mon, 17 Aug 2020 02:23:24 +0200 Subject: [PATCH 5/9] only redirect a streamtheworld url if we can assume it is not a stream.\nOtherwise, the GET needed to determine the possible url may hang forever reading a stream --- ycast.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/ycast.py b/ycast.py index 1bbb017..7957869 100755 --- a/ycast.py +++ b/ycast.py @@ -27,10 +27,18 @@ def filter_url(url): ''' Check the url and translate it if needed into a direct url ''' - if 'streamtheworld.com/' in url: + # TODO: an incorrect url can keep this urlopen hanging: if the url is a valid stream, get may be getting the stream data + # find out how to detect this (only retrieve first n bytes ?) + # specifically, using https://20043.live.streamtheworld.com/WILLY.mp3?pname=rp_external gave an error + # whereas https://playerservices.streamtheworld.com/api/livestream-redirect/WILLY.mp3 is OK + if 'playerservices.streamtheworld.com/' in url: http = urllib3.PoolManager() - resp = http.urlopen('GET', url, redirect=False) - newurl = resp.get_redirect_location() + try: + resp = http.urlopen('GET', url, redirect=False, retries=False, timeout=2.0) + newurl = resp.get_redirect_location() + except Exception as e: + logger(f'Exception {e} while filtering url {url}') + newurl = url return newurl return url @@ -145,7 +153,7 @@ def reply_with_dir(self, stations, start=0, max_size=8): max_size: the max number of elements to display ''' xml = self.create_root() - count = etree.SubElement(xml,'DirCount').text = '9' + count = etree.SubElement(xml,'DirCount').text = str(len(stations)) for category in sorted(stations, key=str.lower)[start:start+max_size]: self.add_dir(xml, category, VTUNER_DNS + '/' + YCAST_LOCATION + '?category=' + parse.quote(category), @@ -259,7 +267,6 @@ def __exit__(self, *args): arguments = parser.parse_args() try: with YCastServer(arguments.station_list, (arguments.address, arguments.port), YCastHandler) as server: - print('listening', server) server.serve_forever() except OSError as err: logger.error(f'OS reports: \"{err.strerror}\"') From 800b657a9806f9ce1858260beadee7d11dc324bd Mon Sep 17 00:00:00 2001 From: jan Date: Wed, 14 Apr 2021 09:04:00 +0200 Subject: [PATCH 6/9] Change https urls to http refactor logging and parameter handling some new stations in example --- examples/stations.yml.example | 16 +++++++++--- ycast.py | 46 +++++++++++++++++++++++------------ 2 files changed, 43 insertions(+), 19 deletions(-) diff --git a/examples/stations.yml.example b/examples/stations.yml.example index 3f3bebf..939a71a 100644 --- a/examples/stations.yml.example +++ b/examples/stations.yml.example @@ -1,11 +1,11 @@ AAA_Bookmarks: Radio Paradise: http://stream.radioparadise.com/mp3-192 RP Mellow: http://icy-7.radioparadise.com/mellow-192 - Willy: https://20723.live.streamtheworld.com/WILLY.mp3 + Willy: https://playerservices.streamtheworld.com/api/livestream-redirect/WILLY.mp3 Zenith Rock: http://148.163.81.10:8006/stream By Location: Belgie: - Willy: https://20723.live.streamtheworld.com/WILLY.mp3 + Willy: https://playerservices.streamtheworld.com/api/livestream-redirect/WILLY.mp3 VRT Radio 1: http://icecast.vrtcdn.be/radio1-high.mp3 VRT Radio 2: VRT Radio 2 Antwerpen: http://icecast.vrtcdn.be/ra2ant-high.mp3 @@ -33,14 +33,15 @@ News + Talk: By Genre: Rock: - Willy: https://20723.live.streamtheworld.com/WILLY.mp3 Nostalgie.be Rock Classics: http://streamingp.shoutcast.com/NostalgieRockClassics + Power Rock: http://streams.movemedia.eu:8330/;stream/1 Rock FM: http://streams.movemedia.eu:8440/;stream + Willy: https://playerservices.streamtheworld.com/api/livestream-redirect/WILLY.mp3 Zenith Rock: http://148.163.81.10:8006/stream Easy Listening: - Positivity Radio: http://ample-07.radiojar.com/d90sb07byuquv Nostalgie Pop: http://streamingp.shoutcast.com/NostalgieLove + Positivity Radio: http://ample-07.radiojar.com/d90sb07byuquv RP Mellow: http://icy-7.radioparadise.com/mellow-192 Classical: @@ -53,6 +54,12 @@ By Genre: Jazz Crooze: http://streams.crooze.fm:8002/listen.pls?sid=1 Nostalgie Jazz: http://streamingp.shoutcast.com/NostalgieJazz + Folk: + Folk Alley: http://freshgrass.streamguys1.com/folkalley-128mp3 + Fresh Cuts (Folk Alley): http://freshgrass.streamguys1.com/freshcuts-128mp3 + Irish (Folk Alley): http://freshgrass.streamguys1.com/irish-128mp3 + FreshGrass (Folk Alley): http://freshgrass.streamguys1.com/ss1-128mp3 + By Language: French: Nostalgie.be Chansons Françaises: http://streamingp.shoutcast.com/NostalgieChansonsFrancaises @@ -84,4 +91,5 @@ Various: Xmas: Kerstmuziek radio: https://20103.live.streamtheworld.com/SRGSTR08.mp3 + Folk Alley Holiday: http://freshgrass.streamguys1.com/folkalley-holiday-128mp3 diff --git a/ycast.py b/ycast.py index 7957869..90d993f 100755 --- a/ycast.py +++ b/ycast.py @@ -2,12 +2,9 @@ import os import sys -import argparse from http.server import BaseHTTPRequestHandler, HTTPServer import urllib.parse as parse import xml.etree.cElementTree as etree -import logging -import logging.handlers import yaml import urllib3 @@ -18,12 +15,34 @@ YCAST_LOCATION = 'ycast' DEFAULTSTATION = 'Radio Paradise - auto:http://stream.radioparadise.com/mp3-192' -logger = logging.getLogger(__name__) -logger.addHandler(logging.StreamHandler()) -facility = logging.handlers.SysLogHandler.LOG_LOCAL0 -logger.addHandler(logging.handlers.SysLogHandler('/dev/log', facility)) -logger.setLevel(logging.INFO) -formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + +def initialize_logger(): + ''' initialize the default logger ''' + import logging + import logging.handlers + logger = logging.getLogger(__name__) + logger.addHandler(logging.StreamHandler()) + facility = logging.handlers.SysLogHandler.LOG_LOCAL0 + logger.addHandler(logging.handlers.SysLogHandler('/dev/log', facility)) + logger.setLevel(logging.INFO) + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + return logger + + +def parse_command_line(): + ''' parse the command line arguments ''' + import argparse + parser = argparse.ArgumentParser(description='vTuner API emulation') + parser.add_argument('-l', action='store', dest='address', help='Listen address', default='0.0.0.0') + parser.add_argument('-p', action='store', dest='port', type=int, help='Listen port', default=80) + parser.add_argument('-s', action='store', dest='station_list', type=str, help='station list file', default='stations.yml') + return parser.parse_args() + + +def remove_SSL_from_url(url): + ''' change any https:// url to a https:// url ''' + return url.replace("https://", "http://", 1) + def filter_url(url): ''' Check the url and translate it if needed into a direct url ''' @@ -237,7 +256,7 @@ def add_station(self, root, name, url, station_id): etree.SubElement(item, 'ItemType').text = 'Station' etree.SubElement(item, 'StationName').text = name etree.SubElement(item, 'StationId').text = str(station_id) - etree.SubElement(item, 'StationUrl').text = filter_url(url) + etree.SubElement(item, 'StationUrl').text = remove_SSL_from_url(filter_url(url)) return item @@ -260,11 +279,8 @@ def __exit__(self, *args): self.server_close() -parser = argparse.ArgumentParser(description='vTuner API emulation') -parser.add_argument('-l', action='store', dest='address', help='Listen address', default='0.0.0.0') -parser.add_argument('-p', action='store', dest='port', type=int, help='Listen port', default=80) -parser.add_argument('-s', action='store', dest='station_list', type=str, help='station list file', default='stations.yml') -arguments = parser.parse_args() +logger = initialize_logger() +arguments = parse_command_line() try: with YCastServer(arguments.station_list, (arguments.address, arguments.port), YCastHandler) as server: server.serve_forever() From 5b2ff49b5c184f0db2709839a8a65795ece227f0 Mon Sep 17 00:00:00 2001 From: jan Date: Wed, 14 Apr 2021 19:48:01 +0200 Subject: [PATCH 7/9] add command line option --nossl refactor filtering --- ycast.py | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/ycast.py b/ycast.py index 90d993f..c3e92fa 100755 --- a/ycast.py +++ b/ycast.py @@ -36,15 +36,16 @@ def parse_command_line(): parser.add_argument('-l', action='store', dest='address', help='Listen address', default='0.0.0.0') parser.add_argument('-p', action='store', dest='port', type=int, help='Listen port', default=80) parser.add_argument('-s', action='store', dest='station_list', type=str, help='station list file', default='stations.yml') + parser.add_argument('--nossl', action='store_true', dest='nossl', help='only use http connections (and attempt to find an alternative for https links)') return parser.parse_args() -def remove_SSL_from_url(url): - ''' change any https:// url to a https:// url ''' - return url.replace("https://", "http://", 1) +def filter_remove_SSL_from_url(url: str) -> str: + ''' change any https:// url to a http:// url ''' + return url.replace("https://", "http://", 1).replace(":443", "", 1) -def filter_url(url): +def filter_streamtheworld_url(url:str ) -> str: ''' Check the url and translate it if needed into a direct url ''' # TODO: an incorrect url can keep this urlopen hanging: if the url is a valid stream, get may be getting the stream data # find out how to detect this (only retrieve first n bytes ?) @@ -62,6 +63,13 @@ def filter_url(url): return url +def apply_filters(url: str, filters: list) -> str: + ''' apply all filters from the given list to the url ''' + for filter in filters: + url=filter(url) + return url + + class StationSource(): def __init__(self, source): self.stations = {} @@ -133,15 +141,14 @@ def do_GET(self): self.reply_with_dir(stations, start - 1, size) elif self.path.startswith(VTUNER_STATURL): + # a request for a single station to play station_id = int(url_query_split['id'][0]) try: station_name, station_url = self.server.source.by_id(station_id) except KeyError: station_id = 999999 station_name, _, station_url = DEFAULTSTATION.partition('&') - xml = self.create_root() - self.add_station(xml, station_name, station_url, station_id) - self.write_message(xml) + self.reply_with_single_station(station_name, station_url, station_id) elif self.path == '/' \ or self.path == '/' + YCAST_LOCATION \ @@ -165,6 +172,14 @@ def do_GET(self): self.send_error(404) + def reply_with_single_station(self, station_name, station_url, station_id): + ''' build an xml reply with the filtered url of a single station ''' + xml = self.create_root() + station_url = apply_filters(station_url, filters_to_apply) # filters_to_apply is a global + self.add_station(xml, station_name, station_url, station_id) + self.write_message(xml) + + def reply_with_dir(self, stations, start=0, max_size=8): ''' Build an xml reply that represents a list of all directories stations: the list of items to display @@ -256,7 +271,7 @@ def add_station(self, root, name, url, station_id): etree.SubElement(item, 'ItemType').text = 'Station' etree.SubElement(item, 'StationName').text = name etree.SubElement(item, 'StationId').text = str(station_id) - etree.SubElement(item, 'StationUrl').text = remove_SSL_from_url(filter_url(url)) + etree.SubElement(item, 'StationUrl').text = url return item @@ -281,6 +296,10 @@ def __exit__(self, *args): logger = initialize_logger() arguments = parse_command_line() +filters_to_apply = [ filter_streamtheworld_url ] +if arguments.nossl: + filters_to_apply.append(filter_remove_SSL_from_url) + try: with YCastServer(arguments.station_list, (arguments.address, arguments.port), YCastHandler) as server: server.serve_forever() From 2d8b2d7300f268147aadda03fe631417e6cf3df1 Mon Sep 17 00:00:00 2001 From: jan Date: Tue, 15 Mar 2022 18:04:24 +0100 Subject: [PATCH 8/9] Added files generated with pipenv --- Pipfile | 15 +++++ Pipfile.lock | 143 +++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 3 + 3 files changed, 161 insertions(+) create mode 100644 Pipfile create mode 100644 Pipfile.lock create mode 100644 requirements.txt diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..342a9f2 --- /dev/null +++ b/Pipfile @@ -0,0 +1,15 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +pyyaml = "*" +urllib3 = "*" +argparse = "*" +lxml = "*" + +[dev-packages] + +[requires] +python_version = "3.9" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..00736cf --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,143 @@ +{ + "_meta": { + "hash": { + "sha256": "79fd95d6b6460b07f312d8022bbc234e958d8013c713cccaf0d009717fd49a52" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.9" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "argparse": { + "hashes": [ + "sha256:62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4", + "sha256:c31647edb69fd3d465a847ea3157d37bed1f95f19760b11a47aa91c04b666314" + ], + "index": "pypi", + "version": "==1.4.0" + }, + "lxml": { + "hashes": [ + "sha256:078306d19a33920004addeb5f4630781aaeabb6a8d01398045fcde085091a169", + "sha256:0c1978ff1fd81ed9dcbba4f91cf09faf1f8082c9d72eb122e92294716c605428", + "sha256:1010042bfcac2b2dc6098260a2ed022968dbdfaf285fc65a3acf8e4eb1ffd1bc", + "sha256:1d650812b52d98679ed6c6b3b55cbb8fe5a5460a0aef29aeb08dc0b44577df85", + "sha256:20b8a746a026017acf07da39fdb10aa80ad9877046c9182442bf80c84a1c4696", + "sha256:2403a6d6fb61c285969b71f4a3527873fe93fd0abe0832d858a17fe68c8fa507", + "sha256:24f5c5ae618395ed871b3d8ebfcbb36e3f1091fd847bf54c4de623f9107942f3", + "sha256:28d1af847786f68bec57961f31221125c29d6f52d9187c01cd34dc14e2b29430", + "sha256:31499847fc5f73ee17dbe1b8e24c6dafc4e8d5b48803d17d22988976b0171f03", + "sha256:31ba2cbc64516dcdd6c24418daa7abff989ddf3ba6d3ea6f6ce6f2ed6e754ec9", + "sha256:330bff92c26d4aee79c5bc4d9967858bdbe73fdbdbacb5daf623a03a914fe05b", + "sha256:5045ee1ccd45a89c4daec1160217d363fcd23811e26734688007c26f28c9e9e7", + "sha256:52cbf2ff155b19dc4d4100f7442f6a697938bf4493f8d3b0c51d45568d5666b5", + "sha256:530f278849031b0eb12f46cca0e5db01cfe5177ab13bd6878c6e739319bae654", + "sha256:545bd39c9481f2e3f2727c78c169425efbfb3fbba6e7db4f46a80ebb249819ca", + "sha256:5804e04feb4e61babf3911c2a974a5b86f66ee227cc5006230b00ac6d285b3a9", + "sha256:5a58d0b12f5053e270510bf12f753a76aaf3d74c453c00942ed7d2c804ca845c", + "sha256:5f148b0c6133fb928503cfcdfdba395010f997aa44bcf6474fcdd0c5398d9b63", + "sha256:5f7d7d9afc7b293147e2d506a4596641d60181a35279ef3aa5778d0d9d9123fe", + "sha256:60d2f60bd5a2a979df28ab309352cdcf8181bda0cca4529769a945f09aba06f9", + "sha256:6259b511b0f2527e6d55ad87acc1c07b3cbffc3d5e050d7e7bcfa151b8202df9", + "sha256:6268e27873a3d191849204d00d03f65c0e343b3bcb518a6eaae05677c95621d1", + "sha256:627e79894770783c129cc5e89b947e52aa26e8e0557c7e205368a809da4b7939", + "sha256:62f93eac69ec0f4be98d1b96f4d6b964855b8255c345c17ff12c20b93f247b68", + "sha256:6d6483b1229470e1d8835e52e0ff3c6973b9b97b24cd1c116dca90b57a2cc613", + "sha256:6f7b82934c08e28a2d537d870293236b1000d94d0b4583825ab9649aef7ddf63", + "sha256:6fe4ef4402df0250b75ba876c3795510d782def5c1e63890bde02d622570d39e", + "sha256:719544565c2937c21a6f76d520e6e52b726d132815adb3447ccffbe9f44203c4", + "sha256:730766072fd5dcb219dd2b95c4c49752a54f00157f322bc6d71f7d2a31fecd79", + "sha256:74eb65ec61e3c7c019d7169387d1b6ffcfea1b9ec5894d116a9a903636e4a0b1", + "sha256:7993232bd4044392c47779a3c7e8889fea6883be46281d45a81451acfd704d7e", + "sha256:80bbaddf2baab7e6de4bc47405e34948e694a9efe0861c61cdc23aa774fcb141", + "sha256:86545e351e879d0b72b620db6a3b96346921fa87b3d366d6c074e5a9a0b8dadb", + "sha256:891dc8f522d7059ff0024cd3ae79fd224752676447f9c678f2a5c14b84d9a939", + "sha256:8a31f24e2a0b6317f33aafbb2f0895c0bce772980ae60c2c640d82caac49628a", + "sha256:8b99ec73073b37f9ebe8caf399001848fced9c08064effdbfc4da2b5a8d07b93", + "sha256:986b7a96228c9b4942ec420eff37556c5777bfba6758edcb95421e4a614b57f9", + "sha256:a1547ff4b8a833511eeaceacbcd17b043214fcdb385148f9c1bc5556ca9623e2", + "sha256:a2bfc7e2a0601b475477c954bf167dee6d0f55cb167e3f3e7cefad906e7759f6", + "sha256:a3c5f1a719aa11866ffc530d54ad965063a8cbbecae6515acbd5f0fae8f48eaa", + "sha256:a9f1c3489736ff8e1c7652e9dc39f80cff820f23624f23d9eab6e122ac99b150", + "sha256:aa0cf4922da7a3c905d000b35065df6184c0dc1d866dd3b86fd961905bbad2ea", + "sha256:ad4332a532e2d5acb231a2e5d33f943750091ee435daffca3fec0a53224e7e33", + "sha256:b2582b238e1658c4061ebe1b4df53c435190d22457642377fd0cb30685cdfb76", + "sha256:b6fc2e2fb6f532cf48b5fed57567ef286addcef38c28874458a41b7837a57807", + "sha256:b92d40121dcbd74831b690a75533da703750f7041b4bf951befc657c37e5695a", + "sha256:bbab6faf6568484707acc052f4dfc3802bdb0cafe079383fbaa23f1cdae9ecd4", + "sha256:c0b88ed1ae66777a798dc54f627e32d3b81c8009967c63993c450ee4cbcbec15", + "sha256:ce13d6291a5f47c1c8dbd375baa78551053bc6b5e5c0e9bb8e39c0a8359fd52f", + "sha256:db3535733f59e5605a88a706824dfcb9bd06725e709ecb017e165fc1d6e7d429", + "sha256:dd10383f1d6b7edf247d0960a3db274c07e96cf3a3fc7c41c8448f93eac3fb1c", + "sha256:e01f9531ba5420838c801c21c1b0f45dbc9607cb22ea2cf132844453bec863a5", + "sha256:e11527dc23d5ef44d76fef11213215c34f36af1608074561fcc561d983aeb870", + "sha256:e1ab2fac607842ac36864e358c42feb0960ae62c34aa4caaf12ada0a1fb5d99b", + "sha256:e1fd7d2fe11f1cb63d3336d147c852f6d07de0d0020d704c6031b46a30b02ca8", + "sha256:e9f84ed9f4d50b74fbc77298ee5c870f67cb7e91dcdc1a6915cb1ff6a317476c", + "sha256:ec4b4e75fc68da9dc0ed73dcdb431c25c57775383fec325d23a770a64e7ebc87", + "sha256:f10ce66fcdeb3543df51d423ede7e238be98412232fca5daec3e54bcd16b8da0", + "sha256:f63f62fc60e6228a4ca9abae28228f35e1bd3ce675013d1dfb828688d50c6e23", + "sha256:fa56bb08b3dd8eac3a8c5b7d075c94e74f755fd9d8a04543ae8d37b1612dd170", + "sha256:fa9b7c450be85bfc6cd39f6df8c5b8cbd76b5d6fc1f69efec80203f9894b885f" + ], + "index": "pypi", + "version": "==4.8.0" + }, + "pyyaml": { + "hashes": [ + "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293", + "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b", + "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57", + "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b", + "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4", + "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07", + "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba", + "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9", + "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287", + "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513", + "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0", + "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0", + "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92", + "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f", + "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2", + "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc", + "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c", + "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86", + "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4", + "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c", + "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34", + "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b", + "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c", + "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb", + "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737", + "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3", + "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d", + "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53", + "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78", + "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803", + "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a", + "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174", + "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5" + ], + "index": "pypi", + "version": "==6.0" + }, + "urllib3": { + "hashes": [ + "sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed", + "sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c" + ], + "index": "pypi", + "version": "==1.26.8" + } + }, + "develop": {} +} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..4257c12 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +lxml==4.8.0 +PyYAML==6.0 +urllib3==1.26.8 From c810ce730551e940aef27d4cb75232cd5e307582 Mon Sep 17 00:00:00 2001 From: jan Date: Mon, 20 Nov 2023 18:31:29 +0100 Subject: [PATCH 9/9] Create a generic filter to handle urls redirecting to replace the filter that was specific for streamtheworld --- ycast.py | 126 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 69 insertions(+), 57 deletions(-) diff --git a/ycast.py b/ycast.py index c3e92fa..f7dbe37 100755 --- a/ycast.py +++ b/ycast.py @@ -17,7 +17,7 @@ def initialize_logger(): - ''' initialize the default logger ''' + """ Initialize the default logger. """ import logging import logging.handlers logger = logging.getLogger(__name__) @@ -30,7 +30,7 @@ def initialize_logger(): def parse_command_line(): - ''' parse the command line arguments ''' + """ Parse the command line arguments. """ import argparse parser = argparse.ArgumentParser(description='vTuner API emulation') parser.add_argument('-l', action='store', dest='address', help='Listen address', default='0.0.0.0') @@ -41,32 +41,38 @@ def parse_command_line(): def filter_remove_SSL_from_url(url: str) -> str: - ''' change any https:// url to a http:// url ''' + """ Change any 'https://' url to a 'http://' url. """ + logger.info(f'filtering: remove ssl from {url}') return url.replace("https://", "http://", 1).replace(":443", "", 1) -def filter_streamtheworld_url(url:str ) -> str: - ''' Check the url and translate it if needed into a direct url ''' - # TODO: an incorrect url can keep this urlopen hanging: if the url is a valid stream, get may be getting the stream data - # find out how to detect this (only retrieve first n bytes ?) - # specifically, using https://20043.live.streamtheworld.com/WILLY.mp3?pname=rp_external gave an error - # whereas https://playerservices.streamtheworld.com/api/livestream-redirect/WILLY.mp3 is OK - if 'playerservices.streamtheworld.com/' in url: +def filter_redirecting_url(url:str) -> str: + """ Check if a URL redirects and if so, return the redirected url. """ + logger.info(f'ycast filtering {url}') + newurl = url + try: http = urllib3.PoolManager() - try: - resp = http.urlopen('GET', url, redirect=False, retries=False, timeout=2.0) + resp = http.urlopen('GET', url, redirect=False, preload_content=False, retries=False, timeout=2.0) + if resp.status == 200: + content_type = resp.getheader('Content-Type') + if content_type == 'audio/mpeg': + newurl = url + elif resp.status in [ 301, 302, 303, 307, 308 ]: + # 30x codes from https://urllib3.readthedocs.io/en/latest/reference/urllib3.response.html + logger.info(f'ycast response headers = {resp.headers}') newurl = resp.get_redirect_location() - except Exception as e: - logger(f'Exception {e} while filtering url {url}') - newurl = url - return newurl - return url + else: + logger.info(f'ycast url {url} returned status {resp.status} with headers\n{resp.headers}') + except Exception as e: + logger.info(f'ycast Exception {e} while filtering url {url}') + return newurl def apply_filters(url: str, filters: list) -> str: - ''' apply all filters from the given list to the url ''' + """ Apply all filters from the given list to the url. """ for filter in filters: url=filter(url) + logger.info(f'ycast filter result = {url}') return url @@ -92,7 +98,7 @@ def get_stations(self): def set_station_by_id(self, station_id=1, long_category=None): - ''' Associate each station with a unique id ''' + """ Associate each station with a unique id. """ def walktree(directory, station_id=1, category=None): for key, data in directory.items(): if isinstance(data, dict): @@ -106,9 +112,10 @@ def walktree(directory, station_id=1, category=None): def by_hierarchy(self, long_category): - ''' Return a dictionary of stations/dirs based on a long category name - a long category is a string with the hierarchy of categories: 'category|subcategory|subcategor|..' - ''' + """ Return a dictionary of stations/dirs based on a long category name. + + A long category is a string with the hierarchy of categories: 'category|subcategory|subcategor|..' + """ hierarchy = long_category.split('|') current_dir = self.stations for category in hierarchy: @@ -116,14 +123,15 @@ def by_hierarchy(self, long_category): return current_dir def by_id(self, station_id): + """ Return a dictionary of stations/dirs based on an entry id. """ return self.stations_by_id[station_id] class YCastHandler(BaseHTTPRequestHandler): - ''' YCastServer creates an instance of this class for each received message. __init__ passes the message to do_GET ''' + """ YCastServer creates an instance of this class for each received message. __init__ passes the message to do_GET. """ def do_GET(self): - ''' Handle the GET request and send reply to the client ''' + """ Handle the GET request and send reply to the client. """ logger.info(f'message received: {self.path}') stations = self.server.source.get_stations() url_split = parse.urlsplit(self.path) @@ -173,7 +181,7 @@ def do_GET(self): def reply_with_single_station(self, station_name, station_url, station_id): - ''' build an xml reply with the filtered url of a single station ''' + """ Build an xml reply with the filtered url of a single station. """ xml = self.create_root() station_url = apply_filters(station_url, filters_to_apply) # filters_to_apply is a global self.add_station(xml, station_name, station_url, station_id) @@ -181,11 +189,12 @@ def reply_with_single_station(self, station_name, station_url, station_id): def reply_with_dir(self, stations, start=0, max_size=8): - ''' Build an xml reply that represents a list of all directories - stations: the list of items to display - start: the first element of the list to display - max_size: the max number of elements to display - ''' + """ Build an xml reply that represents a list of all directories. + + stations: the list of items to display + start: the first element of the list to display + max_size: the max number of elements to display + """ xml = self.create_root() count = etree.SubElement(xml,'DirCount').text = str(len(stations)) for category in sorted(stations, key=str.lower)[start:start+max_size]: @@ -196,11 +205,12 @@ def reply_with_dir(self, stations, start=0, max_size=8): def reply_with_station_list(self, station_list, start=0, max_size=8): - ''' Build an xml reply that represents a list of all stations - station_list: the list of items to display - start: the first element of the list to display - max_size: the max number of elements to display - ''' + """ Build an xml reply that represents a list of all stations. + + station_list: the list of items to display + start: the first element of the list to display + max_size: the max number of elements to display + """ xml = self.create_root() for station in sorted(station_list, key=str.lower)[start:start+max_size]: station_id, station_url = station_list[station] @@ -208,11 +218,12 @@ def reply_with_station_list(self, station_list, start=0, max_size=8): self.write_message(xml) def reply_with_mixed_list(self, hierarchy, start=0, max_size=8): - ''' Build an xml reply that represents a list of mixed stations/directories - hierarchy: the list of items to display - start: the first element of the list to display - max_size: the max number of elements to display - ''' + """ Build an xml reply that represents a list of mixed stations/directories. + + hierarchy: the list of items to display + start: the first element of the list to display + max_size: the max number of elements to display + """ station_list = self.server.source.by_hierarchy(hierarchy) xml = self.create_root() for item in sorted(station_list, key=str.lower)[start:start+max_size]: @@ -229,7 +240,7 @@ def reply_with_mixed_list(self, hierarchy, start=0, max_size=8): def write_message(self, xml, add_xml_header=True): - ''' Write a message containing the given xml to send to the ycast client ''' + """ Write a message containing the given xml to send to the ycast client. """ self.send_response(200) self.send_header('Content-type', 'text/html') self.end_headers() @@ -241,17 +252,18 @@ def write_message(self, xml, add_xml_header=True): def create_root(self): - ''' Create the root of an xml tree ''' + """ Create the root of an xml tree. """ return etree.Element('ListOfItems') def add_dir(self, root, name, dest, dircount): - ''' Add a directory entry to the xml node - root: the node to add the directory information to - name: the name of the directory - dest: the url to visit to get the contents of the directory - dircount: the number of items in the directory (allows the client to say it is showing page 1 of 3) - ''' + """ Add a directory entry to the xml node. + + root: the node to add the directory information to + name: the name of the directory + dest: the url to visit to get the contents of the directory + dircount: the number of items in the directory (allows the client to say it is showing page 1 of 3) + """ item = etree.SubElement(root, 'Item') etree.SubElement(item, 'ItemType').text = 'Dir' etree.SubElement(item, 'Title').text = name @@ -261,12 +273,13 @@ def add_dir(self, root, name, dest, dircount): def add_station(self, root, name, url, station_id): - ''' Add a station entry to the xml node - root: the node to add the directory information to - name: the name of the station - url: the url to visit to listen to the station - station_id: the unique id of the station (that will be sent to the server if the client wants it) - ''' + """ Add a station entry to the xml node. + + root: the node to add the directory information to + name: the name of the station + url: the url to visit to listen to the station + station_id: the unique id of the station (that will be sent to the server if the client wants it) + """ item = etree.SubElement(root, 'Item') etree.SubElement(item, 'ItemType').text = 'Station' etree.SubElement(item, 'StationName').text = name @@ -276,8 +289,7 @@ def add_station(self, root, name, url, station_id): class YCastServer(HTTPServer): - ''' A HTTPServer that retains a pointer to the source to be used by the BaseHTTPRequestHandler - ''' + """ A HTTPServer that retains a pointer to the source to be used by the BaseHTTPRequestHandler. """ def __init__(self, source, *args, **kwargs): self.source = StationSource(source) address,port = args[0] @@ -296,7 +308,7 @@ def __exit__(self, *args): logger = initialize_logger() arguments = parse_command_line() -filters_to_apply = [ filter_streamtheworld_url ] +filters_to_apply = [ filter_redirecting_url ] if arguments.nossl: filters_to_apply.append(filter_remove_SSL_from_url)