mirror of
https://github.com/OpenBB-finance/OpenBB.git
synced 2026-10-27 04:31:13 +00:00
Pyinstaller : remove property_cached, user_agent and vaderSentiment
This commit is contained in:
Chavithra PARANA
@@ -1,159 +0,0 @@
|
||||
import asyncio
|
||||
import functools
|
||||
import pkg_resources
|
||||
import threading
|
||||
import weakref
|
||||
from time import time
|
||||
|
||||
|
||||
__author__ = "Martin Larralde"
|
||||
__email__ = "martin.larralde@ens-paris-saclay.fr"
|
||||
__license__ = "BSD"
|
||||
__version__ = pkg_resources.resource_string(__name__, "_version.txt").decode('utf-8').strip()
|
||||
|
||||
|
||||
class cached_property(property):
|
||||
"""
|
||||
A property that is only computed once per instance and then replaces itself
|
||||
with an ordinary attribute. Deleting the attribute resets the property.
|
||||
Source: https://github.com/bottlepy/bottle/commit/fa7733e075da0d790d809aa3d2f53071897e6f76
|
||||
""" # noqa
|
||||
|
||||
_sentinel = object()
|
||||
_update_wrapper = functools.update_wrapper
|
||||
|
||||
def __init__(self, func):
|
||||
self.cache = weakref.WeakKeyDictionary()
|
||||
self.func = func
|
||||
self._update_wrapper(func)
|
||||
|
||||
def __get__(self, obj, cls):
|
||||
if obj is None:
|
||||
return self
|
||||
|
||||
if asyncio and asyncio.iscoroutinefunction(self.func):
|
||||
return self._wrap_in_coroutine(obj)
|
||||
|
||||
value = self.cache.get(obj, self._sentinel)
|
||||
if value is self._sentinel:
|
||||
value = self.cache[obj] = self.func(obj)
|
||||
|
||||
return value
|
||||
|
||||
def __set_name__(self, owner, name):
|
||||
self.__name__ = name
|
||||
|
||||
def __set__(self, obj, value):
|
||||
self.cache[obj] = value
|
||||
|
||||
def __delete__(self, obj):
|
||||
del self.cache[obj]
|
||||
|
||||
def _wrap_in_coroutine(self, obj):
|
||||
|
||||
@functools.wraps(obj)
|
||||
@asyncio.coroutine
|
||||
def wrapper():
|
||||
value = self.cache.get(obj, self._sentinel)
|
||||
if value is self._sentinel:
|
||||
self.cache[obj] = value = asyncio.ensure_future(self.func(obj))
|
||||
return value
|
||||
|
||||
return wrapper()
|
||||
|
||||
|
||||
class threaded_cached_property(cached_property):
|
||||
"""
|
||||
A cached_property version for use in environments where multiple threads
|
||||
might concurrently try to access the property.
|
||||
"""
|
||||
|
||||
def __init__(self, func):
|
||||
super(threaded_cached_property, self).__init__(func)
|
||||
self.lock = threading.RLock()
|
||||
|
||||
def __get__(self, obj, cls):
|
||||
if obj is None:
|
||||
return self
|
||||
with self.lock:
|
||||
return super(threaded_cached_property, self).__get__(obj, cls)
|
||||
|
||||
def __set__(self, obj, value):
|
||||
with self.lock:
|
||||
super(threaded_cached_property, self).__set__(obj, value)
|
||||
|
||||
def __delete__(self, obj):
|
||||
with self.lock:
|
||||
super(threaded_cached_property, self).__delete__(obj)
|
||||
|
||||
|
||||
class cached_property_with_ttl(cached_property):
|
||||
"""
|
||||
A property that is only computed once per instance and then replaces itself
|
||||
with an ordinary attribute. Setting the ttl to a number expresses how long
|
||||
the property will last before being timed out.
|
||||
"""
|
||||
|
||||
def __init__(self, ttl=None):
|
||||
if callable(ttl):
|
||||
func = ttl
|
||||
ttl = None
|
||||
else:
|
||||
func = None
|
||||
self.ttl = ttl
|
||||
super(cached_property_with_ttl, self).__init__(func)
|
||||
|
||||
def __call__(self, func):
|
||||
super(cached_property_with_ttl, self).__init__(func)
|
||||
return self
|
||||
|
||||
def __get__(self, obj, cls):
|
||||
if obj is None:
|
||||
return self
|
||||
|
||||
now = time()
|
||||
if obj in self.cache:
|
||||
value, last_updated = self.cache[obj]
|
||||
if not self.ttl or self.ttl > now - last_updated:
|
||||
return value
|
||||
|
||||
value, _ = self.cache[obj] = (self.func(obj), now)
|
||||
return value
|
||||
|
||||
def __set__(self, obj, value):
|
||||
super(cached_property_with_ttl, self).__set__(obj, (value, time()))
|
||||
|
||||
|
||||
# Aliases to make cached_property_with_ttl easier to use
|
||||
cached_property_ttl = cached_property_with_ttl
|
||||
timed_cached_property = cached_property_with_ttl
|
||||
|
||||
|
||||
class threaded_cached_property_with_ttl(
|
||||
cached_property_with_ttl, threaded_cached_property
|
||||
):
|
||||
"""
|
||||
A cached_property version for use in environments where multiple threads
|
||||
might concurrently try to access the property.
|
||||
"""
|
||||
|
||||
def __init__(self, ttl=None):
|
||||
super(threaded_cached_property_with_ttl, self).__init__(ttl)
|
||||
self.lock = threading.RLock()
|
||||
|
||||
def __get__(self, obj, cls):
|
||||
with self.lock:
|
||||
return super(threaded_cached_property_with_ttl, self).__get__(obj, cls)
|
||||
|
||||
def __set__(self, obj, value):
|
||||
with self.lock:
|
||||
return super(threaded_cached_property_with_ttl, self).__set__(obj, value)
|
||||
|
||||
def __delete__(self, obj):
|
||||
with self.lock:
|
||||
return super(threaded_cached_property_with_ttl, self).__delete__(obj)
|
||||
|
||||
|
||||
# Alias to make threaded_cached_property_with_ttl easier to use
|
||||
threaded_cached_property_ttl = threaded_cached_property_with_ttl
|
||||
timed_threaded_cached_property = threaded_cached_property_with_ttl
|
||||
@@ -1 +0,0 @@
|
||||
1.6.4
|
||||
@@ -1,4 +0,0 @@
|
||||
from user_agent.base import * # noqa: F403 pylint: disable=wildcard-import
|
||||
from user_agent.error import * # noqa: F403 pylint: disable=wildcard-import
|
||||
|
||||
__version__ = '0.1.10'
|
||||
@@ -1,627 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# pylint: disable=line-too-long
|
||||
"""
|
||||
This module is for generating random, valid web navigator's
|
||||
configs & User-Agent HTTP headers.
|
||||
|
||||
Functions:
|
||||
* generate_user_agent: generates User-Agent HTTP header
|
||||
* generate_navigator: generates web navigator's config
|
||||
* generate_navigator_js: generates web navigator's config with keys
|
||||
identical keys used in navigator object
|
||||
|
||||
FIXME:
|
||||
* add Edge, Safari and Opera support
|
||||
* add random config i.e. windows is more common than linux
|
||||
|
||||
Specs:
|
||||
* https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent/Firefox
|
||||
* http://msdn.microsoft.com/en-us/library/ms537503(VS.85).aspx
|
||||
* https://developer.chrome.com/multidevice/user-agent
|
||||
* http://www.javascriptkit.com/javatutors/navigator.shtml
|
||||
|
||||
Release history:
|
||||
* https://en.wikipedia.org/wiki/Firefox_release_history
|
||||
* https://en.wikipedia.org/wiki/Google_Chrome_release_history
|
||||
* https://en.wikipedia.org/wiki/Internet_Explorer_version_history
|
||||
* https://en.wikipedia.org/wiki/Android_version_history
|
||||
|
||||
Lists of user agents:
|
||||
* http://www.useragentstring.com/
|
||||
* http://www.user-agents.org/
|
||||
* http://www.webapps-online.com/online-tools/user-agent-strings
|
||||
|
||||
"""
|
||||
# pylint: enable=line-too-long
|
||||
|
||||
from random import SystemRandom
|
||||
from datetime import datetime, timedelta
|
||||
from itertools import product
|
||||
|
||||
import six
|
||||
|
||||
from .warning import warn
|
||||
# pylint: disable=unused-import
|
||||
from .device import SMARTPHONE_DEV_IDS, TABLET_DEV_IDS
|
||||
# pylint: enable=unused-import
|
||||
from .error import InvalidOption
|
||||
|
||||
__all__ = ['generate_user_agent', 'generate_navigator',
|
||||
'generate_navigator_js']
|
||||
|
||||
randomizer = SystemRandom()
|
||||
DEVICE_TYPE_OS = {
|
||||
'desktop': ('win', 'mac', 'linux'),
|
||||
'smartphone': ('android',),
|
||||
'tablet': ('android',),
|
||||
}
|
||||
OS_DEVICE_TYPE = {
|
||||
'win': ('desktop',),
|
||||
'linux': ('desktop',),
|
||||
'mac': ('desktop',),
|
||||
'android': ('smartphone', 'tablet'),
|
||||
}
|
||||
DEVICE_TYPE_NAVIGATOR = {
|
||||
'desktop': ('chrome', 'firefox', 'ie'),
|
||||
'smartphone': ('firefox', 'chrome'),
|
||||
'tablet': ('firefox', 'chrome'),
|
||||
}
|
||||
NAVIGATOR_DEVICE_TYPE = {
|
||||
'ie': ('desktop',),
|
||||
'chrome': ('desktop', 'smartphone', 'tablet'),
|
||||
'firefox': ('desktop', 'smartphone', 'tablet'),
|
||||
}
|
||||
OS_PLATFORM = {
|
||||
'win': (
|
||||
'Windows NT 5.1', # Windows XP
|
||||
'Windows NT 6.1', # Windows 7
|
||||
'Windows NT 6.2', # Windows 8
|
||||
'Windows NT 6.3', # Windows 8.1
|
||||
'Windows NT 10.0', # Windows 10
|
||||
),
|
||||
'mac': (
|
||||
'Macintosh; Intel Mac OS X 10.8',
|
||||
'Macintosh; Intel Mac OS X 10.9',
|
||||
'Macintosh; Intel Mac OS X 10.10',
|
||||
'Macintosh; Intel Mac OS X 10.11',
|
||||
'Macintosh; Intel Mac OS X 10.12',
|
||||
),
|
||||
'linux': (
|
||||
'X11; Linux',
|
||||
'X11; Ubuntu; Linux',
|
||||
),
|
||||
'android': (
|
||||
'Android 4.4', # 2013-10-31
|
||||
'Android 4.4.1', # 2013-12-05
|
||||
'Android 4.4.2', # 2013-12-09
|
||||
'Android 4.4.3', # 2014-06-02
|
||||
'Android 4.4.4', # 2014-06-19
|
||||
'Android 5.0', # 2014-11-12
|
||||
'Android 5.0.1', # 2014-12-02
|
||||
'Android 5.0.2', # 2014-12-19
|
||||
'Android 5.1', # 2015-03-09
|
||||
'Android 5.1.1', # 2015-04-21
|
||||
'Android 6.0', # 2015-10-05
|
||||
'Android 6.0.1', # 2015-12-07
|
||||
#'Android 7.0', # 2016-08-22
|
||||
#'Android 7.1', # 2016-10-04
|
||||
#'Android 7.1.1', # 2016-12-05
|
||||
),
|
||||
}
|
||||
OS_CPU = {
|
||||
'win': (
|
||||
'', # 32bit
|
||||
'Win64; x64', # 64bit
|
||||
'WOW64', # 32bit process on 64bit system
|
||||
),
|
||||
'linux': (
|
||||
'i686', # 32bit
|
||||
'x86_64', # 64bit
|
||||
'i686 on x86_64', # 32bit process on 64bit system
|
||||
),
|
||||
'mac': (
|
||||
'',
|
||||
),
|
||||
'android': (
|
||||
'armv7l', # 32bit
|
||||
'armv8l', # 64bit
|
||||
),
|
||||
}
|
||||
OS_NAVIGATOR = {
|
||||
'win': ('chrome', 'firefox', 'ie'),
|
||||
'mac': ('firefox', 'chrome'),
|
||||
'linux': ('chrome', 'firefox'),
|
||||
'android': ('firefox', 'chrome'),
|
||||
}
|
||||
NAVIGATOR_OS = {
|
||||
'chrome': ('win', 'linux', 'mac', 'android'),
|
||||
'firefox': ('win', 'linux', 'mac', 'android'),
|
||||
'ie': ('win',),
|
||||
}
|
||||
FIREFOX_VERSION = (
|
||||
('45.0', datetime(2016, 3, 8)),
|
||||
('46.0', datetime(2016, 4, 26)),
|
||||
('47.0', datetime(2016, 6, 7)),
|
||||
('48.0', datetime(2016, 8, 2)),
|
||||
('49.0', datetime(2016, 9, 20)),
|
||||
('50.0', datetime(2016, 11, 15)),
|
||||
('51.0', datetime(2017, 1, 24)),
|
||||
)
|
||||
|
||||
# Top chrome builds from website access log
|
||||
# for september, october 2020
|
||||
CHROME_BUILD = '''
|
||||
80.0.3987.132
|
||||
80.0.3987.149
|
||||
80.0.3987.99
|
||||
81.0.4044.117
|
||||
81.0.4044.138
|
||||
83.0.4103.101
|
||||
83.0.4103.106
|
||||
83.0.4103.96
|
||||
84.0.4147.105
|
||||
84.0.4147.111
|
||||
84.0.4147.125
|
||||
84.0.4147.135
|
||||
84.0.4147.89
|
||||
85.0.4183.101
|
||||
85.0.4183.102
|
||||
85.0.4183.120
|
||||
85.0.4183.121
|
||||
85.0.4183.127
|
||||
85.0.4183.81
|
||||
85.0.4183.83
|
||||
86.0.4240.110
|
||||
86.0.4240.111
|
||||
86.0.4240.114
|
||||
86.0.4240.183
|
||||
86.0.4240.185
|
||||
86.0.4240.75
|
||||
86.0.4240.78
|
||||
86.0.4240.80
|
||||
86.0.4240.96
|
||||
86.0.4240.99
|
||||
'''.strip().splitlines()
|
||||
|
||||
IE_VERSION = (
|
||||
# (numeric ver, string ver, trident ver) # release year
|
||||
(8, 'MSIE 8.0', '4.0'), # 2009
|
||||
(9, 'MSIE 9.0', '5.0'), # 2011
|
||||
(10, 'MSIE 10.0', '6.0'), # 2012
|
||||
(11, 'MSIE 11.0', '7.0'), # 2013
|
||||
)
|
||||
USER_AGENT_TEMPLATE = {
|
||||
'firefox': (
|
||||
'Mozilla/5.0'
|
||||
' ({system[ua_platform]}; rv:{app[build_version]})'
|
||||
' Gecko/{app[geckotrail]}'
|
||||
' Firefox/{app[build_version]}'
|
||||
),
|
||||
'chrome': (
|
||||
'Mozilla/5.0'
|
||||
' ({system[ua_platform]}) AppleWebKit/537.36'
|
||||
' (KHTML, like Gecko)'
|
||||
' Chrome/{app[build_version]} Safari/537.36'
|
||||
),
|
||||
'chrome_smartphone': (
|
||||
'Mozilla/5.0'
|
||||
' ({system[ua_platform]}) AppleWebKit/537.36'
|
||||
' (KHTML, like Gecko)'
|
||||
' Chrome/{app[build_version]} Mobile Safari/537.36'
|
||||
),
|
||||
'chrome_tablet': (
|
||||
'Mozilla/5.0'
|
||||
' ({system[ua_platform]}) AppleWebKit/537.36'
|
||||
' (KHTML, like Gecko)'
|
||||
' Chrome/{app[build_version]} Safari/537.36'
|
||||
),
|
||||
'ie_less_11': (
|
||||
'Mozilla/5.0'
|
||||
' (compatible; {app[build_version]}; {system[ua_platform]};'
|
||||
' Trident/{app[trident_version]})'
|
||||
),
|
||||
'ie_11': (
|
||||
'Mozilla/5.0'
|
||||
' ({system[ua_platform]}; Trident/{app[trident_version]};'
|
||||
' rv:11.0) like Gecko'
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def get_firefox_build():
|
||||
build_ver, date_from = randomizer.choice(FIREFOX_VERSION)
|
||||
try:
|
||||
idx = FIREFOX_VERSION.index((build_ver, date_from))
|
||||
_, date_to = FIREFOX_VERSION[idx + 1]
|
||||
except IndexError:
|
||||
date_to = date_from + timedelta(days=1)
|
||||
sec_range = (date_to - date_from).total_seconds() - 1
|
||||
build_rnd_time = (
|
||||
date_from + timedelta(seconds=randomizer.randint(0, int(sec_range)))
|
||||
)
|
||||
return build_ver, build_rnd_time.strftime('%Y%m%d%H%M%S')
|
||||
|
||||
|
||||
def get_chrome_build():
|
||||
return randomizer.choice(CHROME_BUILD)
|
||||
|
||||
|
||||
def get_ie_build():
|
||||
"""
|
||||
Return random IE version as tuple
|
||||
(numeric_version, us-string component)
|
||||
|
||||
Example: (8, 'MSIE 8.0')
|
||||
"""
|
||||
|
||||
return randomizer.choice(IE_VERSION)
|
||||
|
||||
|
||||
MACOSX_CHROME_BUILD_RANGE = {
|
||||
# https://en.wikipedia.org/wiki/MacOS#Release_history
|
||||
'10.8': (0, 8),
|
||||
'10.9': (0, 5),
|
||||
'10.10': (0, 5),
|
||||
'10.11': (0, 6),
|
||||
'10.12': (0, 2)
|
||||
}
|
||||
|
||||
|
||||
def fix_chrome_mac_platform(platform):
|
||||
"""
|
||||
Chrome on Mac OS adds minor version number and uses underscores instead
|
||||
of dots. E.g. platform for Firefox will be: 'Intel Mac OS X 10.11'
|
||||
but for Chrome it will be 'Intel Mac OS X 10_11_6'.
|
||||
|
||||
:param platform: - string like "Macintosh; Intel Mac OS X 10.8"
|
||||
:return: platform with version number including minor number and formatted
|
||||
with underscores, e.g. "Macintosh; Intel Mac OS X 10_8_2"
|
||||
"""
|
||||
ver = platform.split('OS X ')[1]
|
||||
build_range = range(*MACOSX_CHROME_BUILD_RANGE[ver])
|
||||
build = randomizer.choice(build_range)
|
||||
mac_ver = ver.replace('.', '_') + '_' + str(build)
|
||||
return 'Macintosh; Intel Mac OS X %s' % mac_ver
|
||||
|
||||
|
||||
def build_system_components(device_type, os_id, navigator_id):
|
||||
"""
|
||||
For given os_id build random platform and oscpu
|
||||
components
|
||||
|
||||
Returns dict {platform_version, platform, ua_platform, oscpu}
|
||||
|
||||
platform_version is OS name used in different places
|
||||
ua_platform goes to navigator.platform
|
||||
platform is used in building navigator.userAgent
|
||||
oscpu goes to navigator.oscpu
|
||||
"""
|
||||
|
||||
if os_id == 'win':
|
||||
platform_version = randomizer.choice(OS_PLATFORM['win'])
|
||||
cpu = randomizer.choice(OS_CPU['win'])
|
||||
if cpu:
|
||||
platform = '%s; %s' % (platform_version, cpu)
|
||||
else:
|
||||
platform = platform_version
|
||||
res = {
|
||||
'platform_version': platform_version,
|
||||
'platform': platform,
|
||||
'ua_platform': platform,
|
||||
'oscpu': platform,
|
||||
}
|
||||
elif os_id == 'linux':
|
||||
cpu = randomizer.choice(OS_CPU['linux'])
|
||||
platform_version = randomizer.choice(OS_PLATFORM['linux'])
|
||||
platform = '%s %s' % (platform_version, cpu)
|
||||
res = {
|
||||
'platform_version': platform_version,
|
||||
'platform': platform,
|
||||
'ua_platform': platform,
|
||||
'oscpu': 'Linux %s' % cpu,
|
||||
}
|
||||
elif os_id == 'mac':
|
||||
cpu = randomizer.choice(OS_CPU['mac'])
|
||||
platform_version = randomizer.choice(OS_PLATFORM['mac'])
|
||||
platform = platform_version
|
||||
if navigator_id == 'chrome':
|
||||
platform = fix_chrome_mac_platform(platform)
|
||||
res = {
|
||||
'platform_version': platform_version,
|
||||
'platform': 'MacIntel',
|
||||
'ua_platform': platform,
|
||||
'oscpu': 'Intel Mac OS X %s' % platform.split(' ')[-1],
|
||||
}
|
||||
elif os_id == 'android':
|
||||
assert navigator_id in ('firefox', 'chrome')
|
||||
assert device_type in ('smartphone', 'tablet')
|
||||
platform_version = randomizer.choice(OS_PLATFORM['android'])
|
||||
if navigator_id == 'firefox':
|
||||
if device_type == 'smartphone':
|
||||
ua_platform = '%s; Mobile' % platform_version
|
||||
elif device_type == 'tablet':
|
||||
ua_platform = '%s; Tablet' % platform_version
|
||||
elif navigator_id == 'chrome':
|
||||
device_id = randomizer.choice(SMARTPHONE_DEV_IDS)
|
||||
ua_platform = 'Linux; %s; %s' % (platform_version, device_id)
|
||||
oscpu = 'Linux %s' % randomizer.choice(OS_CPU['android'])
|
||||
res = {
|
||||
'platform_version': platform_version,
|
||||
'ua_platform': ua_platform,
|
||||
'platform': oscpu,
|
||||
'oscpu': oscpu,
|
||||
}
|
||||
return res
|
||||
|
||||
|
||||
def build_app_components(os_id, navigator_id):
|
||||
"""
|
||||
For given navigator_id build app features
|
||||
|
||||
Returns dict {name, product_sub, vendor, build_version, build_id}
|
||||
"""
|
||||
|
||||
if navigator_id == 'firefox':
|
||||
build_version, build_id = get_firefox_build()
|
||||
if os_id in ('win', 'linux', 'mac'):
|
||||
geckotrail = '20100101'
|
||||
else:
|
||||
geckotrail = build_version
|
||||
res = {
|
||||
'name': 'Netscape',
|
||||
'product_sub': '20100101',
|
||||
'vendor': '',
|
||||
'build_version': build_version,
|
||||
'build_id': build_id,
|
||||
'geckotrail': geckotrail,
|
||||
}
|
||||
elif navigator_id == 'chrome':
|
||||
res = {
|
||||
'name': 'Netscape',
|
||||
'product_sub': '20030107',
|
||||
'vendor': 'Google Inc.',
|
||||
'build_version': get_chrome_build(),
|
||||
'build_id': None,
|
||||
}
|
||||
elif navigator_id == 'ie':
|
||||
num_ver, build_version, trident_version = get_ie_build()
|
||||
if num_ver >= 11:
|
||||
app_name = 'Netscape'
|
||||
else:
|
||||
app_name = 'Microsoft Internet Explorer'
|
||||
res = {
|
||||
'name': app_name,
|
||||
'product_sub': None,
|
||||
'vendor': '',
|
||||
'build_version': build_version,
|
||||
'build_id': None,
|
||||
'trident_version': trident_version,
|
||||
}
|
||||
return res
|
||||
|
||||
|
||||
def get_option_choices(opt_name, opt_value, default_value, all_choices):
|
||||
"""
|
||||
Generate possible choices for the option `opt_name`
|
||||
limited to `opt_value` value with default value
|
||||
as `default_value`
|
||||
"""
|
||||
|
||||
choices = []
|
||||
if isinstance(opt_value, six.string_types):
|
||||
choices = [opt_value]
|
||||
elif isinstance(opt_value, (list, tuple)):
|
||||
choices = list(opt_value)
|
||||
elif opt_value is None:
|
||||
choices = default_value
|
||||
else:
|
||||
raise InvalidOption('Option %s has invalid'
|
||||
' value: %s' % (opt_name, opt_value))
|
||||
if 'all' in choices:
|
||||
choices = all_choices
|
||||
for item in choices:
|
||||
if item not in all_choices:
|
||||
raise InvalidOption('Choices of option %s contains invalid'
|
||||
' item: %s' % (opt_name, item))
|
||||
return choices
|
||||
|
||||
|
||||
def pick_config_ids(device_type, os, navigator):
|
||||
"""
|
||||
Select one random pair (device_type, os_id, navigator_id) from
|
||||
all possible combinations matching the given os and
|
||||
navigator filters.
|
||||
|
||||
:param os: allowed os(es)
|
||||
:type os: string or list/tuple or None
|
||||
:param navigator: allowed browser engine(s)
|
||||
:type navigator: string or list/tuple or None
|
||||
:param device_type: limit possible oses by device type
|
||||
:type device_type: list/tuple or None, possible values:
|
||||
"desktop", "smartphone", "tablet", "all"
|
||||
"""
|
||||
|
||||
if os is None:
|
||||
default_dev_types = ['desktop']
|
||||
else:
|
||||
default_dev_types = list(DEVICE_TYPE_OS.keys())
|
||||
dev_type_choices = get_option_choices(
|
||||
'device_type', device_type, default_dev_types,
|
||||
list(DEVICE_TYPE_OS.keys())
|
||||
)
|
||||
os_choices = get_option_choices('os', os, list(OS_NAVIGATOR.keys()),
|
||||
list(OS_NAVIGATOR.keys()))
|
||||
nav_choices = get_option_choices('navigator', navigator,
|
||||
list(NAVIGATOR_OS.keys()),
|
||||
list(NAVIGATOR_OS.keys()))
|
||||
|
||||
variants = []
|
||||
for dev, os, nav in product(dev_type_choices, os_choices,
|
||||
nav_choices):
|
||||
|
||||
if (os in DEVICE_TYPE_OS[dev]
|
||||
and nav in DEVICE_TYPE_NAVIGATOR[dev]
|
||||
and nav in OS_NAVIGATOR[os]):
|
||||
variants.append((dev, os, nav))
|
||||
if not variants:
|
||||
raise InvalidOption('Options device_type, os and navigator'
|
||||
' conflicts with each other')
|
||||
device_type, os_id, navigator_id = randomizer.choice(variants)
|
||||
|
||||
assert os_id in OS_PLATFORM
|
||||
assert navigator_id in NAVIGATOR_OS
|
||||
assert device_type in DEVICE_TYPE_OS
|
||||
|
||||
return device_type, os_id, navigator_id
|
||||
|
||||
|
||||
def choose_ua_template(device_type, navigator_id, app):
|
||||
tpl_name = navigator_id
|
||||
if navigator_id == 'ie':
|
||||
tpl_name = ('ie_11' if app['build_version'] == 'MSIE 11.0'
|
||||
else 'ie_less_11')
|
||||
if navigator_id == 'chrome':
|
||||
if device_type == 'smartphone':
|
||||
tpl_name = 'chrome_smartphone'
|
||||
if device_type == 'tablet':
|
||||
tpl_name = 'chrome_tablet'
|
||||
return USER_AGENT_TEMPLATE[tpl_name]
|
||||
|
||||
|
||||
def build_navigator_app_version(os_id, navigator_id,
|
||||
platform_version, user_agent):
|
||||
if navigator_id in ('chrome', 'ie'):
|
||||
assert user_agent.startswith('Mozilla/')
|
||||
app_version = user_agent.split('Mozilla/', 1)[1]
|
||||
elif navigator_id == 'firefox':
|
||||
if os_id == 'android':
|
||||
app_version = '5.0 (%s)' % platform_version
|
||||
else:
|
||||
os_token = {
|
||||
'win': 'Windows',
|
||||
'mac': 'Macintosh',
|
||||
'linux': 'X11',
|
||||
}[os_id]
|
||||
app_version = '5.0 (%s)' % os_token
|
||||
return app_version
|
||||
|
||||
|
||||
def generate_navigator(os=None, navigator=None, platform=None,
|
||||
device_type=None):
|
||||
"""
|
||||
Generates web navigator's config
|
||||
|
||||
:param os: limit list of oses for generation
|
||||
:type os: string or list/tuple or None
|
||||
:param navigator: limit list of browser engines for generation
|
||||
:type navigator: string or list/tuple or None
|
||||
:param device_type: limit possible oses by device type
|
||||
:type device_type: list/tuple or None, possible values:
|
||||
"desktop", "smartphone", "tablet", "all"
|
||||
|
||||
:return: User-Agent config
|
||||
:rtype: dict with keys (os, name, platform, oscpu, build_version,
|
||||
build_id, app_version, app_name, app_code_name,
|
||||
product, product_sub, vendor, vendor_sub,
|
||||
user_agent)
|
||||
:raises InvalidOption: if could not generate user-agent for
|
||||
any combination of allowed platforms and navigators
|
||||
:raise InvalidOption: if any of passed options is invalid
|
||||
"""
|
||||
|
||||
if platform is not None:
|
||||
os = platform
|
||||
warn('The `platform` option is deprecated.'
|
||||
' Use `os` option instead.', stacklevel=3)
|
||||
device_type, os_id, navigator_id = (
|
||||
pick_config_ids(device_type, os, navigator)
|
||||
)
|
||||
system = build_system_components(
|
||||
device_type, os_id, navigator_id)
|
||||
app = build_app_components(os_id, navigator_id)
|
||||
ua_template = choose_ua_template(
|
||||
device_type, navigator_id, app)
|
||||
user_agent = ua_template.format(system=system, app=app)
|
||||
app_version = build_navigator_app_version(
|
||||
os_id, navigator_id, system['platform_version'], user_agent)
|
||||
return {
|
||||
# ids
|
||||
'os_id': os_id,
|
||||
'navigator_id': navigator_id,
|
||||
# system components
|
||||
'platform': system['platform'],
|
||||
'oscpu': system['oscpu'],
|
||||
# app components
|
||||
'build_version': app['build_version'],
|
||||
'build_id': app['build_id'],
|
||||
'app_version': app_version,
|
||||
'app_name': app['name'],
|
||||
'app_code_name': 'Mozilla',
|
||||
'product': 'Gecko',
|
||||
'product_sub': app['product_sub'],
|
||||
'vendor': app['vendor'],
|
||||
'vendor_sub': '',
|
||||
# compiled user agent
|
||||
'user_agent': user_agent,
|
||||
}
|
||||
|
||||
|
||||
def generate_user_agent(os=None, navigator=None, platform=None,
|
||||
device_type=None):
|
||||
"""
|
||||
Generates HTTP User-Agent header
|
||||
|
||||
:param os: limit list of os for generation
|
||||
:type os: string or list/tuple or None
|
||||
:param navigator: limit list of browser engines for generation
|
||||
:type navigator: string or list/tuple or None
|
||||
:param device_type: limit possible oses by device type
|
||||
:type device_type: list/tuple or None, possible values:
|
||||
"desktop", "smartphone", "tablet", "all"
|
||||
:return: User-Agent string
|
||||
:rtype: string
|
||||
:raises InvalidOption: if could not generate user-agent for
|
||||
any combination of allowed oses and navigators
|
||||
:raise InvalidOption: if any of passed options is invalid
|
||||
"""
|
||||
return generate_navigator(os=os, navigator=navigator,
|
||||
platform=platform,
|
||||
device_type=device_type)['user_agent']
|
||||
|
||||
|
||||
def generate_navigator_js(os=None, navigator=None, platform=None,
|
||||
device_type=None):
|
||||
"""
|
||||
Generates web navigator's config with keys corresponding
|
||||
to keys of `windows.navigator` JavaScript object.
|
||||
|
||||
:param os: limit list of oses for generation
|
||||
:type os: string or list/tuple or None
|
||||
:param navigator: limit list of browser engines for generation
|
||||
:type navigator: string or list/tuple or None
|
||||
:param device_type: limit possible oses by device type
|
||||
:type device_type: list/tuple or None, possible values:
|
||||
"desktop", "smartphone", "tablet", "all"
|
||||
:return: User-Agent config
|
||||
:rtype: dict with keys (TODO)
|
||||
:raises InvalidOption: if could not generate user-agent for
|
||||
any combination of allowed oses and navigators
|
||||
:raise InvalidOption: if any of passed options is invalid
|
||||
"""
|
||||
|
||||
config = generate_navigator(os=os, navigator=navigator,
|
||||
platform=platform,
|
||||
device_type=device_type)
|
||||
return {
|
||||
'appCodeName': config['app_code_name'],
|
||||
'appName': config['app_name'],
|
||||
'appVersion': config['app_version'],
|
||||
'platform': config['platform'],
|
||||
'userAgent': config['user_agent'],
|
||||
'oscpu': config['oscpu'],
|
||||
'product': config['product'],
|
||||
'productSub': config['product_sub'],
|
||||
'vendor': config['vendor'],
|
||||
'vendorSub': config['vendor_sub'],
|
||||
'buildID': config['build_id'],
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
from argparse import ArgumentParser
|
||||
import json
|
||||
|
||||
from user_agent import generate_navigator_js
|
||||
|
||||
|
||||
def script_ua():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument('-e', '--extended', action='store_true',
|
||||
default=False)
|
||||
parser.add_argument('-o', '--os')
|
||||
parser.add_argument('-n', '--navigator')
|
||||
parser.add_argument('-d', '--device-type')
|
||||
opts = parser.parse_args()
|
||||
nav = generate_navigator_js(os=opts.os,
|
||||
navigator=opts.navigator,
|
||||
device_type=opts.device_type)
|
||||
if opts.extended:
|
||||
print(json.dumps(nav, indent=2))
|
||||
else:
|
||||
print(nav['userAgent'])
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,180 +0,0 @@
|
||||
[
|
||||
"6043D Build/KOT49H",
|
||||
"7045Y Build/KOT49H",
|
||||
"7050Y Build/JLS36C",
|
||||
"ALE-L21 Build/HuaweiALE-L21",
|
||||
"Archos 50c Oxygen Build/KOT49H",
|
||||
"D2005 Build/20.1.A.2.19",
|
||||
"D2105 Build/20.1.B.2.29",
|
||||
"D2203 Build/18.5.C.0.19",
|
||||
"D2212 Build/18.4.B.1.20",
|
||||
"D2302 Build/18.6.A.0.182",
|
||||
"D2403 Build/18.6.A.0.182",
|
||||
"D2502 Build/19.2.1.A.0.72",
|
||||
"D5322 Build/19.1.C.0.116",
|
||||
"D5503 Build/14.6.A.1.236",
|
||||
"E2003 Build/25.0.A.2.14",
|
||||
"E2003 Build/25.0.A.2.31",
|
||||
"E2033 Build/25.0.B.2.31",
|
||||
"E2115 Build/24.0.B.5.14",
|
||||
"E2303 Build/26.1.A.3.111",
|
||||
"E5303 Build/27.2.A.0.169",
|
||||
"E5533 Build/29.1.B.0.101",
|
||||
"E6533 Build/32.2.A.0.253",
|
||||
"E6553 Build/32.1.A.1.185",
|
||||
"E6653 Build/32.2.A.0.305",
|
||||
"E6683 Build/32.2.A.0.253",
|
||||
"E6853 Build/32.2.A.0.224",
|
||||
"F3311 Build/37.0.A.2.108",
|
||||
"F5121 Build/34.2.A.0.292",
|
||||
"F8132 Build/35.0.A.1.297",
|
||||
"F8331 Build/39.2.A.0.417",
|
||||
"GSmart Mika M3 Build/GSmart",
|
||||
"GT-I9060 Build/JDQ39",
|
||||
"GT-I9300 Build/JRO03C",
|
||||
"GT-I9301I Build/LMY48Y",
|
||||
"H60-L04 Build/HDH60-L04",
|
||||
"H60-L12 Build/HDH60-L12",
|
||||
"HTC Desire 210 dual sim Build/JDQ39",
|
||||
"HTC Desire 320 Build/KOT49H",
|
||||
"HTC Desire 326G dual sim Build/KOT49H",
|
||||
"HTC Desire 516 dual sim Build/JWR66Y",
|
||||
"HTC Desire 526G dual sim Build/KOT49H",
|
||||
"HTC Desire 610 Build/KOT49H",
|
||||
"HTC Desire 616 dual sim Build/JDQ39",
|
||||
"HTC Desire 620G dual sim Build/KOT49H",
|
||||
"HTC Desire 628 Build/LMY47D",
|
||||
"HTC Desire 628 dual sim Build/LMY47D",
|
||||
"HTC Desire 728G dual sim Build/LMY47D",
|
||||
"HTC Desire EYE Build/MMB29M",
|
||||
"HTC One M8s Build/MMB29M",
|
||||
"HUAWEI MT2-L02 Build/HuaweiMT2-L02",
|
||||
"HUAWEI P8max Build/HUAWEIDAV-703L",
|
||||
"HUAWEI TIT-AL00 Build/HUAWEITIT-AL00",
|
||||
"HUAWEI TIT-U02 Build/HUAWEITIT-U02",
|
||||
"LGLS675 Build/LMY47V",
|
||||
"LGLS990 Build/KVT49L.LS990ZV4",
|
||||
"LGLS992 Build/NRD90U",
|
||||
"LGMS330 Build/LMY47V",
|
||||
"Lenovo A316i Build/JDQ39",
|
||||
"Lenovo A319 Build/MocorDroid4.4.2",
|
||||
"Lenovo A328 Build/KOT49H",
|
||||
"Lenovo A5000 Build/LRX22G",
|
||||
"Lenovo A526 Build/JDQ39",
|
||||
"Lenovo A536 Build/KOT49H",
|
||||
"Lenovo A606 Build/KOT49H",
|
||||
"Lenovo A806 Build/KOT49H",
|
||||
"Lenovo A859 Build/JDQ39",
|
||||
"Lenovo A916 Build/KOT49H",
|
||||
"Lenovo K10a40 Build/MRA58K",
|
||||
"Lenovo K30-T Build/KTU84Q",
|
||||
"Lenovo K50-t5 Build/LRX21M",
|
||||
"Lenovo K920 Build/LRX22G",
|
||||
"Lenovo P1a42 Build/LMY47V",
|
||||
"Lenovo PB1-770M Build/S100",
|
||||
"Lenovo S580 Build/JLS36C",
|
||||
"Lenovo S660 Build/KOT49H",
|
||||
"Lenovo S850 Build/KOT49H",
|
||||
"Lenovo S856 Build/KVT49L",
|
||||
"Lenovo S860 Build/JDQ39",
|
||||
"Lenovo S90-A Build/LRX22G",
|
||||
"MI 5s Plus Build/MXB48T",
|
||||
"MI NOTE Pro Build/LMY47V",
|
||||
"MX4 Pro Build/LMY48W",
|
||||
"Mi 4i Build/LRX22G",
|
||||
"Micromax A104 Build/KOT49H",
|
||||
"Micromax A106 Build/KOT49H",
|
||||
"Micromax AQ5001 Build/LRX21M",
|
||||
"Micromax D303 Build/KOT49H",
|
||||
"Micromax D320 Build/KOT49H",
|
||||
"Micromax Q324 Build/KOT49H",
|
||||
"Micromax Q380 Build/LRX22G",
|
||||
"Micromax Q392 Build/LMY47I",
|
||||
"Nexus 5X Build/NMF26F",
|
||||
"NokiaX2DS Build/JLS36C",
|
||||
"Nokia_XL Build/JZO54K",
|
||||
"Philips I908 Build/KOT49H",
|
||||
"Philips S337 Build/LMY47D",
|
||||
"Philips S388 Build/JDQ39",
|
||||
"Philips W6610 Build/JDQ39",
|
||||
"Phoenix 2 Build/JDQ39",
|
||||
"RIDGE 4G Build/LRX22G",
|
||||
"ROBBY Build/MRA58K",
|
||||
"Redmi Note 2 Build/LRX22G",
|
||||
"Rover Build/LMY47D",
|
||||
"SAMSUNG SM-A5100 Build/LMY47X",
|
||||
"SAMSUNG SM-A710F Build/MMB29K",
|
||||
"SAMSUNG SM-A710L Build/MMB29K",
|
||||
"SAMSUNG SM-G361F Build/LMY48B",
|
||||
"SAMSUNG SM-G906S Build/LRX22C",
|
||||
"SAMSUNG SM-J110H Build/KTU84P",
|
||||
"SAMSUNG SM-J500F Build/LMY48B",
|
||||
"SAMSUNG SM-J500FN Build/MMB29M",
|
||||
"SAMSUNG SM-J500G Build/LMY48B",
|
||||
"SAMSUNG SM-J500H Build/LMY48B",
|
||||
"SAMSUNG SM-J500H Build/MMB29M",
|
||||
"SAMSUNG SM-J510FN Build/MMB29M",
|
||||
"SM-A310F Build/MMB29K",
|
||||
"SM-A500FU Build/LRX22G",
|
||||
"SM-A510F Build/LMY47X",
|
||||
"SM-A700F Build/LRX22G",
|
||||
"SM-A700FD Build/MMB29M",
|
||||
"SM-A700H Build/LRX22G",
|
||||
"SM-A710F Build/LMY47X",
|
||||
"SM-C111 Build/KOT49H",
|
||||
"SM-E500F Build/LMY47X",
|
||||
"SM-E500H Build/LMY47X",
|
||||
"SM-G313HU Build/KOT49H",
|
||||
"SM-G318H Build/KTU84P",
|
||||
"SM-G355H Build/KOT49H",
|
||||
"SM-G360F Build/KTU84P",
|
||||
"SM-G360H Build/KTU84P",
|
||||
"SM-G360T Build/LMY47X",
|
||||
"SM-G530FZ Build/LRX22G",
|
||||
"SM-G530H Build/LRX22G",
|
||||
"SM-G531F Build/LMY48B",
|
||||
"SM-G7102 Build/KOT49H",
|
||||
"SM-G800F Build/LMY47X",
|
||||
"SM-G850F Build/LRX22G",
|
||||
"SM-G900F Build/MMB29M",
|
||||
"SM-G900H Build/LRX21T",
|
||||
"SM-G900I Build/LRX21T",
|
||||
"SM-G900V Build/KTU84P",
|
||||
"SM-G903F Build/MMB29K",
|
||||
"SM-G920F Build/LRX22G",
|
||||
"SM-G925F Build/MMB29K",
|
||||
"SM-G935F Build/MMB29K",
|
||||
"SM-G935V Build/MMB29M",
|
||||
"SM-J100FN Build/KTU84P",
|
||||
"SM-J100H Build/KTU84P",
|
||||
"SM-J120F Build/LMY47X",
|
||||
"SM-J120H Build/LMY47V",
|
||||
"SM-J120M Build/LMY47X",
|
||||
"SM-J200H Build/LMY48B",
|
||||
"SM-J320F Build/LMY47V",
|
||||
"SM-J500FN Build/LMY48B",
|
||||
"SM-J500M Build/LMY48B",
|
||||
"SM-J510FN Build/MMB29M",
|
||||
"SM-J700H Build/MMB29K",
|
||||
"SM-N910C Build/LMY47X",
|
||||
"SM-N920G Build/LMY47X",
|
||||
"SM-N920T Build/MMB29K",
|
||||
"VIE-L29 Build/HUAWEIVIE-L29",
|
||||
"X5pro Build/LMY47D",
|
||||
"XT1068 Build/LXB22.46-28",
|
||||
"XT1562 Build/MPD24.107-52",
|
||||
"XT1572 Build/MPHS24.107-58-1",
|
||||
"XT1585 Build/MCK24.78-13.16",
|
||||
"Z220 Build/LRX21M",
|
||||
"ZB551KL Build/LMY47V",
|
||||
"ZTE A2017G Build/MMB29M",
|
||||
"ZTE BLADE A610 Build/MRA58K",
|
||||
"ZTE Blade A5 Build/KOT49H",
|
||||
"ZTE Star 1 Build/KVT49L",
|
||||
"ZTE T320 Build/LRX21M",
|
||||
"ZTE T620 Build/LMY47D",
|
||||
"m1 note Build/KTU84P",
|
||||
"m2 note Build/LMY47D",
|
||||
"m3 note Build/LMY47I",
|
||||
"vivo V3Max Build/LMY47V"
|
||||
]
|
||||
@@ -1,757 +0,0 @@
|
||||
[
|
||||
{
|
||||
"android_versionss": [
|
||||
"v4.2.2",
|
||||
"v4.2.2"
|
||||
],
|
||||
"cpu": "Quad-core 1.3 GHz Cortex-A7",
|
||||
"dev_ids": [
|
||||
"Lenovo A5500-F Build/KOT49H"
|
||||
],
|
||||
"name": "Lenovo A8-50 A5500",
|
||||
"released": 2014,
|
||||
"resolution": [
|
||||
800,
|
||||
1280
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v4.2.2",
|
||||
"v4.2.2"
|
||||
],
|
||||
"cpu": "Quad-core 1.3 GHz Cortex-A7",
|
||||
"dev_ids": [
|
||||
"Lenovo A5500-H Build/KOT49H"
|
||||
],
|
||||
"name": "Lenovo A8-50 A5500",
|
||||
"released": 2014,
|
||||
"resolution": [
|
||||
800,
|
||||
1280
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v4.2.2",
|
||||
"v4.4.2"
|
||||
],
|
||||
"cpu": "Quad-core 1.3 GHz Cortex-A7",
|
||||
"dev_ids": [
|
||||
"Lenovo A7600-H Build/KOT49H"
|
||||
],
|
||||
"name": "Lenovo A10-70 A7600",
|
||||
"released": 2014,
|
||||
"resolution": [
|
||||
800,
|
||||
1280
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v4.2.2",
|
||||
"v4.4.2"
|
||||
],
|
||||
"cpu": "Quad-core 1.3 GHz Cortex-A7",
|
||||
"dev_ids": [
|
||||
"Lenovo A3500-H Build/KOT49H"
|
||||
],
|
||||
"name": "Lenovo A7-50 A3500",
|
||||
"released": 2014,
|
||||
"resolution": [
|
||||
800,
|
||||
1280
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v4.4.2",
|
||||
"v5.0.2"
|
||||
],
|
||||
"cpu": "Quad-core 1.2 GHz",
|
||||
"dev_ids": [
|
||||
"SM-T531 Build/KOT49H"
|
||||
],
|
||||
"name": "Samsung Galaxy Tab 4 10.1 3G",
|
||||
"released": 2014,
|
||||
"resolution": [
|
||||
1280,
|
||||
800
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v4.4.2",
|
||||
"v4.4.2"
|
||||
],
|
||||
"cpu": "Quad-core 1.2 GHz",
|
||||
"dev_ids": [
|
||||
"SM-T230 Build/KOT49H"
|
||||
],
|
||||
"name": "Samsung Galaxy Tab 4 7.0",
|
||||
"released": 2014,
|
||||
"resolution": [
|
||||
800,
|
||||
1280
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v4.4",
|
||||
"v4.4"
|
||||
],
|
||||
"cpu": "Octa-core (4x1.9 GHz Cortex-A15 & 4x1.3 GHz Cortex-A7)",
|
||||
"dev_ids": [
|
||||
"SM-P901 Build/KOT49H"
|
||||
],
|
||||
"name": "Samsung Galaxy Note Pro 12.2 3G",
|
||||
"released": 2014,
|
||||
"resolution": [
|
||||
2560,
|
||||
1600
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v4.4",
|
||||
"v4.4"
|
||||
],
|
||||
"cpu": "Quad-core 2.3 GHz Krait 400",
|
||||
"dev_ids": [
|
||||
"SM-T325 Build/KOT49H"
|
||||
],
|
||||
"name": "Samsung Galaxy Tab Pro 8.4 3G/LTE",
|
||||
"released": 2014,
|
||||
"resolution": [
|
||||
1600,
|
||||
2560
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v4.4.2",
|
||||
"v4.4.2"
|
||||
],
|
||||
"cpu": "Quad-core 1.2 GHz Cortex-A7",
|
||||
"dev_ids": [
|
||||
"SM-T231 Build/KOT49H"
|
||||
],
|
||||
"name": "Samsung Galaxy Tab 4 7.0 3G",
|
||||
"released": 2014,
|
||||
"resolution": [
|
||||
800,
|
||||
1280
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v4.4.4",
|
||||
"v4.4.4"
|
||||
],
|
||||
"cpu": "Quad-core 1.7 GHz",
|
||||
"dev_ids": [
|
||||
"Lenovo TAB 2 A10-70L Build/KTU84P"
|
||||
],
|
||||
"name": "Lenovo Tab 2 A10-70",
|
||||
"released": 2015,
|
||||
"resolution": [
|
||||
1200,
|
||||
1920
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v5.1",
|
||||
"v5.1"
|
||||
],
|
||||
"cpu": "Quad-core 1.3 GHzQuad-core 1.2 GHz - USA model",
|
||||
"dev_ids": [
|
||||
"SM-T560 Build/KTU84P"
|
||||
],
|
||||
"name": "Samsung Galaxy Tab E 9.6",
|
||||
"released": 2015,
|
||||
"resolution": [
|
||||
800,
|
||||
1280
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v4.4.4",
|
||||
"v4.4.4"
|
||||
],
|
||||
"cpu": "Quad-core 1.3 GHz",
|
||||
"dev_ids": [
|
||||
"SM-T113 Build/KTU84P"
|
||||
],
|
||||
"name": "Samsung Galaxy Tab 3 Lite 7.0 VE",
|
||||
"released": 2015,
|
||||
"resolution": [
|
||||
600,
|
||||
1024
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v5.1",
|
||||
"v5.1"
|
||||
],
|
||||
"cpu": "Quad-core 1.3 GHzQuad-core 1.2 GHz - USA model",
|
||||
"dev_ids": [
|
||||
"SM-T561 Build/KTU84P"
|
||||
],
|
||||
"name": "Samsung Galaxy Tab E 9.6",
|
||||
"released": 2015,
|
||||
"resolution": [
|
||||
800,
|
||||
1280
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v5.0",
|
||||
"v5.0"
|
||||
],
|
||||
"cpu": "Quad-core 1.33 GHz",
|
||||
"dev_ids": [
|
||||
"A3-A30 Build/LRX22G"
|
||||
],
|
||||
"name": "Acer Iconia Tab 10 A3-A30",
|
||||
"released": 2015,
|
||||
"resolution": [
|
||||
1920,
|
||||
1200
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v4.4.2",
|
||||
"v4.4.2"
|
||||
],
|
||||
"cpu": "Quad-core 1.33 GHz",
|
||||
"dev_ids": [
|
||||
"YOGA Tablet 2 Pro-1380F Build/LRX22C"
|
||||
],
|
||||
"name": "Lenovo Yoga Tablet 2 Pro",
|
||||
"released": 2014,
|
||||
"resolution": [
|
||||
2560,
|
||||
1440
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v5.0",
|
||||
"v6.0.1"
|
||||
],
|
||||
"cpu": "Quad-core 1.2 GHz",
|
||||
"dev_ids": [
|
||||
"SM-T555 Build/LRX22G"
|
||||
],
|
||||
"name": "Samsung Galaxy Tab A 9.7",
|
||||
"released": 2015,
|
||||
"resolution": [
|
||||
768,
|
||||
1024
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v5.0.2",
|
||||
"v6.0.1"
|
||||
],
|
||||
"cpu": "Octa-core (4x1.9 GHz & 4x1.3 GHz) - T710, T715Octa-core (4x1.8 GHz Cortex-A72 & 4x1.4 GHz Cortex-A53) - T719N",
|
||||
"dev_ids": [
|
||||
"SM-T710 Build/LRX22G"
|
||||
],
|
||||
"name": "Samsung Galaxy Tab S2 8.0",
|
||||
"released": 2015,
|
||||
"resolution": [
|
||||
1536,
|
||||
2048
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v4.4.2",
|
||||
"v6.0"
|
||||
],
|
||||
"cpu": "Quad-core 2.3 GHz Krait 400 (S800)Quad-core 1.9GHz Cortex-A15 & quad-core 1.3 GHz Cortex-A7 (Exynos 5420)",
|
||||
"dev_ids": [
|
||||
"SM-T705 Build/LRX22G"
|
||||
],
|
||||
"name": "Samsung Galaxy Tab S 8.4 LTE",
|
||||
"released": 2014,
|
||||
"resolution": [
|
||||
1600,
|
||||
2560
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v4.4.2",
|
||||
"v4.4.2"
|
||||
],
|
||||
"cpu": "Quad-core 1.2 GHz",
|
||||
"dev_ids": [
|
||||
"SM-T331 Build/LMY47X"
|
||||
],
|
||||
"name": "Samsung Galaxy Tab 4 8.0 3G",
|
||||
"released": 2014,
|
||||
"resolution": [
|
||||
800,
|
||||
1280
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v5.0.2",
|
||||
"v6.0.1"
|
||||
],
|
||||
"cpu": "Octa-core (4x1.9 GHz & 4x1.3 GHz) - T810, T815Octa-core (4x1.8 GHz Cortex-A72 & 4x1.4 GHz Cortex-A53) - T813N, T819N",
|
||||
"dev_ids": [
|
||||
"SM-T813 Build/MMB29M"
|
||||
],
|
||||
"name": "Samsung Galaxy Tab S2 9.7",
|
||||
"released": 2015,
|
||||
"resolution": [
|
||||
1536,
|
||||
2048
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v4.3",
|
||||
"v4.3"
|
||||
],
|
||||
"cpu": "Quad-core 1.2 GHz Cortex-A7",
|
||||
"dev_ids": [
|
||||
"Archos 80 Helium 4G Build/JLS36C"
|
||||
],
|
||||
"name": "Archos 80 Helium 4G",
|
||||
"released": 2014,
|
||||
"resolution": [
|
||||
768,
|
||||
1024
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v4.4.2",
|
||||
"v6.0.1"
|
||||
],
|
||||
"cpu": "Octa-core (4x1.9 GHz Cortex-A15 & 4x1.3 GHz Cortex-A7)",
|
||||
"dev_ids": [
|
||||
"SM-T800 Build/MMB29K"
|
||||
],
|
||||
"name": "Samsung Galaxy Tab S 10.5",
|
||||
"released": 2014,
|
||||
"resolution": [
|
||||
2560,
|
||||
1600
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v4.4.2",
|
||||
"v6.0"
|
||||
],
|
||||
"cpu": "Quad-core 2.3 GHz Krait 400 (S800)Quad-core 1.9GHz Cortex-A15 & quad-core 1.3 GHz Cortex-A7 (Exynos 5420)",
|
||||
"dev_ids": [
|
||||
"SM-T805 Build/KOT49H"
|
||||
],
|
||||
"name": "Samsung Galaxy Tab S 10.5 LTE",
|
||||
"released": 2014,
|
||||
"resolution": [
|
||||
2560,
|
||||
1600
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v4.4",
|
||||
"v4.4"
|
||||
],
|
||||
"cpu": "Quad-core 1.3 GHz",
|
||||
"dev_ids": [
|
||||
"SM-T116 Build/KTU84P"
|
||||
],
|
||||
"name": "Samsung Galaxy Tab 3 V",
|
||||
"released": 2015,
|
||||
"resolution": [
|
||||
600,
|
||||
1024
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v4.4.2",
|
||||
"v4.4.2"
|
||||
],
|
||||
"cpu": "Quad-core 1.2 GHz Cortex-A7",
|
||||
"dev_ids": [
|
||||
"SM-T235 Build/LMY47X"
|
||||
],
|
||||
"name": "Samsung Galaxy Tab 4 7.0 LTE",
|
||||
"released": 2014,
|
||||
"resolution": [
|
||||
800,
|
||||
1280
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v5.0",
|
||||
"v7.1.1"
|
||||
],
|
||||
"cpu": "Dual-core 2.3 GHz Denver",
|
||||
"dev_ids": [
|
||||
"Nexus 9 Build/MOB30M",
|
||||
"Nexus 9 Build/NRD91N"
|
||||
],
|
||||
"name": "HTC Nexus 9",
|
||||
"released": 2014,
|
||||
"resolution": [
|
||||
1536,
|
||||
2048
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v4.4",
|
||||
"v4.4"
|
||||
],
|
||||
"cpu": "Quad-core 2.3 GHz Krait 400",
|
||||
"dev_ids": [
|
||||
"SM-T525 Build/KOT49H"
|
||||
],
|
||||
"name": "Samsung Galaxy Tab Pro 10.1 LTE",
|
||||
"released": 2014,
|
||||
"resolution": [
|
||||
2560,
|
||||
1600
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v5.0.2",
|
||||
"v6.0.1"
|
||||
],
|
||||
"cpu": "Octa-core (4x1.9 GHz & 4x1.3 GHz) - T810, T815Octa-core (4x1.8 GHz Cortex-A72 & 4x1.4 GHz Cortex-A53) - T813N, T819N",
|
||||
"dev_ids": [
|
||||
"SAMSUNG SM-T815Y Build/MMB29K"
|
||||
],
|
||||
"name": "Samsung Galaxy Tab S2 9.7",
|
||||
"released": 2015,
|
||||
"resolution": [
|
||||
1536,
|
||||
2048
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v5.0",
|
||||
"v5.0"
|
||||
],
|
||||
"cpu": "Quad-core 1.3 GHz",
|
||||
"dev_ids": [
|
||||
"Lenovo TAB 2 A8-50LC Build/LRX22G",
|
||||
"Lenovo TAB 2 A8-50LC Build/LMY47D"
|
||||
],
|
||||
"name": "Lenovo Tab 2 A8-50",
|
||||
"released": 2015,
|
||||
"resolution": [
|
||||
800,
|
||||
1280
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v4.3",
|
||||
"v4.3"
|
||||
],
|
||||
"cpu": "Quad-core 1.2 GHz Cortex-A7",
|
||||
"dev_ids": [
|
||||
"MediaPad T1 8.0 Build/HuaweiMediaPad"
|
||||
],
|
||||
"name": "Huawei MediaPad T1 8.0",
|
||||
"released": 2014,
|
||||
"resolution": [
|
||||
800,
|
||||
1280
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v4.4.2",
|
||||
"v4.4.2"
|
||||
],
|
||||
"cpu": "Quad-core",
|
||||
"dev_ids": [
|
||||
"A1-713HD Build/KOT49H"
|
||||
],
|
||||
"name": "Acer Iconia Tab 7 A1-713HD",
|
||||
"released": 2014,
|
||||
"resolution": [
|
||||
800,
|
||||
1280
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v6.0",
|
||||
"v6.0"
|
||||
],
|
||||
"cpu": "Octa-core (4x2.3 GHz Cortex-A72 & 4x1.8 GHz Cortex A53)",
|
||||
"dev_ids": [
|
||||
"BTV-DL09 Build/HUAWEIBEETHOVEN-DL09"
|
||||
],
|
||||
"name": "Huawei MediaPad M3 8.4",
|
||||
"released": 2016,
|
||||
"resolution": [
|
||||
1600,
|
||||
2560
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v4.2.2",
|
||||
"v4.4.2"
|
||||
],
|
||||
"cpu": "Quad-core 1.3 GHz Cortex-A7",
|
||||
"dev_ids": [
|
||||
"Lenovo A3500-HV Build/KOT49H"
|
||||
],
|
||||
"name": "Lenovo A7-50 A3500",
|
||||
"released": 2014,
|
||||
"resolution": [
|
||||
800,
|
||||
1280
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v4.4.2",
|
||||
"v4.4.2"
|
||||
],
|
||||
"cpu": "Quad-core 1.2 GHz",
|
||||
"dev_ids": [
|
||||
"SM-T535 Build/KOT49H"
|
||||
],
|
||||
"name": "Samsung Galaxy Tab 4 10.1 LTE",
|
||||
"released": 2014,
|
||||
"resolution": [
|
||||
1280,
|
||||
800
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v5.1",
|
||||
"v5.1"
|
||||
],
|
||||
"cpu": "Octa-core (4x2.0 GHz Cortex A53 & 4x1.5 GHz Cortex-A53)",
|
||||
"dev_ids": [
|
||||
"HUAWEI M2-A01L Build/HUAWEIM2-A01L"
|
||||
],
|
||||
"name": "Huawei MediaPad M2 10.0",
|
||||
"released": 2016,
|
||||
"resolution": [
|
||||
1200,
|
||||
1920
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v4.4.2",
|
||||
"v4.4.2"
|
||||
],
|
||||
"cpu": "Quad-core 1.2 GHz Cortex-A7",
|
||||
"dev_ids": [
|
||||
"T1-701u Build/HuaweiMediaPad"
|
||||
],
|
||||
"name": "Huawei MediaPad T1 7.0 Plus",
|
||||
"released": 2016,
|
||||
"resolution": [
|
||||
600,
|
||||
1024
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v5.0.2",
|
||||
"v6.0.1"
|
||||
],
|
||||
"cpu": "Octa-core (4x1.9 GHz & 4x1.3 GHz) - T710, T715Octa-core (4x1.8 GHz Cortex-A72 & 4x1.4 GHz Cortex-A53) - T719N",
|
||||
"dev_ids": [
|
||||
"SM-T719 Build/MMB29M"
|
||||
],
|
||||
"name": "Samsung Galaxy Tab S2 8.0",
|
||||
"released": 2015,
|
||||
"resolution": [
|
||||
1536,
|
||||
2048
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v4.2",
|
||||
"v4.2"
|
||||
],
|
||||
"cpu": "Dual-core 1.2 GHz",
|
||||
"dev_ids": [
|
||||
"SM-T110 Build/JDQ39"
|
||||
],
|
||||
"name": "Samsung Galaxy Tab 3 Lite 7.0",
|
||||
"released": 2014,
|
||||
"resolution": [
|
||||
600,
|
||||
1024
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v4.4",
|
||||
"v4.4"
|
||||
],
|
||||
"cpu": "Quad-core 1.3 GHz",
|
||||
"dev_ids": [
|
||||
"SM-T116BU Build/KTU84P"
|
||||
],
|
||||
"name": "Samsung Galaxy Tab 3 V",
|
||||
"released": 2015,
|
||||
"resolution": [
|
||||
600,
|
||||
1024
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v5.1",
|
||||
"v5.1"
|
||||
],
|
||||
"cpu": "Octa-core (4x1.2 GHz Cortex-A53 & 4x1.5 GHz Cortex-A53)",
|
||||
"dev_ids": [
|
||||
"FDR-A01L Build/HuaweiMediaPad"
|
||||
],
|
||||
"name": "Huawei MediaPad T2 10.0 Pro",
|
||||
"released": 2016,
|
||||
"resolution": [
|
||||
1200,
|
||||
1920
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v4.4.2",
|
||||
"v4.4.2"
|
||||
],
|
||||
"cpu": "Quad-core 1.2 GHz",
|
||||
"dev_ids": [
|
||||
"SM-T530 Build/KOT49H",
|
||||
"SM-T530 Build/LRX22G"
|
||||
],
|
||||
"name": "Samsung Galaxy Tab 4 10.1",
|
||||
"released": 2014,
|
||||
"resolution": [
|
||||
1280,
|
||||
800
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v4.4.2",
|
||||
"v4.4.2"
|
||||
],
|
||||
"cpu": "Quad-core 1.2 GHz Cortex-A7",
|
||||
"dev_ids": [
|
||||
"SM-T365 Build/LMY47X"
|
||||
],
|
||||
"name": "Samsung Galaxy Tab Active LTE",
|
||||
"released": 2014,
|
||||
"resolution": [
|
||||
800,
|
||||
1280
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v5.0",
|
||||
"v5.0"
|
||||
],
|
||||
"cpu": "Octa-core (4x1.5 GHz Cortex-A53 & 4x2.0 GHz Cortex-A57)",
|
||||
"dev_ids": [
|
||||
"SGP771 Build/28.0.A.8.260"
|
||||
],
|
||||
"name": "Sony Xperia Z4 Tablet LTE",
|
||||
"released": 2015,
|
||||
"resolution": [
|
||||
2560,
|
||||
1600
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v5.0.2",
|
||||
"v6.0.1"
|
||||
],
|
||||
"cpu": "Octa-core (4x1.9 GHz & 4x1.3 GHz) - T710, T715Octa-core (4x1.8 GHz Cortex-A72 & 4x1.4 GHz Cortex-A53) - T719N",
|
||||
"dev_ids": [
|
||||
"SM-T715 Build/MMB29K"
|
||||
],
|
||||
"name": "Samsung Galaxy Tab S2 8.0",
|
||||
"released": 2015,
|
||||
"resolution": [
|
||||
1536,
|
||||
2048
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v4.2.2",
|
||||
"v4.4.2"
|
||||
],
|
||||
"cpu": "Quad-core 1.6 GHz Cortex-A9",
|
||||
"dev_ids": [
|
||||
"X1 7.0 Build/HuaweiMediaPad"
|
||||
],
|
||||
"name": "Huawei MediaPad X1",
|
||||
"released": 2014,
|
||||
"resolution": [
|
||||
1200,
|
||||
1920
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v4.4.2",
|
||||
"v6.0"
|
||||
],
|
||||
"cpu": "Quad-core 2.3 GHz Krait 400 (S800)Quad-core 1.9GHz Cortex-A15 & quad-core 1.3 GHz Cortex-A7 (Exynos 5420)",
|
||||
"dev_ids": [
|
||||
"SAMSUNG SM-T805 Build/LRX22G"
|
||||
],
|
||||
"name": "Samsung Galaxy Tab S 10.5 LTE",
|
||||
"released": 2014,
|
||||
"resolution": [
|
||||
2560,
|
||||
1600
|
||||
]
|
||||
},
|
||||
{
|
||||
"android_versionss": [
|
||||
"v4.4.2",
|
||||
"v6.0"
|
||||
],
|
||||
"cpu": "Octa-core (4x1.9 GHz Cortex-A15 & 4x1.3 GHz Cortex-A7)",
|
||||
"dev_ids": [
|
||||
"SM-T700 Build/LRX22G"
|
||||
],
|
||||
"name": "Samsung Galaxy Tab S 8.4",
|
||||
"released": 2014,
|
||||
"resolution": [
|
||||
1600,
|
||||
2560
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -1,52 +0,0 @@
|
||||
[
|
||||
"A1-713HD Build/KOT49H",
|
||||
"A3-A30 Build/LRX22G",
|
||||
"Archos 80 Helium 4G Build/JLS36C",
|
||||
"BTV-DL09 Build/HUAWEIBEETHOVEN-DL09",
|
||||
"FDR-A01L Build/HuaweiMediaPad",
|
||||
"HUAWEI M2-A01L Build/HUAWEIM2-A01L",
|
||||
"Lenovo A3500-H Build/KOT49H",
|
||||
"Lenovo A3500-HV Build/KOT49H",
|
||||
"Lenovo A5500-F Build/KOT49H",
|
||||
"Lenovo A5500-H Build/KOT49H",
|
||||
"Lenovo A7600-H Build/KOT49H",
|
||||
"Lenovo TAB 2 A10-70L Build/KTU84P",
|
||||
"Lenovo TAB 2 A8-50LC Build/LMY47D",
|
||||
"Lenovo TAB 2 A8-50LC Build/LRX22G",
|
||||
"MediaPad T1 8.0 Build/HuaweiMediaPad",
|
||||
"Nexus 9 Build/MOB30M",
|
||||
"Nexus 9 Build/NRD91N",
|
||||
"SAMSUNG SM-T805 Build/LRX22G",
|
||||
"SAMSUNG SM-T815Y Build/MMB29K",
|
||||
"SGP771 Build/28.0.A.8.260",
|
||||
"SM-P901 Build/KOT49H",
|
||||
"SM-T110 Build/JDQ39",
|
||||
"SM-T113 Build/KTU84P",
|
||||
"SM-T116 Build/KTU84P",
|
||||
"SM-T116BU Build/KTU84P",
|
||||
"SM-T230 Build/KOT49H",
|
||||
"SM-T231 Build/KOT49H",
|
||||
"SM-T235 Build/LMY47X",
|
||||
"SM-T325 Build/KOT49H",
|
||||
"SM-T331 Build/LMY47X",
|
||||
"SM-T365 Build/LMY47X",
|
||||
"SM-T525 Build/KOT49H",
|
||||
"SM-T530 Build/KOT49H",
|
||||
"SM-T530 Build/LRX22G",
|
||||
"SM-T531 Build/KOT49H",
|
||||
"SM-T535 Build/KOT49H",
|
||||
"SM-T555 Build/LRX22G",
|
||||
"SM-T560 Build/KTU84P",
|
||||
"SM-T561 Build/KTU84P",
|
||||
"SM-T700 Build/LRX22G",
|
||||
"SM-T705 Build/LRX22G",
|
||||
"SM-T710 Build/LRX22G",
|
||||
"SM-T715 Build/MMB29K",
|
||||
"SM-T719 Build/MMB29M",
|
||||
"SM-T800 Build/MMB29K",
|
||||
"SM-T805 Build/KOT49H",
|
||||
"SM-T813 Build/MMB29M",
|
||||
"T1-701u Build/HuaweiMediaPad",
|
||||
"X1 7.0 Build/HuaweiMediaPad",
|
||||
"YOGA Tablet 2 Pro-1380F Build/LRX22C"
|
||||
]
|
||||
@@ -1,15 +0,0 @@
|
||||
import os.path
|
||||
import json
|
||||
|
||||
|
||||
PACKAGE_DIR = os.path.dirname(os.path.realpath(__file__))
|
||||
|
||||
|
||||
def load_json_data(rel_path):
|
||||
path = os.path.join(PACKAGE_DIR, rel_path)
|
||||
with open(path, encoding='utf-8') as inp:
|
||||
return json.load(inp)
|
||||
|
||||
|
||||
SMARTPHONE_DEV_IDS = load_json_data('data/smartphone_dev_id.json')
|
||||
TABLET_DEV_IDS = load_json_data('data/tablet_dev_id.json')
|
||||
@@ -1,15 +0,0 @@
|
||||
__all__ = ('UserAgentError', 'InvalidOption')
|
||||
|
||||
|
||||
class UserAgentError(Exception):
|
||||
"""
|
||||
Base class for all errors
|
||||
raising from the user_agent library
|
||||
"""
|
||||
|
||||
|
||||
class InvalidOption(Exception):
|
||||
"""
|
||||
Raises when user call user_agent library methods
|
||||
with incorrect arguments.
|
||||
"""
|
||||
@@ -1,13 +0,0 @@
|
||||
import warnings
|
||||
|
||||
|
||||
class UserAgentDeprecationWarning(UserWarning):
|
||||
"""
|
||||
Warning category used to generate warning message
|
||||
about deprecated feature of user_agent library.
|
||||
"""
|
||||
|
||||
|
||||
def warn(msg, stacklevel=2):
|
||||
warnings.warn(msg, category=UserAgentDeprecationWarning,
|
||||
stacklevel=stacklevel)
|
||||
@@ -1 +0,0 @@
|
||||
#!/usr/bin/python
|
||||
3570
build/pyinstaller/vaderSentiment/emoji_utf8_lexicon.txt
vendored
3570
build/pyinstaller/vaderSentiment/emoji_utf8_lexicon.txt
vendored
File diff suppressed because it is too large
Load Diff
@@ -1,687 +0,0 @@
|
||||
# coding: utf-8
|
||||
# Author: C.J. Hutto
|
||||
# Thanks to George Berry for reducing the time complexity from something like O(N^4) to O(N).
|
||||
# Thanks to Ewan Klein and Pierpaolo Pantone for bringing VADER into NLTK. Those modifications were awesome.
|
||||
# For license information, see LICENSE.TXT
|
||||
|
||||
"""
|
||||
If you use the VADER sentiment analysis tools, please cite:
|
||||
Hutto, C.J. & Gilbert, E.E. (2014). VADER: A Parsimonious Rule-based Model for
|
||||
Sentiment Analysis of Social Media Text. Eighth International Conference on
|
||||
Weblogs and Social Media (ICWSM-14). Ann Arbor, MI, June 2014.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import math
|
||||
import string
|
||||
import codecs
|
||||
import json
|
||||
from itertools import product
|
||||
from inspect import getsourcefile
|
||||
from io import open
|
||||
|
||||
# ##Constants##
|
||||
|
||||
# (empirically derived mean sentiment intensity rating increase for booster words)
|
||||
B_INCR = 0.293
|
||||
B_DECR = -0.293
|
||||
|
||||
# (empirically derived mean sentiment intensity rating increase for using ALLCAPs to emphasize a word)
|
||||
C_INCR = 0.733
|
||||
N_SCALAR = -0.74
|
||||
|
||||
NEGATE = \
|
||||
["aint", "arent", "cannot", "cant", "couldnt", "darent", "didnt", "doesnt",
|
||||
"ain't", "aren't", "can't", "couldn't", "daren't", "didn't", "doesn't",
|
||||
"dont", "hadnt", "hasnt", "havent", "isnt", "mightnt", "mustnt", "neither",
|
||||
"don't", "hadn't", "hasn't", "haven't", "isn't", "mightn't", "mustn't",
|
||||
"neednt", "needn't", "never", "none", "nope", "nor", "not", "nothing", "nowhere",
|
||||
"oughtnt", "shant", "shouldnt", "uhuh", "wasnt", "werent",
|
||||
"oughtn't", "shan't", "shouldn't", "uh-uh", "wasn't", "weren't",
|
||||
"without", "wont", "wouldnt", "won't", "wouldn't", "rarely", "seldom", "despite"]
|
||||
|
||||
# booster/dampener 'intensifiers' or 'degree adverbs'
|
||||
# http://en.wiktionary.org/wiki/Category:English_degree_adverbs
|
||||
|
||||
BOOSTER_DICT = \
|
||||
{"absolutely": B_INCR, "amazingly": B_INCR, "awfully": B_INCR,
|
||||
"completely": B_INCR, "considerable": B_INCR, "considerably": B_INCR,
|
||||
"decidedly": B_INCR, "deeply": B_INCR, "effing": B_INCR, "enormous": B_INCR, "enormously": B_INCR,
|
||||
"entirely": B_INCR, "especially": B_INCR, "exceptional": B_INCR, "exceptionally": B_INCR,
|
||||
"extreme": B_INCR, "extremely": B_INCR,
|
||||
"fabulously": B_INCR, "flipping": B_INCR, "flippin": B_INCR, "frackin": B_INCR, "fracking": B_INCR,
|
||||
"fricking": B_INCR, "frickin": B_INCR, "frigging": B_INCR, "friggin": B_INCR, "fully": B_INCR,
|
||||
"fuckin": B_INCR, "fucking": B_INCR, "fuggin": B_INCR, "fugging": B_INCR,
|
||||
"greatly": B_INCR, "hella": B_INCR, "highly": B_INCR, "hugely": B_INCR,
|
||||
"incredible": B_INCR, "incredibly": B_INCR, "intensely": B_INCR,
|
||||
"major": B_INCR, "majorly": B_INCR, "more": B_INCR, "most": B_INCR, "particularly": B_INCR,
|
||||
"purely": B_INCR, "quite": B_INCR, "really": B_INCR, "remarkably": B_INCR,
|
||||
"so": B_INCR, "substantially": B_INCR,
|
||||
"thoroughly": B_INCR, "total": B_INCR, "totally": B_INCR, "tremendous": B_INCR, "tremendously": B_INCR,
|
||||
"uber": B_INCR, "unbelievably": B_INCR, "unusually": B_INCR, "utter": B_INCR, "utterly": B_INCR,
|
||||
"very": B_INCR,
|
||||
"almost": B_DECR, "barely": B_DECR, "hardly": B_DECR, "just enough": B_DECR,
|
||||
"kind of": B_DECR, "kinda": B_DECR, "kindof": B_DECR, "kind-of": B_DECR,
|
||||
"less": B_DECR, "little": B_DECR, "marginal": B_DECR, "marginally": B_DECR,
|
||||
"occasional": B_DECR, "occasionally": B_DECR, "partly": B_DECR,
|
||||
"scarce": B_DECR, "scarcely": B_DECR, "slight": B_DECR, "slightly": B_DECR, "somewhat": B_DECR,
|
||||
"sort of": B_DECR, "sorta": B_DECR, "sortof": B_DECR, "sort-of": B_DECR}
|
||||
|
||||
# check for sentiment laden idioms that do not contain lexicon words (future work, not yet implemented)
|
||||
SENTIMENT_LADEN_IDIOMS = {"cut the mustard": 2, "hand to mouth": -2,
|
||||
"back handed": -2, "blow smoke": -2, "blowing smoke": -2,
|
||||
"upper hand": 1, "break a leg": 2,
|
||||
"cooking with gas": 2, "in the black": 2, "in the red": -2,
|
||||
"on the ball": 2, "under the weather": -2}
|
||||
|
||||
# check for special case idioms and phrases containing lexicon words
|
||||
SPECIAL_CASES = {"the shit": 3, "the bomb": 3, "bad ass": 1.5, "badass": 1.5, "bus stop": 0.0,
|
||||
"yeah right": -2, "kiss of death": -1.5, "to die for": 3, "beating heart": 3.5}
|
||||
|
||||
|
||||
# #Static methods# #
|
||||
|
||||
def negated(input_words, include_nt=True):
|
||||
"""
|
||||
Determine if input contains negation words
|
||||
"""
|
||||
input_words = [str(w).lower() for w in input_words]
|
||||
neg_words = []
|
||||
neg_words.extend(NEGATE)
|
||||
for word in neg_words:
|
||||
if word in input_words:
|
||||
return True
|
||||
if include_nt:
|
||||
for word in input_words:
|
||||
if "n't" in word:
|
||||
return True
|
||||
'''if "least" in input_words:
|
||||
i = input_words.index("least")
|
||||
if i > 0 and input_words[i - 1] != "at":
|
||||
return True'''
|
||||
return False
|
||||
|
||||
|
||||
def normalize(score, alpha=15):
|
||||
"""
|
||||
Normalize the score to be between -1 and 1 using an alpha that
|
||||
approximates the max expected value
|
||||
"""
|
||||
norm_score = score / math.sqrt((score * score) + alpha)
|
||||
if norm_score < -1.0:
|
||||
return -1.0
|
||||
elif norm_score > 1.0:
|
||||
return 1.0
|
||||
else:
|
||||
return norm_score
|
||||
|
||||
|
||||
def allcap_differential(words):
|
||||
"""
|
||||
Check whether just some words in the input are ALL CAPS
|
||||
:param list words: The words to inspect
|
||||
:returns: `True` if some but not all items in `words` are ALL CAPS
|
||||
"""
|
||||
is_different = False
|
||||
allcap_words = 0
|
||||
for word in words:
|
||||
if word.isupper():
|
||||
allcap_words += 1
|
||||
cap_differential = len(words) - allcap_words
|
||||
if 0 < cap_differential < len(words):
|
||||
is_different = True
|
||||
return is_different
|
||||
|
||||
|
||||
def scalar_inc_dec(word, valence, is_cap_diff):
|
||||
"""
|
||||
Check if the preceding words increase, decrease, or negate/nullify the
|
||||
valence
|
||||
"""
|
||||
scalar = 0.0
|
||||
word_lower = word.lower()
|
||||
if word_lower in BOOSTER_DICT:
|
||||
scalar = BOOSTER_DICT[word_lower]
|
||||
if valence < 0:
|
||||
scalar *= -1
|
||||
# check if booster/dampener word is in ALLCAPS (while others aren't)
|
||||
if word.isupper() and is_cap_diff:
|
||||
if valence > 0:
|
||||
scalar += C_INCR
|
||||
else:
|
||||
scalar -= C_INCR
|
||||
return scalar
|
||||
|
||||
|
||||
class SentiText(object):
|
||||
"""
|
||||
Identify sentiment-relevant string-level properties of input text.
|
||||
"""
|
||||
|
||||
def __init__(self, text):
|
||||
if not isinstance(text, str):
|
||||
text = str(text).encode('utf-8')
|
||||
self.text = text
|
||||
self.words_and_emoticons = self._words_and_emoticons()
|
||||
# doesn't separate words from\
|
||||
# adjacent punctuation (keeps emoticons & contractions)
|
||||
self.is_cap_diff = allcap_differential(self.words_and_emoticons)
|
||||
|
||||
@staticmethod
|
||||
def _strip_punc_if_word(token):
|
||||
"""
|
||||
Removes all trailing and leading punctuation
|
||||
If the resulting string has two or fewer characters,
|
||||
then it was likely an emoticon, so return original string
|
||||
(ie ":)" stripped would be "", so just return ":)"
|
||||
"""
|
||||
stripped = token.strip(string.punctuation)
|
||||
if len(stripped) <= 2:
|
||||
return token
|
||||
return stripped
|
||||
|
||||
def _words_and_emoticons(self):
|
||||
"""
|
||||
Removes leading and trailing puncutation
|
||||
Leaves contractions and most emoticons
|
||||
Does not preserve punc-plus-letter emoticons (e.g. :D)
|
||||
"""
|
||||
wes = self.text.split()
|
||||
stripped = list(map(self._strip_punc_if_word, wes))
|
||||
return stripped
|
||||
|
||||
class SentimentIntensityAnalyzer(object):
|
||||
"""
|
||||
Give a sentiment intensity score to sentences.
|
||||
"""
|
||||
|
||||
def __init__(self, lexicon_file="vader_lexicon.txt", emoji_lexicon="emoji_utf8_lexicon.txt"):
|
||||
_this_module_file_path_ = os.path.abspath(getsourcefile(lambda: 0))
|
||||
lexicon_full_filepath = os.path.join(os.path.dirname(_this_module_file_path_), lexicon_file)
|
||||
with codecs.open(lexicon_full_filepath, encoding='utf-8') as f:
|
||||
self.lexicon_full_filepath = f.read()
|
||||
self.lexicon = self.make_lex_dict()
|
||||
|
||||
emoji_full_filepath = os.path.join(os.path.dirname(_this_module_file_path_), emoji_lexicon)
|
||||
with codecs.open(emoji_full_filepath, encoding='utf-8') as f:
|
||||
self.emoji_full_filepath = f.read()
|
||||
self.emojis = self.make_emoji_dict()
|
||||
|
||||
def make_lex_dict(self):
|
||||
"""
|
||||
Convert lexicon file to a dictionary
|
||||
"""
|
||||
lex_dict = {}
|
||||
for line in self.lexicon_full_filepath.rstrip('\n').split('\n'):
|
||||
if not line:
|
||||
continue
|
||||
(word, measure) = line.strip().split('\t')[0:2]
|
||||
lex_dict[word] = float(measure)
|
||||
return lex_dict
|
||||
|
||||
def make_emoji_dict(self):
|
||||
"""
|
||||
Convert emoji lexicon file to a dictionary
|
||||
"""
|
||||
emoji_dict = {}
|
||||
for line in self.emoji_full_filepath.rstrip('\n').split('\n'):
|
||||
(emoji, description) = line.strip().split('\t')[0:2]
|
||||
emoji_dict[emoji] = description
|
||||
return emoji_dict
|
||||
|
||||
def polarity_scores(self, text):
|
||||
"""
|
||||
Return a float for sentiment strength based on the input text.
|
||||
Positive values are positive valence, negative value are negative
|
||||
valence.
|
||||
"""
|
||||
# convert emojis to their textual descriptions
|
||||
text_no_emoji = ""
|
||||
prev_space = True
|
||||
for chr in text:
|
||||
if chr in self.emojis:
|
||||
# get the textual description
|
||||
description = self.emojis[chr]
|
||||
if not prev_space:
|
||||
text_no_emoji += ' '
|
||||
text_no_emoji += description
|
||||
prev_space = False
|
||||
else:
|
||||
text_no_emoji += chr
|
||||
prev_space = chr == ' '
|
||||
text = text_no_emoji.strip()
|
||||
|
||||
sentitext = SentiText(text)
|
||||
|
||||
sentiments = []
|
||||
words_and_emoticons = sentitext.words_and_emoticons
|
||||
for i, item in enumerate(words_and_emoticons):
|
||||
valence = 0
|
||||
# check for vader_lexicon words that may be used as modifiers or negations
|
||||
if item.lower() in BOOSTER_DICT:
|
||||
sentiments.append(valence)
|
||||
continue
|
||||
if (i < len(words_and_emoticons) - 1 and item.lower() == "kind" and
|
||||
words_and_emoticons[i + 1].lower() == "of"):
|
||||
sentiments.append(valence)
|
||||
continue
|
||||
|
||||
sentiments = self.sentiment_valence(valence, sentitext, item, i, sentiments)
|
||||
|
||||
sentiments = self._but_check(words_and_emoticons, sentiments)
|
||||
|
||||
valence_dict = self.score_valence(sentiments, text)
|
||||
|
||||
return valence_dict
|
||||
|
||||
def sentiment_valence(self, valence, sentitext, item, i, sentiments):
|
||||
is_cap_diff = sentitext.is_cap_diff
|
||||
words_and_emoticons = sentitext.words_and_emoticons
|
||||
item_lowercase = item.lower()
|
||||
if item_lowercase in self.lexicon:
|
||||
# get the sentiment valence
|
||||
valence = self.lexicon[item_lowercase]
|
||||
|
||||
# check for "no" as negation for an adjacent lexicon item vs "no" as its own stand-alone lexicon item
|
||||
if item_lowercase == "no" and i != len(words_and_emoticons)-1 and words_and_emoticons[i + 1].lower() in self.lexicon:
|
||||
# don't use valence of "no" as a lexicon item. Instead set it's valence to 0.0 and negate the next item
|
||||
valence = 0.0
|
||||
if (i > 0 and words_and_emoticons[i - 1].lower() == "no") \
|
||||
or (i > 1 and words_and_emoticons[i - 2].lower() == "no") \
|
||||
or (i > 2 and words_and_emoticons[i - 3].lower() == "no" and words_and_emoticons[i - 1].lower() in ["or", "nor"] ):
|
||||
valence = self.lexicon[item_lowercase] * N_SCALAR
|
||||
|
||||
# check if sentiment laden word is in ALL CAPS (while others aren't)
|
||||
if item.isupper() and is_cap_diff:
|
||||
if valence > 0:
|
||||
valence += C_INCR
|
||||
else:
|
||||
valence -= C_INCR
|
||||
|
||||
for start_i in range(0, 3):
|
||||
# dampen the scalar modifier of preceding words and emoticons
|
||||
# (excluding the ones that immediately preceed the item) based
|
||||
# on their distance from the current item.
|
||||
if i > start_i and words_and_emoticons[i - (start_i + 1)].lower() not in self.lexicon:
|
||||
s = scalar_inc_dec(words_and_emoticons[i - (start_i + 1)], valence, is_cap_diff)
|
||||
if start_i == 1 and s != 0:
|
||||
s = s * 0.95
|
||||
if start_i == 2 and s != 0:
|
||||
s = s * 0.9
|
||||
valence = valence + s
|
||||
valence = self._negation_check(valence, words_and_emoticons, start_i, i)
|
||||
if start_i == 2:
|
||||
valence = self._special_idioms_check(valence, words_and_emoticons, i)
|
||||
|
||||
valence = self._least_check(valence, words_and_emoticons, i)
|
||||
sentiments.append(valence)
|
||||
return sentiments
|
||||
|
||||
def _least_check(self, valence, words_and_emoticons, i):
|
||||
# check for negation case using "least"
|
||||
if i > 1 and words_and_emoticons[i - 1].lower() not in self.lexicon \
|
||||
and words_and_emoticons[i - 1].lower() == "least":
|
||||
if words_and_emoticons[i - 2].lower() != "at" and words_and_emoticons[i - 2].lower() != "very":
|
||||
valence = valence * N_SCALAR
|
||||
elif i > 0 and words_and_emoticons[i - 1].lower() not in self.lexicon \
|
||||
and words_and_emoticons[i - 1].lower() == "least":
|
||||
valence = valence * N_SCALAR
|
||||
return valence
|
||||
|
||||
@staticmethod
|
||||
def _but_check(words_and_emoticons, sentiments):
|
||||
# check for modification in sentiment due to contrastive conjunction 'but'
|
||||
words_and_emoticons_lower = [str(w).lower() for w in words_and_emoticons]
|
||||
if 'but' in words_and_emoticons_lower:
|
||||
bi = words_and_emoticons_lower.index('but')
|
||||
for sentiment in sentiments:
|
||||
si = sentiments.index(sentiment)
|
||||
if si < bi:
|
||||
sentiments.pop(si)
|
||||
sentiments.insert(si, sentiment * 0.5)
|
||||
elif si > bi:
|
||||
sentiments.pop(si)
|
||||
sentiments.insert(si, sentiment * 1.5)
|
||||
return sentiments
|
||||
|
||||
@staticmethod
|
||||
def _special_idioms_check(valence, words_and_emoticons, i):
|
||||
words_and_emoticons_lower = [str(w).lower() for w in words_and_emoticons]
|
||||
onezero = "{0} {1}".format(words_and_emoticons_lower[i - 1], words_and_emoticons_lower[i])
|
||||
|
||||
twoonezero = "{0} {1} {2}".format(words_and_emoticons_lower[i - 2],
|
||||
words_and_emoticons_lower[i - 1], words_and_emoticons_lower[i])
|
||||
|
||||
twoone = "{0} {1}".format(words_and_emoticons_lower[i - 2], words_and_emoticons_lower[i - 1])
|
||||
|
||||
threetwoone = "{0} {1} {2}".format(words_and_emoticons_lower[i - 3],
|
||||
words_and_emoticons_lower[i - 2], words_and_emoticons_lower[i - 1])
|
||||
|
||||
threetwo = "{0} {1}".format(words_and_emoticons_lower[i - 3], words_and_emoticons_lower[i - 2])
|
||||
|
||||
sequences = [onezero, twoonezero, twoone, threetwoone, threetwo]
|
||||
|
||||
for seq in sequences:
|
||||
if seq in SPECIAL_CASES:
|
||||
valence = SPECIAL_CASES[seq]
|
||||
break
|
||||
|
||||
if len(words_and_emoticons_lower) - 1 > i:
|
||||
zeroone = "{0} {1}".format(words_and_emoticons_lower[i], words_and_emoticons_lower[i + 1])
|
||||
if zeroone in SPECIAL_CASES:
|
||||
valence = SPECIAL_CASES[zeroone]
|
||||
if len(words_and_emoticons_lower) - 1 > i + 1:
|
||||
zeroonetwo = "{0} {1} {2}".format(words_and_emoticons_lower[i], words_and_emoticons_lower[i + 1],
|
||||
words_and_emoticons_lower[i + 2])
|
||||
if zeroonetwo in SPECIAL_CASES:
|
||||
valence = SPECIAL_CASES[zeroonetwo]
|
||||
|
||||
# check for booster/dampener bi-grams such as 'sort of' or 'kind of'
|
||||
n_grams = [threetwoone, threetwo, twoone]
|
||||
for n_gram in n_grams:
|
||||
if n_gram in BOOSTER_DICT:
|
||||
valence = valence + BOOSTER_DICT[n_gram]
|
||||
return valence
|
||||
|
||||
@staticmethod
|
||||
def _sentiment_laden_idioms_check(valence, senti_text_lower):
|
||||
# Future Work
|
||||
# check for sentiment laden idioms that don't contain a lexicon word
|
||||
idioms_valences = []
|
||||
for idiom in SENTIMENT_LADEN_IDIOMS:
|
||||
if idiom in senti_text_lower:
|
||||
print(idiom, senti_text_lower)
|
||||
valence = SENTIMENT_LADEN_IDIOMS[idiom]
|
||||
idioms_valences.append(valence)
|
||||
if len(idioms_valences) > 0:
|
||||
valence = sum(idioms_valences) / float(len(idioms_valences))
|
||||
return valence
|
||||
|
||||
@staticmethod
|
||||
def _negation_check(valence, words_and_emoticons, start_i, i):
|
||||
words_and_emoticons_lower = [str(w).lower() for w in words_and_emoticons]
|
||||
if start_i == 0:
|
||||
if negated([words_and_emoticons_lower[i - (start_i + 1)]]): # 1 word preceding lexicon word (w/o stopwords)
|
||||
valence = valence * N_SCALAR
|
||||
if start_i == 1:
|
||||
if words_and_emoticons_lower[i - 2] == "never" and \
|
||||
(words_and_emoticons_lower[i - 1] == "so" or
|
||||
words_and_emoticons_lower[i - 1] == "this"):
|
||||
valence = valence * 1.25
|
||||
elif words_and_emoticons_lower[i - 2] == "without" and \
|
||||
words_and_emoticons_lower[i - 1] == "doubt":
|
||||
valence = valence
|
||||
elif negated([words_and_emoticons_lower[i - (start_i + 1)]]): # 2 words preceding the lexicon word position
|
||||
valence = valence * N_SCALAR
|
||||
if start_i == 2:
|
||||
if words_and_emoticons_lower[i - 3] == "never" and \
|
||||
(words_and_emoticons_lower[i - 2] == "so" or words_and_emoticons_lower[i - 2] == "this") or \
|
||||
(words_and_emoticons_lower[i - 1] == "so" or words_and_emoticons_lower[i - 1] == "this"):
|
||||
valence = valence * 1.25
|
||||
elif words_and_emoticons_lower[i - 3] == "without" and \
|
||||
(words_and_emoticons_lower[i - 2] == "doubt" or words_and_emoticons_lower[i - 1] == "doubt"):
|
||||
valence = valence
|
||||
elif negated([words_and_emoticons_lower[i - (start_i + 1)]]): # 3 words preceding the lexicon word position
|
||||
valence = valence * N_SCALAR
|
||||
return valence
|
||||
|
||||
def _punctuation_emphasis(self, text):
|
||||
# add emphasis from exclamation points and question marks
|
||||
ep_amplifier = self._amplify_ep(text)
|
||||
qm_amplifier = self._amplify_qm(text)
|
||||
punct_emph_amplifier = ep_amplifier + qm_amplifier
|
||||
return punct_emph_amplifier
|
||||
|
||||
@staticmethod
|
||||
def _amplify_ep(text):
|
||||
# check for added emphasis resulting from exclamation points (up to 4 of them)
|
||||
ep_count = text.count("!")
|
||||
if ep_count > 4:
|
||||
ep_count = 4
|
||||
# (empirically derived mean sentiment intensity rating increase for
|
||||
# exclamation points)
|
||||
ep_amplifier = ep_count * 0.292
|
||||
return ep_amplifier
|
||||
|
||||
@staticmethod
|
||||
def _amplify_qm(text):
|
||||
# check for added emphasis resulting from question marks (2 or 3+)
|
||||
qm_count = text.count("?")
|
||||
qm_amplifier = 0
|
||||
if qm_count > 1:
|
||||
if qm_count <= 3:
|
||||
# (empirically derived mean sentiment intensity rating increase for
|
||||
# question marks)
|
||||
qm_amplifier = qm_count * 0.18
|
||||
else:
|
||||
qm_amplifier = 0.96
|
||||
return qm_amplifier
|
||||
|
||||
@staticmethod
|
||||
def _sift_sentiment_scores(sentiments):
|
||||
# want separate positive versus negative sentiment scores
|
||||
pos_sum = 0.0
|
||||
neg_sum = 0.0
|
||||
neu_count = 0
|
||||
for sentiment_score in sentiments:
|
||||
if sentiment_score > 0:
|
||||
pos_sum += (float(sentiment_score) + 1) # compensates for neutral words that are counted as 1
|
||||
if sentiment_score < 0:
|
||||
neg_sum += (float(sentiment_score) - 1) # when used with math.fabs(), compensates for neutrals
|
||||
if sentiment_score == 0:
|
||||
neu_count += 1
|
||||
return pos_sum, neg_sum, neu_count
|
||||
|
||||
def score_valence(self, sentiments, text):
|
||||
if sentiments:
|
||||
sum_s = float(sum(sentiments))
|
||||
# compute and add emphasis from punctuation in text
|
||||
punct_emph_amplifier = self._punctuation_emphasis(text)
|
||||
if sum_s > 0:
|
||||
sum_s += punct_emph_amplifier
|
||||
elif sum_s < 0:
|
||||
sum_s -= punct_emph_amplifier
|
||||
|
||||
compound = normalize(sum_s)
|
||||
# discriminate between positive, negative and neutral sentiment scores
|
||||
pos_sum, neg_sum, neu_count = self._sift_sentiment_scores(sentiments)
|
||||
|
||||
if pos_sum > math.fabs(neg_sum):
|
||||
pos_sum += punct_emph_amplifier
|
||||
elif pos_sum < math.fabs(neg_sum):
|
||||
neg_sum -= punct_emph_amplifier
|
||||
|
||||
total = pos_sum + math.fabs(neg_sum) + neu_count
|
||||
pos = math.fabs(pos_sum / total)
|
||||
neg = math.fabs(neg_sum / total)
|
||||
neu = math.fabs(neu_count / total)
|
||||
|
||||
else:
|
||||
compound = 0.0
|
||||
pos = 0.0
|
||||
neg = 0.0
|
||||
neu = 0.0
|
||||
|
||||
sentiment_dict = \
|
||||
{"neg": round(neg, 3),
|
||||
"neu": round(neu, 3),
|
||||
"pos": round(pos, 3),
|
||||
"compound": round(compound, 4)}
|
||||
|
||||
return sentiment_dict
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# --- examples -------
|
||||
sentences = ["VADER is smart, handsome, and funny.", # positive sentence example
|
||||
"VADER is smart, handsome, and funny!",
|
||||
# punctuation emphasis handled correctly (sentiment intensity adjusted)
|
||||
"VADER is very smart, handsome, and funny.",
|
||||
# booster words handled correctly (sentiment intensity adjusted)
|
||||
"VADER is VERY SMART, handsome, and FUNNY.", # emphasis for ALLCAPS handled
|
||||
"VADER is VERY SMART, handsome, and FUNNY!!!",
|
||||
# combination of signals - VADER appropriately adjusts intensity
|
||||
"VADER is VERY SMART, uber handsome, and FRIGGIN FUNNY!!!",
|
||||
# booster words & punctuation make this close to ceiling for score
|
||||
"VADER is not smart, handsome, nor funny.", # negation sentence example
|
||||
"The book was good.", # positive sentence
|
||||
"At least it isn't a horrible book.", # negated negative sentence with contraction
|
||||
"The book was only kind of good.",
|
||||
# qualified positive sentence is handled correctly (intensity adjusted)
|
||||
"The plot was good, but the characters are uncompelling and the dialog is not great.",
|
||||
# mixed negation sentence
|
||||
"Today SUX!", # negative slang with capitalization emphasis
|
||||
"Today only kinda sux! But I'll get by, lol",
|
||||
# mixed sentiment example with slang and constrastive conjunction "but"
|
||||
"Make sure you :) or :D today!", # emoticons handled
|
||||
"Catch utf-8 emoji such as 💘 and 💋 and 😁", # emojis handled
|
||||
"Not bad at all" # Capitalized negation
|
||||
]
|
||||
|
||||
analyzer = SentimentIntensityAnalyzer()
|
||||
|
||||
print("----------------------------------------------------")
|
||||
print(" - Analyze typical example cases, including handling of:")
|
||||
print(" -- negations")
|
||||
print(" -- punctuation emphasis & punctuation flooding")
|
||||
print(" -- word-shape as emphasis (capitalization difference)")
|
||||
print(" -- degree modifiers (intensifiers such as 'very' and dampeners such as 'kind of')")
|
||||
print(" -- slang words as modifiers such as 'uber' or 'friggin' or 'kinda'")
|
||||
print(" -- contrastive conjunction 'but' indicating a shift in sentiment; sentiment of later text is dominant")
|
||||
print(" -- use of contractions as negations")
|
||||
print(" -- sentiment laden emoticons such as :) and :D")
|
||||
print(" -- utf-8 encoded emojis such as 💘 and 💋 and 😁")
|
||||
print(" -- sentiment laden slang words (e.g., 'sux')")
|
||||
print(" -- sentiment laden initialisms and acronyms (for example: 'lol') \n")
|
||||
for sentence in sentences:
|
||||
vs = analyzer.polarity_scores(sentence)
|
||||
print("{:-<65} {}".format(sentence, str(vs)))
|
||||
print("----------------------------------------------------")
|
||||
print(" - About the scoring: ")
|
||||
print(""" -- The 'compound' score is computed by summing the valence scores of each word in the lexicon, adjusted
|
||||
according to the rules, and then normalized to be between -1 (most extreme negative) and +1 (most extreme positive).
|
||||
This is the most useful metric if you want a single unidimensional measure of sentiment for a given sentence.
|
||||
Calling it a 'normalized, weighted composite score' is accurate.""")
|
||||
print(""" -- The 'pos', 'neu', and 'neg' scores are ratios for proportions of text that fall in each category (so these
|
||||
should all add up to be 1... or close to it with float operation). These are the most useful metrics if
|
||||
you want multidimensional measures of sentiment for a given sentence.""")
|
||||
print("----------------------------------------------------")
|
||||
|
||||
# input("\nPress Enter to continue the demo...\n") # for DEMO purposes...
|
||||
|
||||
tricky_sentences = ["Sentiment analysis has never been good.",
|
||||
"Sentiment analysis has never been this good!",
|
||||
"Most automated sentiment analysis tools are shit.",
|
||||
"With VADER, sentiment analysis is the shit!",
|
||||
"Other sentiment analysis tools can be quite bad.",
|
||||
"On the other hand, VADER is quite bad ass",
|
||||
"VADER is such a badass!", # slang with punctuation emphasis
|
||||
"Without a doubt, excellent idea.",
|
||||
"Roger Dodger is one of the most compelling variations on this theme.",
|
||||
"Roger Dodger is at least compelling as a variation on the theme.",
|
||||
"Roger Dodger is one of the least compelling variations on this theme.",
|
||||
"Not such a badass after all.", # Capitalized negation with slang
|
||||
"Without a doubt, an excellent idea." # "without {any} doubt" as negation
|
||||
]
|
||||
print("----------------------------------------------------")
|
||||
print(" - Analyze examples of tricky sentences that cause trouble to other sentiment analysis tools.")
|
||||
print(" -- special case idioms - e.g., 'never good' vs 'never this good', or 'bad' vs 'bad ass'.")
|
||||
print(" -- special uses of 'least' as negation versus comparison \n")
|
||||
for sentence in tricky_sentences:
|
||||
vs = analyzer.polarity_scores(sentence)
|
||||
print("{:-<69} {}".format(sentence, str(vs)))
|
||||
print("----------------------------------------------------")
|
||||
|
||||
# input("\nPress Enter to continue the demo...\n") # for DEMO purposes...
|
||||
|
||||
print("----------------------------------------------------")
|
||||
print(
|
||||
" - VADER works best when analysis is done at the sentence level (but it can work on single words or entire novels).")
|
||||
paragraph = "It was one of the worst movies I've seen, despite good reviews. Unbelievably bad acting!! Poor direction. VERY poor production. The movie was bad. Very bad movie. VERY BAD movie!"
|
||||
print(" -- For example, given the following paragraph text from a hypothetical movie review:\n\t'{}'".format(
|
||||
paragraph))
|
||||
print(
|
||||
" -- You could use NLTK to break the paragraph into sentence tokens for VADER, then average the results for the paragraph like this: \n")
|
||||
# simple example to tokenize paragraph into sentences for VADER
|
||||
from nltk import tokenize
|
||||
|
||||
sentence_list = tokenize.sent_tokenize(paragraph)
|
||||
paragraphSentiments = 0.0
|
||||
for sentence in sentence_list:
|
||||
vs = analyzer.polarity_scores(sentence)
|
||||
print("{:-<69} {}".format(sentence, str(vs["compound"])))
|
||||
paragraphSentiments += vs["compound"]
|
||||
print("AVERAGE SENTIMENT FOR PARAGRAPH: \t" + str(round(paragraphSentiments / len(sentence_list), 4)))
|
||||
print("----------------------------------------------------")
|
||||
|
||||
# input("\nPress Enter to continue the demo...\n") # for DEMO purposes...
|
||||
|
||||
print("----------------------------------------------------")
|
||||
print(" - Analyze sentiment of IMAGES/VIDEO data based on annotation 'tags' or image labels. \n")
|
||||
conceptList = ["balloons", "cake", "candles", "happy birthday", "friends", "laughing", "smiling", "party"]
|
||||
conceptSentiments = 0.0
|
||||
for concept in conceptList:
|
||||
vs = analyzer.polarity_scores(concept)
|
||||
print("{:-<15} {}".format(concept, str(vs['compound'])))
|
||||
conceptSentiments += vs["compound"]
|
||||
print("AVERAGE SENTIMENT OF TAGS/LABELS: \t" + str(round(conceptSentiments / len(conceptList), 4)))
|
||||
print("\t")
|
||||
conceptList = ["riot", "fire", "fight", "blood", "mob", "war", "police", "tear gas"]
|
||||
conceptSentiments = 0.0
|
||||
for concept in conceptList:
|
||||
vs = analyzer.polarity_scores(concept)
|
||||
print("{:-<15} {}".format(concept, str(vs['compound'])))
|
||||
conceptSentiments += vs["compound"]
|
||||
print("AVERAGE SENTIMENT OF TAGS/LABELS: \t" + str(round(conceptSentiments / len(conceptList), 4)))
|
||||
print("----------------------------------------------------")
|
||||
|
||||
# input("\nPress Enter to continue the demo...") # for DEMO purposes...
|
||||
|
||||
do_translate = input(
|
||||
"\nWould you like to run VADER demo examples with NON-ENGLISH text? \n (Note: requires Internet access and uses the 'requests' library) \n Type 'y' or 'n', then press Enter: ")
|
||||
if do_translate.lower().lstrip().__contains__("y"):
|
||||
import requests
|
||||
print("\n----------------------------------------------------")
|
||||
print(" - Analyze sentiment of NON ENGLISH text...for example:")
|
||||
print(" -- French, German, Spanish, Italian, Russian, Japanese, Arabic, Chinese(Simplified) , Chinese(Traditional)")
|
||||
print(" -- many other languages supported. \n")
|
||||
languages = ["English", "French", "German", "Spanish", "Italian", "Russian", "Japanese", "Arabic", "Chinese(Simplified)", "Chinese(Traditional)"]
|
||||
language_codes = ["en", "fr", "de", "es", "it", "ru", "ja", "ar", "zh-CN", "zh-TW"]
|
||||
nonEnglish_sentences = ["I'm surprised to see just how amazingly helpful VADER is!",
|
||||
"Je suis surpris de voir comment VADER est incroyablement utile !",
|
||||
"Ich bin überrascht zu sehen, nur wie erstaunlich nützlich VADER!",
|
||||
"Me sorprende ver sólo cómo increíblemente útil VADER!",
|
||||
"Sono sorpreso di vedere solo come incredibilmente utile VADER è!",
|
||||
"Я удивлен увидеть, как раз как удивительно полезно ВЕЙДЕРА!",
|
||||
"私はちょうどどのように驚くほど役に立つベイダーを見て驚いています!",
|
||||
"أنا مندهش لرؤية فقط كيف مثير للدهشة فيدر فائدة!",
|
||||
"我很惊讶地看到VADER是如此有用!",
|
||||
"我很驚訝地看到VADER是如此有用!"
|
||||
]
|
||||
for sentence in nonEnglish_sentences:
|
||||
to_lang = "en"
|
||||
from_lang = language_codes[nonEnglish_sentences.index(sentence)]
|
||||
if (from_lang == "en") or (from_lang == "en-US"):
|
||||
translation = sentence
|
||||
translator_name = "No translation needed"
|
||||
else: # please note usage limits for My Memory Translation Service: http://mymemory.translated.net/doc/usagelimits.php
|
||||
# using MY MEMORY NET http://mymemory.translated.net
|
||||
api_url = "http://mymemory.translated.net/api/get?q={}&langpair={}|{}".format(sentence, from_lang,
|
||||
to_lang)
|
||||
hdrs = {
|
||||
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
|
||||
'Accept-Encoding': 'none',
|
||||
'Accept-Language': 'en-US,en;q=0.8',
|
||||
'Connection': 'keep-alive'}
|
||||
response = requests.get(api_url, headers=hdrs)
|
||||
response_json = json.loads(response.text)
|
||||
translation = response_json["responseData"]["translatedText"]
|
||||
translator_name = "MemoryNet Translation Service"
|
||||
vs = analyzer.polarity_scores(translation)
|
||||
print("- {: <8}: {: <69}\t {} ({})".format(languages[nonEnglish_sentences.index(sentence)], sentence,
|
||||
str(vs['compound']), translator_name))
|
||||
print("----------------------------------------------------")
|
||||
|
||||
print("\n\n Demo Done!")
|
||||
7520
build/pyinstaller/vaderSentiment/vader_lexicon.txt
vendored
7520
build/pyinstaller/vaderSentiment/vader_lexicon.txt
vendored
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user