1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889 |
- # -*- coding: utf-8 -*-
- import logging
- import os
- import sys
- log_console = logging.StreamHandler(sys.stderr)
- default_logger = logging.getLogger(__name__)
- default_logger.setLevel(logging.DEBUG)
- def setLogLevel(log_level):
- default_logger.setLevel(log_level)
- check_paddle_install = {'is_paddle_installed': False}
- try:
- import pkg_resources
- get_module_res = lambda *res: pkg_resources.resource_stream(__name__,
- os.path.join(*res))
- except ImportError:
- get_module_res = lambda *res: open(os.path.normpath(os.path.join(
- os.getcwd(), os.path.dirname(__file__), *res)), 'rb')
- def enable_paddle():
- try:
- import paddle
- except ImportError:
- default_logger.debug("Installing paddle-tiny, please wait a minute......")
- os.system("pip install paddlepaddle-tiny")
- try:
- import paddle
- except ImportError:
- default_logger.debug(
- "Import paddle error, please use command to install: pip install paddlepaddle-tiny==1.6.1."
- "Now, back to jieba basic cut......")
- if paddle.__version__ < '1.6.1':
- default_logger.debug("Find your own paddle version doesn't satisfy the minimum requirement (1.6.1), "
- "please install paddle tiny by 'pip install --upgrade paddlepaddle-tiny', "
- "or upgrade paddle full version by "
- "'pip install --upgrade paddlepaddle (-gpu for GPU version)' ")
- else:
- try:
- import jieba.lac_small.predict as predict
- default_logger.debug("Paddle enabled successfully......")
- check_paddle_install['is_paddle_installed'] = True
- except ImportError:
- default_logger.debug("Import error, cannot find paddle.fluid and jieba.lac_small.predict module. "
- "Now, back to jieba basic cut......")
- PY2 = sys.version_info[0] == 2
- default_encoding = sys.getfilesystemencoding()
- if PY2:
- text_type = unicode
- string_types = (str, unicode)
- iterkeys = lambda d: d.iterkeys()
- itervalues = lambda d: d.itervalues()
- iteritems = lambda d: d.iteritems()
- else:
- text_type = str
- string_types = (str,)
- xrange = range
- iterkeys = lambda d: iter(d.keys())
- itervalues = lambda d: iter(d.values())
- iteritems = lambda d: iter(d.items())
- def strdecode(sentence):
- if not isinstance(sentence, text_type):
- try:
- sentence = sentence.decode('utf-8')
- except UnicodeDecodeError:
- sentence = sentence.decode('gbk', 'ignore')
- return sentence
- def resolve_filename(f):
- try:
- return f.name
- except AttributeError:
- return repr(f)
|