pip._vendor.pygments.lexer Namespace Reference

Data Structures
class	_inherit
class	_PseudoMatch
class	_This
class	combined
class	default
class	DelegatingLexer
class	ExtendedRegexLexer
class	include
class	Lexer
class	LexerContext
class	LexerMeta
class	ProfilingRegexLexer
class	ProfilingRegexLexerMeta
class	RegexLexer
class	RegexLexerMeta
class	words

Functions
	bygroups (*args)
	using (_other, **kwargs)
	do_insertions (insertions, tokens)

Variables
	line_re = re.compile('.*?\n')
list	_encoding_map
	_default_analyse = staticmethod(lambda x: 0.0)
	inherit = _inherit()
	this = _This()

Detailed Description

    pygments.lexer
    ~~~~~~~~~~~~~~

    Base lexer classes.

    :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.

Function Documentation

bygroups()

bygroups

(

*

args

)

Callback that yields multiple actions for each group in the match.

Definition at line 365 of file lexer.py.

def bygroups(*args):
    """
    Callback that yields multiple actions for each group in the match.
    """
    def callback(lexer, match, ctx=None):
        for i, action in enumerate(args):
            if action is None:
                continue
            elif type(action) is _TokenType:
                data = match.group(i + 1)
                if data:
                    yield match.start(i + 1), action, data
            else:
                data = match.group(i + 1)
                if data is not None:
                    if ctx:
                        ctx.pos = match.start(i + 1)
                    for item in action(lexer,
                                       _PseudoMatch(match.start(i + 1), data), ctx):
                        if item:
                            yield item
        if ctx:
            ctx.pos = match.end()
    return callback
 
 

References action, and i.

do_insertions()

do_insertions	(		insertions,
			tokens
	)

Helper for lexers which must combine the results of several
sublexers.

``insertions`` is a list of ``(index, itokens)`` pairs.
Each ``itokens`` iterable should be inserted at position
``index`` into the token stream given by the ``tokens``
argument.

The result is a combined token stream.

TODO: clean up the code here.

Definition at line 831 of file lexer.py.

def do_insertions(insertions, tokens):
    """
    Helper for lexers which must combine the results of several
    sublexers.
 
    ``insertions`` is a list of ``(index, itokens)`` pairs.
    Each ``itokens`` iterable should be inserted at position
    ``index`` into the token stream given by the ``tokens``
    argument.
 
    The result is a combined token stream.
 
    TODO: clean up the code here.
    """
    insertions = iter(insertions)
    try:
        index, itokens = next(insertions)
    except StopIteration:
        # no insertions
        yield from tokens
        return
 
    realpos = None
    insleft = True
 
    # iterate over the token stream where we want to insert
    # the tokens from the insertion list.
    for i, t, v in tokens:
        # first iteration. store the position of first item
        if realpos is None:
            realpos = i
        oldi = 0
        while insleft and i + len(v) >= index:
            tmpval = v[oldi:index - i]
            if tmpval:
                yield realpos, t, tmpval
                realpos += len(tmpval)
            for it_index, it_token, it_value in itokens:
                yield realpos, it_token, it_value
                realpos += len(it_value)
            oldi = index - i
            try:
                index, itokens = next(insertions)
            except StopIteration:
                insleft = False
                break  # not strictly necessary
        if oldi < len(v):
            yield realpos, t, v[oldi:]
            realpos += len(v) - oldi
 
    # leftover tokens
    while insleft:
        # no normal tokens, set realpos to zero
        realpos = realpos or 0
        for p, t, v in itokens:
            yield realpos, t, v
            realpos += len(v)
        try:
            index, itokens = next(insertions)
        except StopIteration:
            insleft = False
            break  # not strictly necessary
 
 

References i.

using()

using	(		_other,
		**	kwargs
	)

Callback that processes the match with a different lexer.

The keyword arguments are forwarded to the lexer, except `state` which
is handled separately.

`state` specifies the state that the new lexer will start in, and can
be an enumerable such as ('root', 'inline', 'string') or a simple
string which is assumed to be on top of the root state.

Note: For that to work, `_other` must not be an `ExtendedRegexLexer`.

Definition at line 400 of file lexer.py.

def using(_other, **kwargs):
    """
    Callback that processes the match with a different lexer.
 
    The keyword arguments are forwarded to the lexer, except `state` which
    is handled separately.
 
    `state` specifies the state that the new lexer will start in, and can
    be an enumerable such as ('root', 'inline', 'string') or a simple
    string which is assumed to be on top of the root state.
 
    Note: For that to work, `_other` must not be an `ExtendedRegexLexer`.
    """
    gt_kwargs = {}
    if 'state' in kwargs:
        s = kwargs.pop('state')
        if isinstance(s, (list, tuple)):
            gt_kwargs['stack'] = s
        else:
            gt_kwargs['stack'] = ('root', s)
 
    if _other is this:
        def callback(lexer, match, ctx=None):
            # if keyword arguments are given the callback
            # function has to create a new lexer instance
            if kwargs:
                # XXX: cache that somehow
                kwargs.update(lexer.options)
                lx = lexer.__class__(**kwargs)
            else:
                lx = lexer
            s = match.start()
            for i, t, v in lx.get_tokens_unprocessed(match.group(), **gt_kwargs):
                yield i + s, t, v
            if ctx:
                ctx.pos = match.end()
    else:
        def callback(lexer, match, ctx=None):
            # XXX: cache that somehow
            kwargs.update(lexer.options)
            lx = _other(**kwargs)
 
            s = match.start()
            for i, t, v in lx.get_tokens_unprocessed(match.group(), **gt_kwargs):
                yield i + s, t, v
            if ctx:
                ctx.pos = match.end()
    return callback
 
 

References i.

Variable Documentation

_default_analyse

_default_analyse = staticmethod(lambda x: 0.0)

protected

Definition at line 34 of file lexer.py.

_encoding_map

list _encoding_map

protected

Initial value:

=  [(b'\xef\xbb\xbf', 'utf-8'),
                 (b'\xff\xfe\0\0', 'utf-32'),
                 (b'\0\0\xfe\xff', 'utf-32be'),
                 (b'\xff\xfe', 'utf-16'),
                 (b'\xfe\xff', 'utf-16be')]

Definition at line 28 of file lexer.py.

inherit

inherit = _inherit()

Definition at line 322 of file lexer.py.

line_re

line_re = re.compile('.*?\n')

Definition at line 26 of file lexer.py.

this

this = _This()

Definition at line 397 of file lexer.py.