lexer.c File Reference

#include <stdio.h>
#include "./lexer.h"
#include "./subtree.h"
#include "./length.h"
#include "./unicode.h"

Go to the source code of this file.

Macros
#define	LOG(message, character)

Functions
static bool	ts_lexer__eof (const TSLexer *_self)
static void	ts_lexer__clear_chunk (Lexer *self)
static void	ts_lexer__get_chunk (Lexer *self)
static void	ts_lexer__get_lookahead (Lexer *self)
static void	ts_lexer_goto (Lexer *self, Length position)
static void	ts_lexer__do_advance (Lexer *self, bool skip)
static void	ts_lexer__advance (TSLexer *_self, bool skip)
static void	ts_lexer__mark_end (TSLexer *_self)
static uint32_t	ts_lexer__get_column (TSLexer *_self)
static bool	ts_lexer__is_at_included_range_start (const TSLexer *_self)
void	ts_lexer_init (Lexer *self)
void	ts_lexer_delete (Lexer *self)
void	ts_lexer_set_input (Lexer *self, TSInput input)
void	ts_lexer_reset (Lexer *self, Length position)
void	ts_lexer_start (Lexer *self)
void	ts_lexer_finish (Lexer *self, uint32_t *lookahead_end_byte)
void	ts_lexer_advance_to_end (Lexer *self)
void	ts_lexer_mark_end (Lexer *self)
bool	ts_lexer_set_included_ranges (Lexer *self, const TSRange *ranges, uint32_t count)
TSRange *	ts_lexer_included_ranges (const Lexer *self, uint32_t *count)

Variables
static const int32_t	BYTE_ORDER_MARK = 0xFEFF
static const TSRange	DEFAULT_RANGE

Macro Definition Documentation

LOG

#define LOG	(		message,
			character
	)

Value:

  if (self->logger.log) {                    \
    snprintf(                                \
      self->debug_buffer,                    \
      TREE_SITTER_SERIALIZATION_BUFFER_SIZE, \
      32 <= character && character < 127 ?   \
        message " character:'%c'" :          \
        message " character:%d",             \
      character                              \
    );                                       \
    self->logger.log(                        \
      self->logger.payload,                  \
      TSLogTypeLex,                          \
      self->debug_buffer                     \
    );                                       \
  }

Definition at line 7 of file lexer.c.

Function Documentation

ts_lexer__advance()

static void ts_lexer__advance ( TSLexer *  _self, bool  skip  )

static

Definition at line 200 of file lexer.c.

                                                         {
  Lexer *self = (Lexer *)_self;
  if (!self->chunk) return;
 
  if (skip) {
    LOG("skip", self->data.lookahead);
  } else {
    LOG("consume", self->data.lookahead);
  }
  
  ts_lexer__do_advance(self, skip);
}

References LOG, skip(), and ts_lexer__do_advance().

Referenced by ts_lexer_advance_to_end(), ts_lexer_init(), and ts_lexer_start().

ts_lexer__clear_chunk()

static void ts_lexer__clear_chunk ( Lexer *  self )

static

Definition at line 49 of file lexer.c.

                                               {
  self->chunk = NULL;
  self->chunk_size = 0;
  self->chunk_start = 0;
}

References NULL.

Referenced by ts_lexer__do_advance(), ts_lexer_goto(), and ts_lexer_set_input().

ts_lexer__do_advance()

static void ts_lexer__do_advance ( Lexer *  self, bool  skip  )

static

Definition at line 156 of file lexer.c.

                                                         {
  if (self->lookahead_size) {
    self->current_position.bytes += self->lookahead_size;
    if (self->data.lookahead == '\n') {
      self->current_position.extent.row++;
      self->current_position.extent.column = 0;
    } else {
      self->current_position.extent.column += self->lookahead_size;
    }
  }
 
  const TSRange *current_range = NULL;
  if (self->current_included_range_index < self->included_range_count) {
    current_range = &self->included_ranges[self->current_included_range_index];
    if (self->current_position.bytes == current_range->end_byte) {
      self->current_included_range_index++;
      if (self->current_included_range_index < self->included_range_count) {
        current_range++;
        self->current_position = (Length) {
          current_range->start_byte,
          current_range->start_point,
        };
      } else {
        current_range = NULL;
      }
    }
  }
 
  if (skip) self->token_start_position = self->current_position;
 
  if (current_range) {
    if (self->current_position.bytes >= self->chunk_start + self->chunk_size) {
      ts_lexer__get_chunk(self);
    }
    ts_lexer__get_lookahead(self);
  } else {
    ts_lexer__clear_chunk(self);
    self->data.lookahead = '\0';
    self->lookahead_size = 1;
  }
}

References TSRange::end_byte, NULL, skip(), TSRange::start_byte, TSRange::start_point, ts_lexer__clear_chunk(), ts_lexer__get_chunk(), and ts_lexer__get_lookahead().

Referenced by ts_lexer__advance(), and ts_lexer__get_column().

ts_lexer__eof()

static bool ts_lexer__eof ( const TSLexer *  _self )

static

Definition at line 42 of file lexer.c.

                                                {
  Lexer *self = (Lexer *)_self;
  return self->current_included_range_index == self->included_range_count;
}

Referenced by ts_lexer__get_column(), ts_lexer__mark_end(), ts_lexer_init(), and ts_lexer_start().

ts_lexer__get_chunk()

static void ts_lexer__get_chunk ( Lexer *  self )

static

Definition at line 57 of file lexer.c.

                                             {
  self->chunk_start = self->current_position.bytes;
  self->chunk = self->input.read(
    self->input.payload,
    self->current_position.bytes,
    self->current_position.extent,
    &self->chunk_size
  );
  if (!self->chunk_size) {
    self->current_included_range_index = self->included_range_count;
    self->chunk = NULL;
  }
}

References NULL.

Referenced by ts_lexer__do_advance(), ts_lexer__get_column(), ts_lexer__get_lookahead(), and ts_lexer_start().

ts_lexer__get_column()

static uint32_t ts_lexer__get_column ( TSLexer *  _self )

static

Definition at line 239 of file lexer.c.

                                                     {
  Lexer *self = (Lexer *)_self;
  
  uint32_t goal_byte = self->current_position.bytes;
  
  self->did_get_column = true;
  self->current_position.bytes -= self->current_position.extent.column;
  self->current_position.extent.column = 0;
 
  if (self->current_position.bytes < self->chunk_start) {
    ts_lexer__get_chunk(self);
  }
 
  uint32_t result = 0;
  ts_lexer__get_lookahead(self);
  while (self->current_position.bytes < goal_byte && !ts_lexer__eof(_self) && self->chunk) {
    ts_lexer__do_advance(self, false);
    result++;
  }
 
  return result;
}

References ts_lexer__do_advance(), ts_lexer__eof(), ts_lexer__get_chunk(), and ts_lexer__get_lookahead().

Referenced by ts_lexer_init().

ts_lexer__get_lookahead()

static void ts_lexer__get_lookahead ( Lexer *  self )

static

Definition at line 74 of file lexer.c.

                                                 {
  uint32_t position_in_chunk = self->current_position.bytes - self->chunk_start;
  uint32_t size = self->chunk_size - position_in_chunk;
 
  if (size == 0) {
    self->lookahead_size = 1;
    self->data.lookahead = '\0';
    return;
  }
 
  const uint8_t *chunk = (const uint8_t *)self->chunk + position_in_chunk;
  UnicodeDecodeFunction decode = self->input.encoding == TSInputEncodingUTF8
    ? ts_decode_utf8
    : ts_decode_utf16;
 
  self->lookahead_size = decode(chunk, size, &self->data.lookahead);
 
  // If this chunk ended in the middle of a multi-byte character,
  // try again with a fresh chunk.
  if (self->data.lookahead == TS_DECODE_ERROR && size < 4) {
    ts_lexer__get_chunk(self);
    chunk = (const uint8_t *)self->chunk;
    size = self->chunk_size;
    self->lookahead_size = decode(chunk, size, &self->data.lookahead);
  }
 
  if (self->data.lookahead == TS_DECODE_ERROR) {
    self->lookahead_size = 1;
  }
}

References TS_DECODE_ERROR, ts_decode_utf16(), ts_decode_utf8(), ts_lexer__get_chunk(), and TSInputEncodingUTF8.

Referenced by ts_lexer__do_advance(), ts_lexer__get_column(), and ts_lexer_start().

ts_lexer__is_at_included_range_start()

static bool ts_lexer__is_at_included_range_start ( const TSLexer *  _self )

static

Definition at line 265 of file lexer.c.

                                                                       {
  const Lexer *self = (const Lexer *)_self;
  if (self->current_included_range_index < self->included_range_count) {
    TSRange *current_range = &self->included_ranges[self->current_included_range_index];
    return self->current_position.bytes == current_range->start_byte;
  } else {
    return false;
  }
}

References TSRange::start_byte.

Referenced by ts_lexer_init().

ts_lexer__mark_end()

static void ts_lexer__mark_end ( TSLexer *  _self )

static

Definition at line 215 of file lexer.c.

                                               {
  Lexer *self = (Lexer *)_self;
  if (!ts_lexer__eof(&self->data)) {
    // If the lexer is right at the beginning of included range,
    // then the token should be considered to end at the *end* of the
    // previous included range, rather than here.
    TSRange *current_included_range = &self->included_ranges[
      self->current_included_range_index
    ];
    if (
      self->current_included_range_index > 0 &&
      self->current_position.bytes == current_included_range->start_byte
    ) {
      TSRange *previous_included_range = current_included_range - 1;
      self->token_end_position = (Length) {
        previous_included_range->end_byte,
        previous_included_range->end_point,
      };
      return;
    }
  }
  self->token_end_position = self->current_position;
}

References TSRange::end_byte, TSRange::end_point, TSRange::start_byte, and ts_lexer__eof().

Referenced by ts_lexer_finish(), ts_lexer_init(), and ts_lexer_mark_end().

ts_lexer_advance_to_end()

void ts_lexer_advance_to_end

(

Lexer *

self

)

Definition at line 357 of file lexer.c.

                                          {
  while (self->chunk) {
    ts_lexer__advance(&self->data, false);
  }
}

References ts_lexer__advance().

Referenced by parser__halt_parse().

ts_lexer_delete()

void ts_lexer_delete

(

Lexer *

self

)

Definition at line 304 of file lexer.c.

                                  {
  ts_free(self->included_ranges);
}

References ts_free.

Referenced by ts_parser_delete().

ts_lexer_finish()

void ts_lexer_finish	(	Lexer *	self,
		uint32_t *	lookahead_end_byte
	)

Definition at line 337 of file lexer.c.

                                                                {
  if (length_is_undefined(self->token_end_position)) {
    ts_lexer__mark_end(&self->data);
  }
 
  uint32_t current_lookahead_end_byte = self->current_position.bytes + 1;
 
  // In order to determine that a byte sequence is invalid UTF8 or UTF16,
  // the character decoding algorithm may have looked at the following byte.
  // Therefore, the next byte *after* the current (invalid) character
  // affects the interpretation of the current character.
  if (self->data.lookahead == TS_DECODE_ERROR) {
    current_lookahead_end_byte++;
  }
 
  if (current_lookahead_end_byte > *lookahead_end_byte) {
    *lookahead_end_byte = current_lookahead_end_byte;
  }
}

References length_is_undefined(), TS_DECODE_ERROR, and ts_lexer__mark_end().

Referenced by ts_parser__lex().

ts_lexer_goto()

static void ts_lexer_goto ( Lexer *  self, Length  position  )

static

Definition at line 105 of file lexer.c.

                                                        {
  self->current_position = position;
  bool found_included_range = false;
 
  // Move to the first valid position at or after the given position.
  for (unsigned i = 0; i < self->included_range_count; i++) {
    TSRange *included_range = &self->included_ranges[i];
    if (included_range->end_byte > position.bytes) {
      if (included_range->start_byte >= position.bytes) {
        self->current_position = (Length) {
          .bytes = included_range->start_byte,
          .extent = included_range->start_point,
        };
      }
 
      self->current_included_range_index = i;
      found_included_range = true;
      break;
    }
  }
 
  if (found_included_range) {
    // If the current position is outside of the current chunk of text,
    // then clear out the current chunk of text.
    if (self->chunk && (
      position.bytes < self->chunk_start ||
      position.bytes >= self->chunk_start + self->chunk_size
    )) {
      ts_lexer__clear_chunk(self);
    }
 
    self->lookahead_size = 0;
    self->data.lookahead = '\0';
  }
 
  // If the given position is beyond any of included ranges, move to the EOF
  // state - past the end of the included ranges.
  else {
    self->current_included_range_index = self->included_range_count;
    TSRange *last_included_range = &self->included_ranges[self->included_range_count - 1];
    self->current_position = (Length) {
      .bytes = last_included_range->end_byte,
      .extent = last_included_range->end_point,
    };
    ts_lexer__clear_chunk(self);
    self->lookahead_size = 1;
    self->data.lookahead = '\0';
  }
}

References Length::bytes, TSRange::end_byte, TSRange::end_point, i, TSRange::start_byte, TSRange::start_point, and ts_lexer__clear_chunk().

Referenced by ts_lexer_reset(), ts_lexer_set_included_ranges(), and ts_lexer_set_input().

ts_lexer_included_ranges()

TSRange* ts_lexer_included_ranges	(	const Lexer *	self,
		uint32_t *	count
	)

Definition at line 395 of file lexer.c.

                                                                      {
  *count = self->included_range_count;
  return self->included_ranges;
}

References count.

Referenced by ts_parser_included_ranges().

ts_lexer_init()

void ts_lexer_init

(

Lexer *

self

)

Definition at line 275 of file lexer.c.

                                {
  *self = (Lexer) {
    .data = {
      // The lexer's methods are stored as struct fields so that generated
      // parsers can call them without needing to be linked against this
      // library.
      .advance = ts_lexer__advance,
      .mark_end = ts_lexer__mark_end,
      .get_column = ts_lexer__get_column,
      .is_at_included_range_start = ts_lexer__is_at_included_range_start,
      .eof = ts_lexer__eof,
      .lookahead = 0,
      .result_symbol = 0,
    },
    .chunk = NULL,
    .chunk_size = 0,
    .chunk_start = 0,
    .current_position = {0, {0, 0}},
    .logger = {
      .payload = NULL,
      .log = NULL
    },
    .included_ranges = NULL,
    .included_range_count = 0,
    .current_included_range_index = 0,
  };
  ts_lexer_set_included_ranges(self, NULL, 0);
}

References NULL, ts_lexer__advance(), ts_lexer__eof(), ts_lexer__get_column(), ts_lexer__is_at_included_range_start(), ts_lexer__mark_end(), and ts_lexer_set_included_ranges().

Referenced by parser_init(), and ts_parser_new().

ts_lexer_mark_end()

void ts_lexer_mark_end

(

Lexer *

self

)

Definition at line 363 of file lexer.c.

                                    {
  ts_lexer__mark_end(&self->data);
}

References ts_lexer__mark_end().

Referenced by ts_parser__handle_error().

ts_lexer_reset()

void ts_lexer_reset	(	Lexer *	self,
		Length	position
	)

Definition at line 316 of file lexer.c.

                                                  {
  if (position.bytes != self->current_position.bytes) {
    ts_lexer_goto(self, position);
  }
}

References Length::bytes, and ts_lexer_goto().

Referenced by parser__lex(), ts_parser__handle_error(), ts_parser__lex(), and ts_parser_reset().

ts_lexer_set_included_ranges()

bool ts_lexer_set_included_ranges	(	Lexer *	self,
		const TSRange *	ranges,
		uint32_t	count
	)

Definition at line 367 of file lexer.c.

  {
  if (count == 0 || !ranges) {
    ranges = &DEFAULT_RANGE;
    count = 1;
  } else {
    uint32_t previous_byte = 0;
    for (unsigned i = 0; i < count; i++) {
      const TSRange *range = &ranges[i];
      if (
        range->start_byte < previous_byte ||
        range->end_byte < range->start_byte
      ) return false;
      previous_byte = range->end_byte;
    }
  }
 
  size_t size = count * sizeof(TSRange);
  self->included_ranges = ts_realloc(self->included_ranges, size);
  memcpy(self->included_ranges, ranges, size);
  self->included_range_count = count;
  ts_lexer_goto(self, self->current_position);
  return true;
}

References count, DEFAULT_RANGE, i, memcpy(), capstone::range, ts_lexer_goto(), and ts_realloc.

Referenced by ts_lexer_init(), and ts_parser_set_included_ranges().

ts_lexer_set_input()

void ts_lexer_set_input	(	Lexer *	self,
		TSInput	input
	)

Definition at line 308 of file lexer.c.

                                                    {
  self->input = input;
  ts_lexer__clear_chunk(self);
  ts_lexer_goto(self, self->current_position);
}

References input(), ts_lexer__clear_chunk(), and ts_lexer_goto().

Referenced by parser__start(), and ts_parser_parse().

ts_lexer_start()

void ts_lexer_start

(

Lexer *

self

)

Definition at line 322 of file lexer.c.

                                 {
  self->token_start_position = self->current_position;
  self->token_end_position = LENGTH_UNDEFINED;
  self->data.result_symbol = 0;
  self->did_get_column = false;
  if (!ts_lexer__eof(&self->data)) {
    if (!self->chunk_size) ts_lexer__get_chunk(self);
    if (!self->lookahead_size) ts_lexer__get_lookahead(self);
    if (
      self->current_position.bytes == 0 &&
      self->data.lookahead == BYTE_ORDER_MARK
    ) ts_lexer__advance(&self->data, true);
  }
}

References BYTE_ORDER_MARK, LENGTH_UNDEFINED, ts_lexer__advance(), ts_lexer__eof(), ts_lexer__get_chunk(), and ts_lexer__get_lookahead().

Referenced by parser__lex(), and ts_parser__lex().

Variable Documentation

BYTE_ORDER_MARK

const int32_t BYTE_ORDER_MARK = 0xFEFF

static

Definition at line 24 of file lexer.c.

Referenced by ts_lexer_start().

DEFAULT_RANGE

const TSRange DEFAULT_RANGE

static

Initial value:

= {
  .start_point = {
    .row = 0,
    .column = 0,
  },
  .end_point = {
    .row = UINT32_MAX,
    .column = UINT32_MAX,
  },
  .start_byte = 0,
  .end_byte = UINT32_MAX
}

Definition at line 26 of file lexer.c.

Referenced by ts_lexer_set_included_ranges().