tokenizer.c-Dateireferenz

#include "stdlib.h"
#include "stddef.h"
#include "string.h"
#include "ctype.h"
#include "stdio.h"
#include "tokenizer.h"

Include-Abhängigkeitsdiagramm für tokenizer.c:

gehe zum Quellcode dieser Datei

Datenstrukturen
struct	keyword_token

Makrodefinitionen
#define	DEBUG_PRINTF(...)
#define	MAX_NUMLEN   6

Funktionen
static int	singlechar (void)
static int	get_next_token (void)
void	tokenizer_init (const char *program)
int	tokenizer_token (void)
void	tokenizer_next (void)
int	tokenizer_num (void)
void	tokenizer_string (char *dest, int len)
void	tokenizer_label (char *dest, int len)
void	tokenizer_error_print (void)
int	tokenizer_finished (void)
int	tokenizer_variable_num (void)
int	tokenizer_line_number (void)

Variablen
static char const *	ptr
static char const *	nextptr
static ubasic_token	current_token = TOKENIZER_ERROR
static int	current_line = 0
static struct keyword_token	keywords []

Makro-Dokumentation

#define DEBUG_PRINTF

(

...)

Definiert in Zeile 34 der Datei tokenizer.c.

#define MAX_NUMLEN   6

Definiert in Zeile 47 der Datei tokenizer.c.

Dokumentation der Funktionen

static int get_next_token ( void  )

static

Definiert in Zeile 316 der Datei tokenizer.c.

{
  struct keyword_token const *kt;
  int i;

  DEBUG_PRINTF("get_next_token(): '%s'\n", ptr);

  // eat all whitespace
  while(*ptr == ' ' || *ptr == '\t' || *ptr == '\r') ptr++;

  if(*ptr == 0) {
    return TOKENIZER_ENDOFINPUT;
  }

    // UnknStatement should have size, otherwise hanging-up in ubasic.c possible for some cases
    nextptr = ptr + 1;

  if(isdigit(*ptr)) {
    for(i = 0; i < (MAX_NUMLEN+1); ++i) {
      if(!isdigit(ptr[i])) {
    if(i > 0) {
      nextptr = ptr + i;
      return TOKENIZER_NUMBER;
    } else {
      DEBUG_PRINTF("get_next_token: error due to too short number\n");
      return TOKENIZER_ERROR;
    }
      }
      if(!isdigit(ptr[i])) {
    DEBUG_PRINTF("get_next_token: error due to malformed number\n");
    return TOKENIZER_ERROR;
      }
    }
    DEBUG_PRINTF("get_next_token: error due to too long number\n");
    return TOKENIZER_ERROR;
  } else if(*ptr == ':') {
    // label
    nextptr = ptr;
    do {
      ++nextptr;
    } while(*nextptr != ' ' && *nextptr != '\r' && *nextptr != '\n' && *nextptr != '\t');
    return TOKENIZER_LABEL;
  } else if((i=singlechar()) != 0) {
    if (i == TOKENIZER_CR){
      // move to next line, and skip all following empty lines as well
      while (singlechar() == TOKENIZER_CR)
      {
        current_line++;
        ptr++;
        // eat all whitespace
        while(*ptr == ' ' || *ptr == '\t' || *ptr == '\r') ptr++;
      };
      ptr--;
      // dangelo: now the last char might point to a whitespace instead of
      // a CR. I hope that doesn't break anything.
    }
    nextptr = ptr + 1;
    return i;
  } else if(*ptr == '"') {
    nextptr = ptr;
    do {
      ++nextptr;
    } while(*nextptr != '"');
    ++nextptr;
    return TOKENIZER_STRING;
  } else {
    for(kt = keywords; kt->keyword != NULL; ++kt) {
      if(strncmp(ptr, kt->keyword, strlen(kt->keyword)) == 0) {
    nextptr = ptr + strlen(kt->keyword);
        if (kt->token == TOKENIZER_REM) {
           while(*nextptr != 0 && *nextptr != '\r' && *nextptr != '\n') ++nextptr;
        }
    return kt->token;
      }
    }
  }

  if((*ptr >= 'a' && *ptr <= 'z') || (*ptr >= 'A' && *ptr <= 'Z')) {
    nextptr = ptr + 1;
    return TOKENIZER_VARIABLE;
  }


  return TOKENIZER_ERROR;
}

static int singlechar ( void  )

static

Definiert in Zeile 281 der Datei tokenizer.c.

{
  if(*ptr == '\n') {
    return TOKENIZER_CR;
  } else if(*ptr == ',') {
    return TOKENIZER_COMMA;
  } else if(*ptr == ';') {
    return TOKENIZER_SEMICOLON;
  } else if(*ptr == '+') {
    return TOKENIZER_PLUS;
  } else if(*ptr == '-') {
    return TOKENIZER_MINUS;
  } else if(*ptr == '&') {
    return TOKENIZER_AND;
  } else if(*ptr == '|') {
    return TOKENIZER_OR;
  } else if(*ptr == '^') {
    return TOKENIZER_XOR;
  } else if(*ptr == '*') {
    return TOKENIZER_ASTR;
  } else if(*ptr == '/') {
    return TOKENIZER_SLASH;
  } else if(*ptr == '%') {
    return TOKENIZER_MOD;
  } else if(*ptr == '(') {
    return TOKENIZER_LEFTPAREN;
  } else if(*ptr == ')') {
    return TOKENIZER_RIGHTPAREN;
  } else if(*ptr == '=') {
    return TOKENIZER_EQ;
  }
  return 0;
}

void tokenizer_error_print

(

void

)

Definiert in Zeile 486 der Datei tokenizer.c.

{
  DEBUG_PRINTF("tokenizer_error_print: '%s'\n", ptr);
}

int tokenizer_finished

(

void

)

Definiert in Zeile 492 der Datei tokenizer.c.

{
  return *ptr == 0 || current_token == TOKENIZER_ENDOFINPUT;
}

void tokenizer_init

(

const char *

program)

Definiert in Zeile 403 der Datei tokenizer.c.

{
  ptr = program;
  current_line = 1;
  current_token = get_next_token();
  while (current_token==TOKENIZER_CR && !tokenizer_finished()) tokenizer_next();
}

void tokenizer_label	(	char *	dest,
		int	len
	)

Definiert in Zeile 463 der Datei tokenizer.c.

{
  char *string_end;
  int string_len;

  if(tokenizer_token() != TOKENIZER_LABEL) {
    return;
  }
  // allow string \r, \n, tabulation and space to end labels
  string_end = strpbrk(ptr + 1, " \t\r\n");

  if(string_end == NULL) {
    return;
  }
  string_len = string_end - ptr - 1;
  if(len < string_len) {
    string_len = len;
  }
  memcpy(dest, ptr + 1, string_len);
  dest[string_len] = 0;
}

int tokenizer_line_number

(

void

)

Definiert in Zeile 503 der Datei tokenizer.c.

{
  return current_line;
}

void tokenizer_next

(

void

)

Definiert in Zeile 418 der Datei tokenizer.c.

{

  if(tokenizer_finished()) {
    return;
  }

  DEBUG_PRINTF("tokenizer_next: %p\n", nextptr);
  ptr = nextptr;
  while((*ptr == ' ') || (*ptr == '\t')) {
    ++ptr;
  }
  current_token = get_next_token();
  DEBUG_PRINTF("tokenizer_next: '%s' %d\n", ptr, current_token);
  return;
}

int tokenizer_num

(

void

)

Definiert in Zeile 436 der Datei tokenizer.c.

{
  return atoi(ptr);
}

void tokenizer_string	(	char *	dest,
		int	len
	)

Definiert in Zeile 442 der Datei tokenizer.c.

{
  char *string_end;
  int string_len;

  if(tokenizer_token() != TOKENIZER_STRING) {
    return;
  }
  string_end = strchr(ptr + 1, '"');
  if(string_end == NULL) {
    return;
  }
  string_len = string_end - ptr - 1;
  if(len < string_len) {
    string_len = len;
  }
  memcpy(dest, ptr + 1, string_len);
  dest[string_len] = 0;
}

int tokenizer_token

(

void

)

Definiert in Zeile 412 der Datei tokenizer.c.

{
  return current_token;
}

int tokenizer_variable_num

(

void

)

Definiert in Zeile 498 der Datei tokenizer.c.

{
  return *ptr - (*ptr>='a'?'a':('A'-26));
}

Variablen-Dokumentation

int current_line = 0

static

Definiert in Zeile 55 der Datei tokenizer.c.

ubasic_token current_token = TOKENIZER_ERROR

static

Definiert in Zeile 54 der Datei tokenizer.c.

struct keyword_token keywords[]

static

Definiert in Zeile 57 der Datei tokenizer.c.

char const * nextptr

static

Definiert in Zeile 45 der Datei tokenizer.c.

char const* ptr

static

Definiert in Zeile 45 der Datei tokenizer.c.