root/lib/ubasic/tokenizer.c

/* [<][>][^][v][top][bottom][index][help] */

DEFINITIONS

This source file includes following definitions.
  1. singlechar
  2. get_next_token
  3. tokenizer_init
  4. tokenizer_token
  5. tokenizer_next
  6. tokenizer_num
  7. tokenizer_string
  8. tokenizer_label
  9. tokenizer_error_print
  10. tokenizer_finished
  11. tokenizer_variable_num
  12. tokenizer_line_number

   1 /*
   2  * Copyright (c) 2006, Adam Dunkels
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  * 3. Neither the name of the author nor the names of its contributors
  14  *    may be used to endorse or promote products derived from this software
  15  *    without specific prior written permission.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  *
  29  */
  30 
  31 #if DEBUG
  32 #define DEBUG_PRINTF(...)  printf(__VA_ARGS__)
  33 #else
  34 #define DEBUG_PRINTF(...)
  35 #endif
  36 
  37 #ifdef UBASIC_TEST
  38 #include <string.h>
  39 #include <ctype.h>
  40 #include <stdio.h>
  41 #endif
  42 
  43 #include "tokenizer.h"
  44 #include "stdlib.h"
  45 
  46 static char const *ptr, *nextptr;
  47 
  48 #define MAX_NUMLEN 6
  49 
  50 struct keyword_token {
  51   const char *keyword;
  52   int token;
  53 };
  54 
  55 static ubasic_token current_token = TOKENIZER_ERROR;
  56 static int current_line = 0;
  57 
  58 static const struct keyword_token keywords[] = {
  59   {"<>",                                  TOKENIZER_NE},
  60   {"<=",                                  TOKENIZER_LE},
  61   {">=",                                  TOKENIZER_GE},
  62   {"<",                           TOKENIZER_LT},
  63   {">",                       TOKENIZER_GT},
  64   {"not",                     TOKENIZER_LNOT},
  65   {"or",                      TOKENIZER_LOR},
  66   {"and",                     TOKENIZER_LAND},
  67 
  68   {"let",                     TOKENIZER_LET},
  69   {"if",                      TOKENIZER_IF},
  70   {"then",                    TOKENIZER_THEN},
  71   {"else",                    TOKENIZER_ELSE},
  72   {"endif",                   TOKENIZER_ENDIF},
  73   {"select",                  TOKENIZER_SELECT},
  74   {"case_else",               TOKENIZER_CASE_ELSE},
  75   {"case",                    TOKENIZER_CASE},
  76   {"end_select",              TOKENIZER_END_SELECT},
  77   {"for",                     TOKENIZER_FOR},
  78   {"to",                      TOKENIZER_TO},
  79   {"next",                    TOKENIZER_NEXT},
  80   {"step",                    TOKENIZER_STEP},
  81   {"do",                      TOKENIZER_DO},
  82   {"until",                   TOKENIZER_UNTIL},
  83   {"while",                   TOKENIZER_WHILE},
  84   {"wend",                    TOKENIZER_WEND},
  85   {"goto",                    TOKENIZER_GOTO},
  86   {"gosub",                   TOKENIZER_GOSUB},
  87   {"return",                  TOKENIZER_RETURN},
  88   {"rem",                     TOKENIZER_REM},
  89   {"cls",                     TOKENIZER_CLS},
  90   {"print_screen",            TOKENIZER_PRINT_SCREEN},
  91   {"print",                   TOKENIZER_PRINT},
  92   {"random",                  TOKENIZER_RANDOM},
  93   {"click",                   TOKENIZER_CLICK},
  94   {"playsound",               TOKENIZER_PLAY_SOUND},
  95   {"press",                   TOKENIZER_PRESS},
  96   {"release",                 TOKENIZER_RELEASE},
  97   //{"shot",                    TOKENIZER_SHOOT}, // for compatibility
  98   {"shoot",                   TOKENIZER_SHOOT},
  99   {"set_console_layout",      TOKENIZER_SET_CONSOLE_LAYOUT},
 100   {"set_console_autoredraw",  TOKENIZER_SET_CONSOLE_AUTOREDRAW},
 101   {"console_redraw",          TOKENIZER_CONSOLE_REDRAW},
 102   {"sleep",                   TOKENIZER_SLEEP}, 
 103 
 104   /* WARNING due to tokenizer limitation longest match must be first */
 105 // GET  
 106   {"get_av96",                TOKENIZER_GET_AV96},
 107   {"get_av",                  TOKENIZER_GET_USER_AV_ID}, //FOR COMPATIBILITY
 108   {"get_bv96",                TOKENIZER_GET_BV96},
 109   {"get_capture_mode",        TOKENIZER_GET_CAPTURE_MODE},
 110   {"get_display_mode",        TOKENIZER_GET_DISPLAY_MODE},
 111   {"get_day_seconds",         TOKENIZER_GET_DAY_SECONDS},
 112   {"get_disk_size",           TOKENIZER_GET_DISK_SIZE},
 113   {"get_dof",                 TOKENIZER_GET_DOF},
 114   {"get_far_limit",           TOKENIZER_GET_FAR_LIMIT},
 115   {"get_free_disk_space",     TOKENIZER_GET_FREE_DISK_SPACE},
 116   {"get_focal_length",        TOKENIZER_GET_FOCAL_LENGTH},
 117   {"get_focus_mode",          TOKENIZER_GET_FOCUS_MODE},
 118   {"get_focus_ok",            TOKENIZER_GET_FOCUS_OK},
 119   {"get_focus_state",         TOKENIZER_GET_FOCUS_STATE},
 120   {"get_focus",               TOKENIZER_GET_FOCUS},
 121   {"get_sd_over_modes",       TOKENIZER_GET_SD_OVER_MODES},
 122   {"get_hyp_dist",            TOKENIZER_GET_HYPERFOCAL_DIST},
 123   {"get_iso_market",          TOKENIZER_GET_ISO_MARKET},
 124   {"get_iso_mode",            TOKENIZER_GET_ISO_MODE},
 125   {"get_iso_real",            TOKENIZER_GET_ISO_REAL},
 126   {"get_iso",                 TOKENIZER_GET_ISO_MODE}, //FOR COMPATIBILITY
 127   {"get_jpg_count",           TOKENIZER_GET_JPG_COUNT},
 128   {"get_min_stack_dist",      TOKENIZER_GET_MIN_STACK_DIST},
 129   {"get_movie_status",        TOKENIZER_GET_MOVIE_STATUS},
 130   {"get_mode",                            TOKENIZER_GET_MODE},  // Returns 0 in recordmode, 1 in playmode
 131   {"get_near_limit",          TOKENIZER_GET_NEAR_LIMIT},
 132   {"get_platform_id",         TOKENIZER_GET_PLATFORM_ID},
 133   {"get_propset",             TOKENIZER_GET_PROPSET},
 134   {"get_prop",                TOKENIZER_GET_PROP},
 135   {"get_quality",             TOKENIZER_GET_QUALITY},
 136   {"get_raw_support",         TOKENIZER_GET_RAW_SUPPORT},
 137   {"get_raw_count",           TOKENIZER_GET_RAW_COUNT},
 138   {"get_raw_nr",              TOKENIZER_GET_RAW_NR},
 139   {"get_raw",                 TOKENIZER_GET_RAW},
 140   {"get_resolution",          TOKENIZER_GET_RESOLUTION},
 141   {"get_sv96",                TOKENIZER_GET_SV96},
 142   {"get_temperature",         TOKENIZER_GET_TEMPERATURE},
 143   {"get_tick_count",          TOKENIZER_GET_TICK_COUNT},
 144   {"get_time",                    TOKENIZER_GET_TIME},
 145   {"get_tv96",                TOKENIZER_GET_TV96},
 146   {"get_user_av_id",          TOKENIZER_GET_USER_AV_ID},
 147   {"get_user_av96",           TOKENIZER_GET_USER_AV96},
 148   {"get_user_tv_id",          TOKENIZER_GET_USER_TV_ID},
 149   {"get_user_tv96",           TOKENIZER_GET_USER_TV96},
 150   {"get_video_recording",     TOKENIZER_GET_VIDEO_RECORDING},
 151   {"get_video_button",        TOKENIZER_GET_VIDEO_BUTTON},
 152   {"get_vbatt",               TOKENIZER_GET_VBATT},  
 153   {"get_zoom_steps",          TOKENIZER_GET_ZOOM_STEPS},
 154   {"get_zoom",                TOKENIZER_GET_ZOOM},
 155   {"get_exp_count",           TOKENIZER_GET_EXP_COUNT},
 156   {"get_config_value",        TOKENIZER_GET_CONFIG_VALUE},
 157   {"swap_partitions",         TOKENIZER_SWAP_PARTITIONS},
 158 //SET
 159   {"set_av96_direct",         TOKENIZER_SET_AV96_DIRECT},
 160   {"set_av_rel",              TOKENIZER_SET_USER_AV_BY_ID_REL}, //FOR COMPATIBILITY
 161   {"set_av96",                TOKENIZER_SET_AV96},
 162   {"set_av",                  TOKENIZER_SET_USER_AV_BY_ID}, //FOR COMPATIBILITY
 163   {"set_capture_mode_canon",  TOKENIZER_SET_CAPTURE_MODE_CANON},
 164   {"set_capture_mode",        TOKENIZER_SET_CAPTURE_MODE},
 165   {"set_backlight",           TOKENIZER_SET_BACKLIGHT},
 166   {"set_lcd_display",         TOKENIZER_SET_LCDDISPLAY},
 167   {"set_draw_title_line",     TOKENIZER_SET_DRAW_TITLE_LINE},
 168   {"get_draw_title_line",     TOKENIZER_GET_DRAW_TITLE_LINE},
 169   {"set_focus",               TOKENIZER_SET_FOCUS},
 170   {"set_iso_mode",            TOKENIZER_SET_ISO_MODE},
 171   {"set_iso_real",            TOKENIZER_SET_ISO_REAL},
 172   {"set_iso",                 TOKENIZER_SET_ISO_MODE}, //FOR COMPATIBILITY
 173   {"set_led",                 TOKENIZER_SET_LED},
 174   {"set_movie_status",        TOKENIZER_SET_MOVIE_STATUS},  
 175   {"set_nd_filter",           TOKENIZER_SET_ND_FILTER},
 176   {"set_prop",                TOKENIZER_SET_PROP},
 177   {"set_quality",             TOKENIZER_SET_QUALITY},
 178   {"set_raw_nr",              TOKENIZER_SET_RAW_NR},
 179   {"set_raw",                 TOKENIZER_SET_RAW},
 180   {"set_resolution",          TOKENIZER_SET_RESOLUTION},
 181   {"set_remote_timing",       TOKENIZER_SET_REMOTE_TIMING},
 182   {"set_sv96",                    TOKENIZER_SET_SV96},
 183   {"set_tv96_direct",         TOKENIZER_SET_TV96_DIRECT},
 184   {"set_tv_rel",              TOKENIZER_SET_USER_TV_BY_ID_REL}, //FOR COMPATIBILITY
 185   {"set_tv96",                TOKENIZER_SET_TV96},
 186   {"set_tv",                  TOKENIZER_SET_USER_TV_BY_ID}, //FOR COMPATIBILITY
 187   {"set_user_av_by_id_rel",   TOKENIZER_SET_USER_AV_BY_ID_REL},
 188   {"set_user_av_by_id",       TOKENIZER_SET_USER_AV_BY_ID},
 189   {"set_user_av96",           TOKENIZER_SET_USER_AV96},
 190   {"set_user_tv_by_id_rel",   TOKENIZER_SET_USER_TV_BY_ID_REL},
 191   {"set_user_tv_by_id",       TOKENIZER_SET_USER_TV_BY_ID},
 192   {"set_user_tv96",           TOKENIZER_SET_USER_TV96},
 193   {"set_zoom_speed",          TOKENIZER_SET_ZOOM_SPEED},
 194   {"set_zoom_rel",            TOKENIZER_SET_ZOOM_REL},
 195   {"set_zoom",                TOKENIZER_SET_ZOOM},
 196   {"set_record",              TOKENIZER_SET_RECORD},
 197   {"set_config_value",        TOKENIZER_SET_CONFIG_VALUE},
 198   {"set_yield",               TOKENIZER_SET_YIELD},
 199   
 200   {"wait_click",              TOKENIZER_WAIT_CLICK},
 201   {"is_pressed",              TOKENIZER_IS_PRESSED},
 202   {"is_key",                  TOKENIZER_IS_KEY},
 203   {"set_exit_key",            TOKENIZER_SET_EXIT_KEY},
 204   
 205   {"wheel_right",             TOKENIZER_WHEEL_RIGHT},
 206   {"wheel_left",              TOKENIZER_WHEEL_LEFT},
 207   
 208   {"@title",                  TOKENIZER_REM},
 209   {"@subtitle",               TOKENIZER_REM},
 210   {"@param",                  TOKENIZER_REM},
 211   {"@default",                TOKENIZER_REM},
 212   {"@range",                  TOKENIZER_REM},
 213   {"@values",                 TOKENIZER_REM},
 214   {"@chdk_version",           TOKENIZER_REM},
 215 
 216   {"md_detect_motion",        TOKENIZER_MD_DETECT_MOTION},
 217   {"md_get_cell_diff",        TOKENIZER_MD_GET_CELL_DIFF},
 218   {"md_get_cell_val",         TOKENIZER_MD_GET_CELL_VAL},
 219   {"md_af_led_control",       TOKENIZER_MD_AF_LED_CONTROL},
 220   {"autostarted",             TOKENIZER_SCRIPT_AUTOSTARTED},
 221   {"get_autostart",           TOKENIZER_GET_SCRIPT_AUTOSTART},
 222   {"set_autostart",           TOKENIZER_SET_SCRIPT_AUTOSTART},
 223   {"get_usb_power",           TOKENIZER_GET_USB_POWER},
 224   {"usb_force_active",        TOKENIZER_FORCE_USB_PRESENT},
 225   {"usb_sync_wait",           TOKENIZER_USB_SYNC_WAIT},
 226   {"exit_alt",                TOKENIZER_EXIT_ALT},
 227   {"enter_alt",               TOKENIZER_ENTER_ALT}, 
 228   {"get_alt_mode",            TOKENIZER_GET_ALT_MODE}, 
 229   {"shut_down",               TOKENIZER_SHUT_DOWN},
 230 
 231   {"get_shooting",            TOKENIZER_GET_SHOOTING},
 232   {"get_drive_mode",          TOKENIZER_GET_DRIVE_MODE},
 233   {"get_flash_mode",          TOKENIZER_GET_FLASH_MODE},
 234   {"get_flash_ready",         TOKENIZER_GET_FLASH_READY},
 235   {"get_IS_mode",             TOKENIZER_GET_IS_MODE},
 236   {"set_ev",                  TOKENIZER_SET_EV},
 237   {"get_ev",                  TOKENIZER_GET_EV},
 238   {"get_orientation_sensor",  TOKENIZER_GET_ORIENTATION_SENSOR},
 239   {"get_nd_present",          TOKENIZER_GET_ND_PRESENT},
 240   {"get_histo_range",         TOKENIZER_GET_HISTO_RANGE},
 241   {"shot_histo_enable",       TOKENIZER_SHOT_HISTO_ENABLE},
 242   {"set_aelock",              TOKENIZER_SET_AELOCK},
 243   {"set_aflock",              TOKENIZER_SET_AFLOCK},
 244   {"set_mf",                  TOKENIZER_SET_MF},
 245   {"is_capture_mode_valid",   TOKENIZER_IS_CAPTURE_MODE_VALID}, 
 246   {"reboot",                  TOKENIZER_REBOOT},
 247   
 248   // APEX functions
 249   {"iso_to_sv96",             TOKENIZER_ISO_TO_SV96},
 250   {"sv96_to_iso",             TOKENIZER_SV96_TO_ISO},
 251   {"iso_real_to_market",      TOKENIZER_ISO_REAL_TO_MARKET},
 252   {"iso_market_to_real",      TOKENIZER_ISO_MARKET_TO_REAL},
 253   {"sv96_real_to_market",     TOKENIZER_SV96_REAL_TO_MARKET},
 254   {"sv96_market_to_real",     TOKENIZER_SV96_MARKET_TO_REAL},
 255   {"aperture_to_av96",        TOKENIZER_APERTURE_TO_AV96},
 256   {"av96_to_aperture",        TOKENIZER_AV96_TO_APERTURE},
 257   {"usec_to_tv96",            TOKENIZER_USEC_TO_TV96},
 258   {"tv96_to_usec",            TOKENIZER_TV96_TO_USEC},
 259   {"seconds_to_tv96",         TOKENIZER_SECONDS_TO_TV96},
 260 
 261   {"end",                     TOKENIZER_END},
 262 
 263   {NULL,                      TOKENIZER_ERROR}
 264 };
 265 
 266 /*---------------------------------------------------------------------------*/
 267 static int
 268 singlechar(void)
 269 {
 270   if(*ptr == '\n') {
 271     return TOKENIZER_CR;
 272   } else if(*ptr == ',') {
 273     return TOKENIZER_COMMA;
 274   } else if(*ptr == ';') {
 275     return TOKENIZER_SEMICOLON;
 276   } else if(*ptr == '+') {
 277     return TOKENIZER_PLUS;
 278   } else if(*ptr == '-') {
 279     return TOKENIZER_MINUS;
 280   } else if(*ptr == '&') {
 281     return TOKENIZER_AND;
 282   } else if(*ptr == '|') {
 283     return TOKENIZER_OR;
 284   } else if(*ptr == '^') {
 285     return TOKENIZER_XOR;
 286   } else if(*ptr == '*') {
 287     return TOKENIZER_ASTR;
 288   } else if(*ptr == '/') {
 289     return TOKENIZER_SLASH;
 290   } else if(*ptr == '%') {
 291     return TOKENIZER_MOD;
 292   } else if(*ptr == '(') {
 293     return TOKENIZER_LEFTPAREN;
 294   } else if(*ptr == ')') {
 295     return TOKENIZER_RIGHTPAREN;
 296   } else if(*ptr == '=') {
 297     return TOKENIZER_EQ;
 298   }
 299   return 0;
 300 }
 301 /*---------------------------------------------------------------------------*/
 302 static int
 303 get_next_token(void)
 304 {
 305   struct keyword_token const *kt;
 306   int i;
 307 
 308   DEBUG_PRINTF("get_next_token(): '%s'\n", ptr);
 309 
 310   // eat all whitespace
 311   while(*ptr == ' ' || *ptr == '\t' || *ptr == '\r') ptr++;
 312 
 313   if(*ptr == 0) {
 314     return TOKENIZER_ENDOFINPUT;
 315   }
 316   
 317     // UnknStatement should have size, otherwise hanging-up in ubasic.c possible for some cases 
 318     nextptr = ptr + 1; 
 319   
 320   if(isdigit(*ptr)) {
 321     for(i = 0; i < (MAX_NUMLEN+1); ++i) {
 322       if(!isdigit(ptr[i])) {
 323         if(i > 0) {
 324           nextptr = ptr + i;
 325           return TOKENIZER_NUMBER;
 326         } else {
 327           DEBUG_PRINTF("get_next_token: error due to too short number\n");
 328           return TOKENIZER_ERROR;
 329         }
 330       }
 331       if(!isdigit(ptr[i])) {
 332         DEBUG_PRINTF("get_next_token: error due to malformed number\n");
 333         return TOKENIZER_ERROR;
 334       }
 335     }
 336     DEBUG_PRINTF("get_next_token: error due to too long number\n");
 337     return TOKENIZER_ERROR;
 338   } else if(*ptr == ':') {
 339     // label
 340     nextptr = ptr;
 341     do {
 342       ++nextptr;
 343     } while(*nextptr != ' ' && *nextptr != '\r' && *nextptr != '\n' && *nextptr != '\t');
 344     return TOKENIZER_LABEL;
 345   } else if((i=singlechar()) != 0) {
 346     if (i == TOKENIZER_CR){
 347       // move to next line, and skip all following empty lines as well
 348       while (singlechar() == TOKENIZER_CR) 
 349       {
 350         current_line++;
 351         ptr++;
 352         // eat all whitespace
 353         while(*ptr == ' ' || *ptr == '\t' || *ptr == '\r') ptr++;
 354       };
 355       ptr--;
 356       // dangelo: now the last char might point to a whitespace instead of
 357       // a CR. I hope that doesn't break anything.
 358     }
 359     nextptr = ptr + 1;
 360     return i;
 361   } else if(*ptr == '"') {
 362     nextptr = ptr;
 363     do {
 364       ++nextptr;
 365     } while(*nextptr != '"');
 366     ++nextptr;
 367     return TOKENIZER_STRING;
 368   } else {
 369     for(kt = keywords; kt->keyword != NULL; ++kt) {
 370       if(strncmp(ptr, kt->keyword, strlen(kt->keyword)) == 0) {
 371         nextptr = ptr + strlen(kt->keyword);
 372         if (kt->token == TOKENIZER_REM) {
 373            while(*nextptr != 0 && *nextptr != '\r' && *nextptr != '\n') ++nextptr;
 374         }
 375         return kt->token;
 376       }
 377     }
 378   }
 379 
 380   if((*ptr >= 'a' && *ptr <= 'z') || (*ptr >= 'A' && *ptr <= 'Z')) {
 381     nextptr = ptr + 1;
 382     return TOKENIZER_VARIABLE;
 383   }
 384 
 385   
 386   return TOKENIZER_ERROR;
 387 }
 388 /*---------------------------------------------------------------------------*/
 389 void
 390 tokenizer_init(const char *program)
 391 {
 392   ptr = program;
 393   current_line = 1;
 394   current_token = get_next_token();
 395   while (current_token==TOKENIZER_CR && !tokenizer_finished()) tokenizer_next();
 396 }
 397 /*---------------------------------------------------------------------------*/
 398 int
 399 tokenizer_token(void)
 400 {
 401   return current_token;
 402 }
 403 /*---------------------------------------------------------------------------*/
 404 void
 405 tokenizer_next(void)
 406 {
 407 
 408   if(tokenizer_finished()) {
 409     return;
 410   }
 411 
 412   DEBUG_PRINTF("tokenizer_next: %p\n", nextptr);
 413   ptr = nextptr;
 414   while((*ptr == ' ') || (*ptr == '\t')) {
 415     ++ptr;
 416   }
 417   current_token = get_next_token();
 418   DEBUG_PRINTF("tokenizer_next: '%s' %d\n", ptr, current_token);
 419   return;
 420 }
 421 /*---------------------------------------------------------------------------*/
 422 int
 423 tokenizer_num(void)
 424 {
 425   return atoi(ptr);
 426 }
 427 /*---------------------------------------------------------------------------*/
 428 void
 429 tokenizer_string(char *dest, int len)
 430 {
 431   char *string_end;
 432   int string_len;
 433   
 434   if(tokenizer_token() != TOKENIZER_STRING) {
 435     return;
 436   }
 437   string_end = strchr(ptr + 1, '"');
 438   if(string_end == NULL) {
 439     return;
 440   }
 441   string_len = string_end - ptr - 1;
 442   if(len < string_len) {
 443     string_len = len;
 444   }
 445   memcpy(dest, ptr + 1, string_len);
 446   dest[string_len] = 0;
 447 }
 448 /*---------------------------------------------------------------------------*/
 449 void
 450 tokenizer_label(char *dest, int len)
 451 {
 452   char *string_end;
 453   int string_len;
 454   
 455   if(tokenizer_token() != TOKENIZER_LABEL) {
 456     return;
 457   }
 458   // allow string \r, \n, tabulation and space to end labels
 459   string_end = strpbrk(ptr + 1, " \t\r\n");
 460 
 461   if(string_end == NULL) {
 462     return;
 463   }
 464   string_len = string_end - ptr - 1;
 465   if(len < string_len) {
 466     string_len = len;
 467   }
 468   memcpy(dest, ptr + 1, string_len);
 469   dest[string_len] = 0;
 470 }
 471 /*---------------------------------------------------------------------------*/
 472 void
 473 tokenizer_error_print(void)
 474 {
 475   DEBUG_PRINTF("tokenizer_error_print: '%s'\n", ptr);
 476 }
 477 /*---------------------------------------------------------------------------*/
 478 int
 479 tokenizer_finished(void)
 480 {
 481   return *ptr == 0 || current_token == TOKENIZER_ENDOFINPUT;
 482 }
 483 /*---------------------------------------------------------------------------*/
 484 int
 485 tokenizer_variable_num(void)
 486 {
 487   return *ptr - (*ptr>='a'?'a':('A'-26));
 488 }
 489 /*---------------------------------------------------------------------------*/
 490 int tokenizer_line_number(void)
 491 {
 492   return current_line;
 493 }

/* [<][>][^][v][top][bottom][index][help] */