vpr/SRC/util/token.c

Go to the documentation of this file.

/**
 * @file
 *
 * Jason Luu
 * July 22, 2009
 * Tokenizer
 */

#include <string.h>
#include <assert.h>
#include "util.h"
#include "token.h"
#include "ezxml.h"
#include "read_xml_util.h"

enum e_token_type GetTokenTypeFromChar(INP enum e_token_type cur_token_type, INP char cur);

/** Returns a token list of the text for a given string. */ 
t_token *GetTokensFromString(INP const char* inString, OUTP int * num_tokens)
{
        const char *cur;
        t_token * tokens;
        int i, in_string_index, prev_in_string_index;
        boolean has_null;
        enum e_token_type cur_token_type, new_token_type;

        *num_tokens = i = 0;
        cur_token_type = TOKEN_NULL;

        if(inString == NULL) { return NULL; };

        cur = inString; 

        /* Count number of tokens */
        while(*cur)
        {
                new_token_type = GetTokenTypeFromChar(cur_token_type, *cur);
                if(new_token_type != cur_token_type) {
                        cur_token_type = new_token_type;
                        if(new_token_type != TOKEN_NULL) {
                                i++;
                        }
                }
            ++cur;
        }
        *num_tokens = i;

        if(*num_tokens > 0) {
                tokens = my_calloc(*num_tokens + 1, sizeof(t_token));
        } else {
                return NULL;
        }

        /* populate tokens */
        i = 0;
        in_string_index = 0;
        has_null = TRUE;
        prev_in_string_index = 0;
        cur_token_type = TOKEN_NULL;

        cur = inString; 

        while(*cur)
        {
            
                new_token_type = GetTokenTypeFromChar(cur_token_type, *cur);
                if(new_token_type != cur_token_type) {
                        if(!has_null) {
                                tokens[i-1].data[in_string_index - prev_in_string_index] = '\0'; /* NULL the end of the data string */
                                has_null = TRUE;
                        }
                        if(new_token_type != TOKEN_NULL) {
                                tokens[i].type = new_token_type;
                                tokens[i].data = my_strdup(inString + in_string_index);
                                prev_in_string_index = in_string_index;
                                has_null = FALSE;                               
                                i++;
                        }
                        cur_token_type = new_token_type;
                }
                ++cur;
                in_string_index++;
        }

        assert(i == *num_tokens);

        tokens[*num_tokens].type = TOKEN_NULL;
        tokens[*num_tokens].data = NULL;

    
        /* Return the list */ 
        return tokens;
}

void freeTokens(INP t_token *tokens, INP int num_tokens) {
        int i;
        for(i = 0; i < num_tokens; i++) {
                free(tokens[i].data);
        }
        free(tokens);
}


enum e_token_type GetTokenTypeFromChar(INP enum e_token_type cur_token_type, INP char cur) {
        if(IsWhitespace(cur)) {
                return TOKEN_NULL;
        } else {
                if(cur == '[') {
                        return TOKEN_OPEN_SQUARE_BRACKET;
                } else if(cur == ']') {
                        return TOKEN_CLOSE_SQUARE_BRACKET;
                } else if(cur == '{') {
                        return TOKEN_OPEN_SQUIG_BRACKET;
                } else if(cur == '}') {
                        return TOKEN_CLOSE_SQUIG_BRACKET;
                } else if(cur == ':') {
                        return TOKEN_COLON;
                } else if(cur == '.') {
                        return TOKEN_DOT;
                } else if(cur >= '0' && cur <= '9' && cur_token_type != TOKEN_STRING) {
                        return TOKEN_INT;
                } else {
                        return TOKEN_STRING;
                }
        }
}

boolean checkTokenType(INP t_token token, OUTP enum e_token_type token_type) {
        if(token.type != token_type) {
                return FALSE;
        }
        return TRUE;
}




void my_atof_2D(INOUTP float **matrix, INP int max_i, INP int max_j, INP char *instring) {
        int i, j;
        char *cur, *cur2, *copy, *final;
        int string_count;

        copy = my_strdup(instring);
        final = copy;
        while(*final != '\0') {
                final++;
        }


        cur = copy;
        i = j = 0;
        while(cur != final) {
                string_count = 0;
                while(IsWhitespace(*cur) && cur != final) {
                        if(j == max_j) {
                                i++;
                                j = 0;
                        }
                        cur++;
                }
                if(cur == final) {
                        break;
                }
                cur2 = cur;
                while(!IsWhitespace(*cur2) && cur2 != final) {
                        cur2++;
                }
                *cur2 = '\0';
                assert(i < max_i && j < max_j);
                matrix[i][j] = atof(cur);
                j++;
                cur = cur2;
                *cur = ' ';
        }

        assert ((i == max_i && j == 0) || (i == max_i - 1 && j == max_j));
        
        free(copy);
}