Tokenization issue

Hey, I was writing a program that is supposed to tokenize strings. I've got it to where the code looks for a delimiter and then uses that to store each part of the string into an array. However when it is compiled and run, for example using string "pickle, company, arm, military, chinese, example," I get things such as 'arm, (' or 'military, v[u[foreign letter]F' Below is the code I've written thus far. Help would be appreciated.

Tokens* createTokens(String* str, char delimiter)
{
int start_size = 1;
Tokens* tokens = initTokens(start_size);
int size = find(str, delimiter, start_size);

String** token = new String*[tokens->max_tokens];
int j = 0;
int start = 0;
while(size != -1)
{
token[j] = substr(str, start, size+1);
addToken(tokens, str);
start = size + 1;
size = find(str, delimiter, start);
j++;
}
tokens->tokens = token;
tokens->tokens[size] = 0;
return tokens;
}

void resize(Tokens* tokens)
{
int token_array_size = tokens->max_tokens;
token_array_size = token_array_size * 2;
String** token = new String*[token_array_size];
for(int i = 0; i < token_array_size; i++)
{
token[i] = tokens->tokens[i];
}

delete[] tokens->tokens;
tokens->tokens = token;
}

void addToken(Tokens* tokens, String* str)
{
if(tokens->sz == tokens->max_tokens) resize(tokens);
tokens->tokens[tokens->max_tokens] = str;
tokens->sz = tokens->sz + 1;
}

String* substr(String* str, int start, int end)
{
int size = end;
char* string = str->char_array;
char sub[size+1];
int j = 0;

for(int i = start; i <= end; i++)
{
sub[j] = string[i];
j++;
}
sub[size] = 0;
String* substr = createString(sub);
return substr;
}
Topic archived. No new replies allowed.