strtok function c element considered problem

When I try to split a string with a delimiter, the strtok function doesn't consider empty strings and I would like to know how to fix it.
For example, in my case, I have to split strings like "a-b-c". When I try to split the string "a--b-c", the result should be "a", "", "b", "c" (note the empty string). But if I print the result of each element after calling the function, it only considers "a", "b", and "c". Why does this happen, and how can I fix it?

    char str[] = "a--b-c";
    char *delim = "-";
    char *token;

    token = strtok(str, delim);

    while (token != NULL) {
        printf("element is: =====%s===== \n ", token);
        token = strtok(NULL, delim);
    }

Elements printed should be 4 (a, empty string, b, c) but with this code, it's only 3 (a, b, c).

According to the C standard, the first action of strtok() is that it

> searches the string pointed to by s1 for the first character that is not
contained in the current separator string pointed to by s2.

If you want different behaviour, you'll need to write your own function that does what you want.

The function strtok searches for the first character that is not contained in the current separator string. So adjacent separator characters are skipped.

Also the function changes the source string.

Instead of using strtok you can use another string function as for example strcspn.

Here is a demonstration program.

#include <stdio.h>
#include <string.h>

int main( void )
{
	char str[] = "a--b-c";
	const char *delim = "-";

	for (char *p = str; *p != '
#include <stdio.h>
#include <string.h>
int main( void )
{
char str[] = "a--b-c";
const char *delim = "-";
for (char *p = str; *p != '\0'; )
{
size_t n = strcspn( p, delim );
printf( "\"%.*s\"\n", ( int )n, p );
if ( *( p += n ) != '\0') ++p;
}
}
'; )
	{
		size_t n = strcspn( p, delim );

		printf( "\"%.*s\"\n", ( int )n, p );

		if ( *( p += n ) != '
#include <stdio.h>
#include <string.h>
int main( void )
{
char str[] = "a--b-c";
const char *delim = "-";
for (char *p = str; *p != '\0'; )
{
size_t n = strcspn( p, delim );
printf( "\"%.*s\"\n", ( int )n, p );
if ( *( p += n ) != '\0') ++p;
}
}
') ++p;
	}
}

The program output is

"a"
""
"b"
"c"

If for a string like that "-" you need to output two substrings like

""
""

or for a string like that "a--b-c-" you need to output

"a"
""
"b"
"c"
""

then it is enough to make the following minor changes in the program

#include <stdio.h>
#include <string.h>

int main( void )
{
	char str[] = "a--b-c-";
	const char *delim = "-";

	for (char *p = str; *p != '
#include <stdio.h>
#include <string.h>
int main( void )
{
char str[] = "a--b-c-";
const char *delim = "-";
for (char *p = str; *p != '\0'; )
{
size_t n = strcspn( p, delim );
printf( "\"%.*s\"\n", ( int )n, p );
if (*( p += n ) != '\0')
{
if (*++p == '\0')
{
printf( "\"%s\"\n", p );
}
}
}
}
'; )
	{
		size_t n = strcspn( p, delim );

		printf( "\"%.*s\"\n", ( int )n, p );

		if (*( p += n ) != '
#include <stdio.h>
#include <string.h>
int main( void )
{
char str[] = "a--b-c-";
const char *delim = "-";
for (char *p = str; *p != '\0'; )
{
size_t n = strcspn( p, delim );
printf( "\"%.*s\"\n", ( int )n, p );
if (*( p += n ) != '\0')
{
if (*++p == '\0')
{
printf( "\"%s\"\n", p );
}
}
}
}
')
		{
			if (*++p == '
#include <stdio.h>
#include <string.h>
int main( void )
{
char str[] = "a--b-c-";
const char *delim = "-";
for (char *p = str; *p != '\0'; )
{
size_t n = strcspn( p, delim );
printf( "\"%.*s\"\n", ( int )n, p );
if (*( p += n ) != '\0')
{
if (*++p == '\0')
{
printf( "\"%s\"\n", p );
}
}
}
}
')
			{
				printf( "\"%s\"\n", p );
			}
		}
	}
}

The program output in this case is

"a"
""
"b"
"c"
""

For each gotten substring pointed to by the pointer p and with the length n you could either define a variable length character array or dynamically allocate a character array and then copy the substring in the array if it is required. For example

#include <stdio.h>
#include <string.h>

int main( void )
{
	char str[] = "a--b-c";
	const char *delim = "-";

	for (char *p = str; *p != '
#include <stdio.h>
#include <string.h>
int main( void )
{
char str[] = "a--b-c";
const char *delim = "-";
for (char *p = str; *p != '\0'; )
{
size_t n = strcspn( p, delim );
char substr[n + 1];
memcpy( substr, p, n );
substr[n] = '\0';
printf( "\"%s\"\n", substr );
if ( *( p += n ) != '\0') ++p;
}
}
'; )
	{
		size_t n = strcspn( p, delim );

		char substr[n + 1];

		memcpy( substr, p, n );
		substr[n] = '
#include <stdio.h>
#include <string.h>
int main( void )
{
char str[] = "a--b-c";
const char *delim = "-";
for (char *p = str; *p != '\0'; )
{
size_t n = strcspn( p, delim );
char substr[n + 1];
memcpy( substr, p, n );
substr[n] = '\0';
printf( "\"%s\"\n", substr );
if ( *( p += n ) != '\0') ++p;
}
}
';
		     
		printf( "\"%s\"\n", substr );

		if ( *( p += n ) != '
#include <stdio.h>
#include <string.h>
int main( void )
{
char str[] = "a--b-c";
const char *delim = "-";
for (char *p = str; *p != '\0'; )
{
size_t n = strcspn( p, delim );
char substr[n + 1];
memcpy( substr, p, n );
substr[n] = '\0';
printf( "\"%s\"\n", substr );
if ( *( p += n ) != '\0') ++p;
}
}
') ++p;
	}
}

The function strtok will not give you any empty tokens. If you want to change that behavior, you will have you write your own strtok function, which is not that hard. Here is an example:

#include <stdio.h>
#include <string.h>

char *my_strtok( char *str, const char *delim )
{
    //this pointer will always point to the next token to
    //be processed and returned, or NULL if we have already
    //returned the last token
    static char *next;

    char *p;

    if ( str == NULL )
    {
        //check whether we have already returned the last
        //token
        if ( next == NULL )
        {
            //we already returned the last token in a
            //previous function call, so return NULL to
            //indicate that there are no more tokens
            return NULL;
        }

        //the next token is available, so process it
        str = next;
    }

    //find the next delimiter character
    p = strpbrk( str, delim );

    if ( p == NULL )
    {
        //this is the last token, so remember that there are
        //no more tokens
        next = NULL;
    }
    else
    {
        //there are more tokens available, so write a null
        //terminating character at the end of the current
        //token and remember the start of the next token
        *p = '
#include <stdio.h>
#include <string.h>
char *my_strtok( char *str, const char *delim )
{
//this pointer will always point to the next token to
//be processed and returned, or NULL if we have already
//returned the last token
static char *next;
char *p;
if ( str == NULL )
{
//check whether we have already returned the last
//token
if ( next == NULL )
{
//we already returned the last token in a
//previous function call, so return NULL to
//indicate that there are no more tokens
return NULL;
}
//the next token is available, so process it
str = next;
}
//find the next delimiter character
p = strpbrk( str, delim );
if ( p == NULL )
{
//this is the last token, so remember that there are
//no more tokens
next = NULL;
}
else
{
//there are more tokens available, so write a null
//terminating character at the end of the current
//token and remember the start of the next token
*p = '\0';
next = p + 1;
}
//return the token
return str;
}
int main( void )
{
char str[] = "a--b-c";
char *delim = "-";
char *token;
token = my_strtok(str, delim);
while ( token != NULL )
{
printf( "Found token: \"%s\"\n", token );
token = my_strtok( NULL, delim );
}
}
';
        next = p + 1;
    }

    //return the token
    return str;
}

int main( void )
{
    char str[] = "a--b-c";
    char *delim = "-";
    char *token;

    token = my_strtok(str, delim);

    while ( token != NULL )
    {
        printf( "Found token: \"%s\"\n", token );
        token = my_strtok( NULL, delim );
    }
}

This program has the following output:

Found token: "a"
Found token: ""
Found token: "b"
Found token: "c"

strtok does not work the way you expect. The specification for this function is clear: all sequences of characters from the delimiter string form single delimiters and initial and trailing delimiters are consumed without producing tokens. Therefore strtok() never produces empty tokens.

The rationale for this is the original purpose of this function: to split words separated by whitespace. Passing a separator string of " \t\f\r\n\v" achieves this goal and it is rather intuitive that sequences of white space characters be considered a single separator.

The behavior you describe corresponds to the strsep() function originally developed for the BSD Unix variant and available on most Unix platforms, including linux. Here is an abstract from the manual page:

> NAME
>
> strsep - separate strings
>
>SYNOPSIS
>
> #include <string.h>
>
> char *strsep(char **stringp, const char *delim);
>
> DESCRIPTION
>
> The strsep() function locates, in the string referenced by *stringp, the first occurrence of any character in the string delim (or the terminating '\0' character) and replaces it with a '\0'. The location of the next character after the delimiter character (or NULL, if the end of the string was reached) is stored in *stringp. The original value of *stringp is returned.
>
> An "empty" field (i.e., a character in the string delim occurs as the first character of *stringp) can be detected by comparing the location referenced by the returned pointer to '\0'.
>
> If *stringp is initially NULL, strsep() returns NULL.

Contrary to strtok(), this function does not store a context in a static variable, so it is thread safe and reentrant, it can be used in nested loops. It is also easier to use in loops as all calls in the sequence have the same arguments.

If this function is not available on your platform, you can use this definition:

#include <string.h>

char *strsep(char **stringp, const char *delim) {
    char *start = *stringp;
    if (start) {
        /* skip all characters in the token (not delimiters) */
        char *p = start + strcspn(start, delim);
        if (*p) {
            *p++ = '
#include <string.h>
char *strsep(char **stringp, const char *delim) {
char *start = *stringp;
if (start) {
/* skip all characters in the token (not delimiters) */
char *p = start + strcspn(start, delim);
if (*p) {
*p++ = '\0';
*stringp = p;
} else {
*stringp = NULL;
}
}
return start;
}
';
            *stringp = p;
        } else {
            *stringp = NULL;
        }
    }
    return start;
}

Here is a modified version of your test code:

#include <stdio.h>
#include <string.h>

int main(void) {
    char str[] = "a--b-c";
    const char *delim = "-";
    char *token;
    char *ctx = str;

    while ((token = strsep(&ctx, delim)) != NULL) {
        printf("element is: >>>%s<<<\n", token);
    }
    return 0;
}

Output:

element is: >>>a<<<
element is: >>><<<
element is: >>>b<<<
element is: >>>c<<<