#include "types.h" #include #include #include #include // Location in the source file typedef struct Location { // Offset in file of the first character i32 line; // Offset on the line of the first character u16 column; // Length in bytes u16 size; } Location_t; typedef struct Token { // identifying tag enum TokenType { t_error, t_size, t_multiply, t_divide, t_add, t_subtract, t_whitespace, t_newline, t_number, } type; // Offset(s) in the buffer where the token occurs. // If type is t_size, then location is an i64 "size" -- only used to store the // number of tokens. // If type is t_newline, then location is a Location_t line, where `line` // holds the offset into the text buffer where the newline is located, column // holds the line number, // and size holds the length of the line. union { Location_t location; Location_t line; i64 size; } location; } Token_t; // #define Location_to_i64(location) \ // (i64)(((i64)location.line << 32) | (location.column << 16) | location.size) // #define i64_to_Location(long_int) \ // (Location_t) { \ // .line = (long_int >> 32), .column = (long_int >> 16) & ((1 << 16) - 1), \ // .size = long_int & ((1 << 16) - 1), \ // } // Should also add helper structure with pointers to each line static isize f_read(FILE* stream, char** dst); static i64 f_size(FILE* stream); static void l_print_token(Token_t token, Location_t** lines, char* text); static Location_t** l_read(char* text, isize text_length, Token_t** dst); int main(int argc, char* argv[]) { if (argc == 2) { FILE* f = NULL; char* contents = NULL; isize content_length = -1; Token_t* tokens = NULL; Location_t** lines = NULL; f = fopen(argv[1], "r"); if (f == NULL) { fprintf(stdout, "! Failed to open file: %s\n", argv[1]); } if (f != NULL) { content_length = f_read(f, &contents); if (content_length > 0) { printf("%s:%ld\n%s\n", argv[1], content_length, contents); } fclose(f); } if (content_length > 0) { lines = l_read(contents, strlen(contents), &tokens); } if (lines != NULL) { i64 num_tokens = tokens[0].location.size; printf("%ld tokens\n", num_tokens); for (int t = 0; t < num_tokens; t++) { l_print_token(tokens[t], lines, contents); } free(contents); } } return 0; } /* Copies the contents of the file stream to *dst. * @param stream An open file stream. * @param dst** Destination buffer. If *dst is NULL allocates a buffer the size * of stream using malloc. * @return The number of bytes copied into dst. * @error returns -1 on error. */ static isize f_read(FILE* stream, char** dst) { i64 file_sz = f_size(stream); if (dst == NULL) { fprintf(stderr, "! Got NULL ptr\n"); } if (dst != NULL && *dst == NULL && file_sz > 0) { *dst = (char*)malloc(file_sz * sizeof(char)); } if (dst != NULL && *dst == NULL && file_sz > 0) { fprintf(stderr, "! Failed to allocate %ld bytes\n", file_sz * sizeof(char)); } isize bytes_read = -1; if (dst != NULL && *dst != NULL && file_sz > 0) { bytes_read = fread(*dst, sizeof(char), file_sz, stream); } if (bytes_read < file_sz) { fprintf(stderr, "! Failed to read file, read %ld out of %ld bytes\n", bytes_read, file_sz); } return bytes_read; } /* Returns the size in bytes and rewinds the stream. * @param stream An open file stream. * @return the size of the file in bytes. * @error Returns -1 on error. */ static i64 f_size(FILE* stream) { // Could ftell and rewind to previous position, but scratch that i64 size = -1; i32 error = fseek(stream, 0, SEEK_END); if (error) { // TODO: handle pipes (check errno == ESPIPE) fprintf(stderr, "! Failed to seek file: %s\n", strerror(errno)); } else { size = ftell(stream); rewind(stream); } return size; } static void l_print_token(Token_t token, Location_t* lines[], char* text) { fprintf(stdout, "[%d,%d:%d]", token.location.location.line + 1, token.location.location.column + 1, token.location.location.size); switch (token.type) { case t_error: fputs(" error ", stdout); break; case t_size: fprintf(stdout, "size"); break; case t_multiply: fputs(" * ", stdout); break; case t_divide: fputs(" / ", stdout); break; case t_add: fputs(" + ", stdout); break; case t_subtract: fputs(" - ", stdout); break; case t_number: fputs("number:", stdout); Location_t* line = lines[token.location.location.line]; fwrite(&text[line->line + token.location.location.column], token.location.location.size, sizeof(char), stdout); break; case t_newline: fprintf(stdout, "\\n offset:%d line:%d length:%d", token.location.line.line, token.location.line.column, token.location.line.size); break; default: break; } puts(""); } /* Reads text and parses it into an array of tokens * @param text A buffer holding the input text * @return a pointer to dst * @error Returns NULL on error. */ static Location_t** l_read(char* text, isize text_length, Token_t** dst) { Location_t** lines = NULL; if (dst == NULL) { fprintf(stderr, "dst is NULL"); } if (text_length > 0 && dst != NULL && *dst == NULL) { *dst = (Token_t*)malloc(text_length * sizeof(Token_t)); } if (text_length > 0 && dst != NULL && *dst == NULL) { fprintf(stderr, "! Failed to allocate %ld bytes\n", text_length * sizeof(Token_t)); } if (text_length > 0 && dst != NULL && *dst != NULL) { // Offset in text (used for Location.line isize i = 0; // Column counter u16 c = 0; // Line offset i32 l = 0; // Token counter isize t = 2; i32 error = 0; enum TokenType token_previous = t_error; enum TokenType token = t_error; (*dst)[1].type = t_newline; Location_t* line_current = &(*dst)[1].location.line; line_current->line = 0; line_current->column = l; line_current->size = 0; // TODO: rewrite to // if token != last-token // dst = token,line,col // last-token = token // t++; // while (last-token != error) do { switch (text[i]) { // case t_size: // break; case '*': token = t_multiply; break; case '/': token = t_divide; break; case t_add: token = t_add; break; case '-': token = t_subtract; break; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': token = t_number; break; case '+': token = t_add; break; case ' ': case '\t': token = t_whitespace; break; case '\n': token = t_newline; break; default: token = t_error; error = 1; break; } if (token == t_newline) { l++; line_current->size = c; (*dst)[t].type = token; (*dst)[t].location.line.line = i + 1; (*dst)[t].location.line.column = l; (*dst)[t].location.line.size = 0; line_current = &(*dst)[t].location.line; c = 0; t++; token_previous = token; } else if (token == token_previous) { (*dst)[t - 1].location.location.size++; c++; } else { (*dst)[t].type = token; (*dst)[t].location.location.line = l; (*dst)[t].location.location.column = c; (*dst)[t].location.location.size = 1; c++; t++; token_previous = token; } i++; } while (!error && i < text_length); (*dst)[0].type = t_size; (*dst)[0].location.size = t; if ((*dst)[t - 1].type != t_newline) { l++; line_current->size = c; } lines = (Location_t**)malloc(sizeof(Location_t*) * l); i32 j = 0; for (int u = 0; u < t; u++) { if ((*dst)[u].type == t_newline) { lines[j] = &(*dst)[u].location.line; j++; } } } return lines; }