invoxiplaygames.uk
converts between regular CSV files and the binary CSV format as used in Harmonix Forge engine games (RB4, RBVR, etc)
1#include <stdio.h>
2#include <stdint.h>
3#include <string.h>
4#include <stdlib.h>
5
6#include "csv.h"
7
8#define DEFAULT_SEPERATOR ','
9
10char *current_string_table = NULL;
11uint32_t current_string_table_sz = 0;
12uint32_t current_string_table_ptr = 0;
13char current_seperator = DEFAULT_SEPERATOR;
14static char quote = '"';
15
16typedef struct _CsvResourceHeader {
17 uint32_t mRevision;
18 uint32_t mUnk;
19 uint8_t mSeperator;
20 uint32_t mStringTableLength;
21} __attribute__((packed)) CsvResourceHeader;
22
23const char *get_string_from_table(uint32_t offset) {
24 if (current_string_table == NULL) {
25 fprintf(stderr, "%s: no string table loaded.\n", __func__);
26 return NULL;
27 }
28 if (offset > current_string_table_sz) {
29 fprintf(stderr, "%s: offset (0x%x) larger than string table (0x%x).\n", __func__, offset, current_string_table_sz);
30 return NULL;
31 }
32 return current_string_table + offset;
33}
34
35int binrow_to_txtrow(FILE *bin, FILE *txt) {
36 char newln[] = { '\r', '\n' };
37 int r = 0;
38 uint32_t num_cols = 0;
39 uint32_t *columns = NULL;
40
41 // if we don't have a string table, bail out
42 if (current_string_table == NULL) {
43 fprintf(stderr, "%s: no string table loaded.\n", __func__);
44 return -1;
45 }
46
47 // read the number of columns from the file
48 num_cols = 0;
49 r = fread(&num_cols, sizeof(int), 1, bin);
50 if (r != 1 || num_cols < 0) { // sanity check
51 fprintf(stderr, "%s: failed to read number of columns.\n", __func__);
52 return -1;
53 }
54
55 // allocate enough room for the columns in memory temporarily
56 columns = malloc(sizeof(uint32_t) * num_cols);
57 if (columns == NULL) {
58 fprintf(stderr, "%s: failed to allocate memory for columns.\n", __func__);
59 return -1;
60 }
61 r = fread(columns, sizeof(uint32_t), num_cols, bin);
62 if (r != num_cols) {
63 fprintf(stderr, "%s: failed to read columns.\n", __func__);
64 free(columns);
65 return -1;
66 }
67
68 // write the column text data out to the text file
69 for (int i = 0; i < num_cols; i++) {
70 int str_esc_len = 0;
71 const char *str = NULL;
72 char *str_esc = NULL;
73 int j = 0;
74 int k = 0;
75
76 str = get_string_from_table(columns[i]);
77 if (str == NULL) {
78 fprintf(stderr, "%s: failed to read column from string table.\n", __func__);
79 free(columns);
80 return -1;
81 }
82
83 // get the size of the string when escaped
84 for (j=0; str[j]; str[j]==quote ? str_esc_len += 2 : str_esc_len++, j++);
85 // and allocate a buffer for it
86 str_esc = malloc(str_esc_len + 1);
87 if (str_esc == NULL) {
88 fprintf(stderr, "%s: failed to allocate buffer for escaped string.\n", __func__);
89 free(columns);
90 return -1;
91 }
92 // and copy the string, escaping it
93 k = 0;
94 for (j = 0; str[j]; j++) {
95 if (str[j] == quote)
96 str_esc[k++] = quote;
97 str_esc[k++] = str[j];
98 }
99 str_esc[k] = '\0';
100
101 // TODO(Emma): check if the writes fail here
102 fwrite("e, 1, sizeof(char), txt);
103 fwrite(str_esc, 1, strlen(str_esc), txt);
104 fwrite("e, 1, sizeof(char), txt);
105 // write the seperator if it isn't
106 if (i < (num_cols - 1))
107 fwrite(¤t_seperator, 1, sizeof(char), txt);
108
109 free(str_esc);
110 }
111
112 // write a CRLF newline
113 fwrite(newln, sizeof(newln), 1, txt);
114
115 // successfully read and output - free the buffer we made before
116 free(columns);
117 return 0;
118}
119
120int bincsv_to_csv(const char *bincsv_file, const char *csv_file) {
121 FILE *bincsv = NULL;
122 FILE *csv = NULL;
123 CsvResourceHeader resHdr = {0};
124 int r = 0;
125 int num_rows = 0;
126
127 // open the files
128 bincsv = fopen(bincsv_file, "rb");
129 if (bincsv == NULL) {
130 fprintf(stderr, "%s: failed to read from '%s'.\n", __func__, bincsv_file);
131 return -1;
132 }
133 csv = fopen(csv_file, "wb"); // load as binary; even tho it's text data, i don't trust it
134 if (csv == NULL) {
135 fprintf(stderr, "%s: failed to open '%s' for writing.\n", __func__, csv_file);
136 fclose(bincsv);
137 return -1;
138 }
139
140 // read the file header
141 r = fread(&resHdr, sizeof(CsvResourceHeader), 1, bincsv);
142 if (r != 1) {
143 fprintf(stderr, "%s: failed to read CsvResource header.\n", __func__);
144 fclose(bincsv);
145 fclose(csv);
146 return -1;
147 }
148 // check the version
149 if (resHdr.mRevision != 0x1 || resHdr.mUnk != 0x2) {
150 fprintf(stderr, "%s: unknown CsvResource type (0x%x, 0x%x)\n", __func__, resHdr.mRevision, resHdr.mUnk);
151 fclose(bincsv);
152 fclose(csv);
153 return -1;
154 }
155 current_seperator = resHdr.mSeperator;
156
157 // read the string table
158 current_string_table_sz = resHdr.mStringTableLength;
159 current_string_table = malloc(resHdr.mStringTableLength);
160 if (current_string_table == NULL) {
161 fprintf(stderr, "%s: failed to allocate string table of size 0x%x\n", __func__, current_string_table_sz);
162 fclose(bincsv);
163 fclose(csv);
164 return -1;
165 }
166 r = fread(current_string_table, 1, current_string_table_sz, bincsv);
167 if (r != current_string_table_sz) {
168 fprintf(stderr, "%s: failed to read string table from file.\n", __func__);
169 fclose(bincsv);
170 fclose(csv);
171 free(current_string_table);
172 current_string_table = NULL;
173 return -1;
174 }
175
176 // read the header row
177 if (binrow_to_txtrow(bincsv, csv) < 0) {
178 fprintf(stderr, "%s: failed to read header row from file.\n", __func__);
179 fclose(bincsv);
180 fclose(csv);
181 free(current_string_table);
182 current_string_table = NULL;
183 return -1;
184 }
185
186 // read out all the rowsnum_cols = 0;
187 r = fread(&num_rows, sizeof(int), 1, bincsv);
188 if (r != 1 || num_rows < 0) { // sanity check
189 fprintf(stderr, "%s: failed to read number of rows.\n", __func__);
190 return -1;
191 }
192 for (int i = 0; i < num_rows; i++) {
193 if (binrow_to_txtrow(bincsv, csv) < 0) {
194 fprintf(stderr, "%s: failed to read row %i from file.\n", __func__, i);
195 fclose(bincsv);
196 fclose(csv);
197 free(current_string_table);
198 current_string_table = NULL;
199 return -1;
200 }
201 }
202
203 fclose(bincsv);
204 fclose(csv);
205 free(current_string_table);
206 current_string_table = NULL;
207 return 0;
208}
209
210uint32_t add_string_to_table(const char *string) {
211 if ((current_string_table_ptr + strlen(string) + 1) > current_string_table_sz) {
212 fprintf(stderr, "%s: bounds check failed adding string of length 0x%lx (ptr 0x%x, sz 0x%x)\n", __func__, strlen(string) + 1, current_string_table_ptr, current_string_table_sz);
213 return 0xFFFFFFFF;
214 }
215
216 uint32_t ptr = current_string_table_ptr;
217 strcpy(current_string_table + current_string_table_ptr, string);
218 current_string_table_ptr += strlen(string) + 1;
219 return ptr;
220}
221
222#define MAX_COLS 16
223int txtrow_to_binrow(CsvHandle csv, char *row, FILE *bin) {
224 int num_cols = 0;
225 uint32_t cols[MAX_COLS] = {0}; // max 16 columns isn't great, but fuck it
226 const char *col = NULL;
227 while ((col = CsvReadNextCol(row, csv))) {
228 cols[num_cols] = add_string_to_table(col);
229 if (cols[num_cols] == 0xFFFFFFFF) {
230 fprintf(stderr, "%s: failed to add string to table!\n", __func__);
231 return -1;
232 }
233 num_cols++;
234 if (num_cols >= MAX_COLS) {
235 fprintf(stderr, "%s: exceeded maximum columns limit.\n", __func__);
236 return -1;
237 }
238 }
239 // TODO(Emma): check for errors when writing
240 fwrite(&num_cols, sizeof(int), 1, bin);
241 fwrite(cols, sizeof(uint32_t), num_cols, bin);
242 return 0;
243}
244
245int csv_to_bincsv(const char *csv_file, const char *bincsv_file) {
246 int r = 0;
247 // This sucks - we read the CSV in its entirety twice to get the size of symbol table,
248 // and number of rows, and then to actually insert those entries to the table.
249 char *row = NULL;
250 int num_rows = 0;
251 uint32_t str_table_size = 0;
252 CsvHandle csv = CsvOpen2(csv_file, current_seperator, '"', '\\');
253 if (csv == NULL) {
254 fprintf(stderr, "%s: failed to open csv file from '%s'.\n", __func__, csv_file);
255 return -1;
256 }
257 while ((row = CsvReadNextRow(csv))) {
258 const char *col = NULL;
259 num_rows++;
260 while ((col = CsvReadNextCol(row, csv)))
261 str_table_size += strlen(col) + 1;
262 }
263 CsvClose(csv);
264
265 // actually allocate the string table
266 current_string_table_sz = str_table_size;
267 current_string_table = malloc(str_table_size);
268 if (current_string_table == NULL) {
269 fprintf(stderr, "%s: failed to allocate string table.\n", __func__);
270 return -1;
271 }
272 memset(current_string_table, 0, str_table_size);
273
274 FILE *bincsv = fopen(bincsv_file, "wb");
275 if (bincsv == NULL) {
276 fprintf(stderr, "%s: failed to open '%s' for writing.\n", __func__, bincsv_file);
277 free(current_string_table);
278 current_string_table = NULL;
279 return -1;
280 }
281
282 // build the resource header and write it to the file
283 CsvResourceHeader resHdr = {0};
284 resHdr.mRevision = 0x1;
285 resHdr.mUnk = 0x2;
286 resHdr.mSeperator = current_seperator;
287 resHdr.mStringTableLength = str_table_size;
288 r = fwrite(&resHdr, sizeof(CsvResourceHeader), 1, bincsv);
289 if (r != 1) {
290 fprintf(stderr, "%s: failed to write header to file.\n", __func__);
291 fclose(bincsv);
292 free(current_string_table);
293 current_string_table = NULL;
294 return -1;
295 }
296 // skip past the size of the string table - we'll write it later
297 fseek(bincsv, str_table_size, SEEK_CUR);
298
299 // read the CSV again and write out each of the lines
300 int num_rows_done = 0;
301 csv = CsvOpen2(csv_file, current_seperator, '"', '\\');
302 if (csv == NULL) {
303 fprintf(stderr, "%s: failed to open csv file from '%s'.\n", __func__, csv_file);
304 fclose(bincsv);
305 free(current_string_table);
306 current_string_table = NULL;
307 return -1;
308 }
309 while ((row = CsvReadNextRow(csv))) {
310 if (txtrow_to_binrow(csv, row, bincsv) < 0) {
311 fprintf(stderr, "%s: failed on row %i ('%s')\n", __func__, num_rows_done, row);
312 CsvClose(csv);
313 fclose(bincsv);
314 free(current_string_table);
315 current_string_table = NULL;
316 return -1;
317 }
318 // after the header row, there's a count of the rest of the rows
319 if (num_rows_done == 0) {
320 int num_rows_file = num_rows - 1; // don't include header row
321 fwrite(&num_rows_file, sizeof(int), 1, bincsv);
322 }
323 num_rows_done++;
324 }
325 CsvClose(csv);
326
327 // write the string table to the file
328 fseek(bincsv, sizeof(CsvResourceHeader), SEEK_SET);
329 r = fwrite(current_string_table, 1, current_string_table_sz, bincsv);
330 if (r != current_string_table_sz) {
331 fprintf(stderr, "%s: failed to write string table to file.\n", __func__);
332 fclose(bincsv);
333 free(current_string_table);
334 current_string_table = NULL;
335 return -1;
336 }
337
338 free(current_string_table);
339 fclose(bincsv);
340 current_string_table = NULL;
341 return 0;
342}
343
344int print_usage(const char *filename) {
345 printf("usage:\n");
346 printf(" %s bin2csv /path/to/input.csv[_xb1|_ps4|_pc] /path/to/output.csv\n", filename);
347 printf(" %s csv2bin /path/to/input.csv /path/to/output.csv[_xb1|_ps4|_pc]\n", filename);
348 return -1;
349}
350
351int main(int argc, const char **argv) {
352 if (argc < 4)
353 return print_usage(argv[0]);
354
355 const char *verb = argv[1];
356 if (strcasecmp(verb, "bin2csv") == 0) {
357 return bincsv_to_csv(argv[2], argv[3]);
358 } else if (strcasecmp(verb, "csv2bin") == 0) {
359 return csv_to_bincsv(argv[2], argv[3]);
360 } else {
361 fprintf(stderr, "invalid verb '%s'\n", verb);
362 return print_usage(argv[0]);
363 }
364}