Skip to content

Read 64-bit multi-page SAS catalog files and skip format default values #370

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
evanmiller merged 2 commits into from
Sep 12, 2026
Merged

Read 64-bit multi-page SAS catalog files and skip format default values #370

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2a94e88
Fix build
hpoettker Sep 12, 2026
3311812
Read 64-bit multi-page SAS catalog files and skip format default values
hpoettker Sep 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion Makefile.am
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,9 @@ EXTRA_DIST = \
resources/compression_type_0x02.sas7bdat \
resources/compression_type_0x06.sas7bdat \
resources/compression_type_0x09.sas7bdat \
resources/compression_type_0x0d.sas7bdat
resources/compression_type_0x0d.sas7bdat \
resources/format_with_default_32bit.sas7bcat \
resources/format_with_default_64bit.sas7bcat

include_HEADERS = src/readstat.h

Expand Down
Binary file added resources/format_with_default_32bit.sas7bcat
View file
Open in desktop
Binary file not shown.
Binary file added resources/format_with_default_64bit.sas7bcat
View file
Open in desktop
Binary file not shown.
119 changes: 72 additions & 47 deletions src/sas/readstat_sas7bcat_read.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,22 +43,34 @@ static void sas7bcat_ctx_free(sas7bcat_ctx_t *ctx) {
}

static readstat_error_t sas7bcat_parse_value_labels(const char *value_start, size_t value_labels_len,
int label_count_used, int label_count_capacity, int string_offset, const char *name,
sas7bcat_ctx_t *ctx) {
int label_count_used, int label_count_capacity, int default_label_pos, int string_offset,
const char *name, sas7bcat_ctx_t *ctx) {
readstat_error_t retval = READSTAT_OK;
int i;
const char *lbp1 = value_start;
uint32_t *value_offset = readstat_calloc(label_count_used, sizeof(uint32_t));
uint32_t *value_offset = NULL;
/* Doubles appear to be stored as big-endian, always */
int bswap_doubles = machine_is_little_endian();
int is_string = (name[0] == '$');
int is_string = 0;
char *label = NULL;
char *string_val = NULL;

if (name[0] == '$') {
is_string = 1;
} else if (name[0] == '@') {
goto cleanup; // Skip informats
}

value_offset = readstat_calloc(label_count_used, sizeof(uint32_t));
if (value_offset == NULL) {
retval = READSTAT_ERROR_MALLOC;
goto cleanup;
}
/* Mark every slot as unset so that a label index which no value entry
* references is detected in pass 2 instead of silently reading entry 0 */
for (i=0; i<label_count_used; i++) {
value_offset[i] = UINT32_MAX;
}

/* Pass 1 -- find out the offset of the labels */
for (i=0; i<label_count_capacity; i++) {
Expand All @@ -72,7 +84,7 @@ static readstat_error_t sas7bcat_parse_value_labels(const char *value_start, siz
goto cleanup;
}
uint32_t label_pos = sas_read4(&lbp1[10+ctx->pad1], ctx->bswap);
if (label_pos >= label_count_used) {
if (label_pos >= label_count_used || (default_label_pos > 0 && label_pos == default_label_pos - 1)) {
retval = READSTAT_ERROR_PARSE;
goto cleanup;
}
Expand All @@ -84,64 +96,73 @@ static readstat_error_t sas7bcat_parse_value_labels(const char *value_start, siz
const char *lbp2 = lbp1;

/* Pass 2 -- parse pairs of values & labels */
for (i=0; i<label_count_used && i<label_count_capacity; i++) {
lbp1 = value_start + value_offset[i];

if (&lbp1[30] - value_start > value_labels_len ||
&lbp2[10] - value_start > value_labels_len) {
retval = READSTAT_ERROR_PARSE;
goto cleanup;
}
readstat_value_t value = { .type = is_string ? READSTAT_TYPE_STRING : READSTAT_TYPE_DOUBLE };
if (is_string) {
size_t value_entry_len = 6 + sas_read2(&lbp1[2], ctx->bswap);
size_t string_start = 22 + string_offset;
if (string_start > value_entry_len ||
&lbp1[value_entry_len] - value_start > value_labels_len) {
for (i=0; i<label_count_used; i++) {
readstat_value_t value = { 0 };
int is_default_label = default_label_pos > 0 && i == default_label_pos - 1;
if (!is_default_label) {
/* A non-default label with no value entry pointing at it would
* otherwise read from offset 0 and emit a duplicate value */
if (value_offset[i] == UINT32_MAX) {
retval = READSTAT_ERROR_PARSE;
goto cleanup;
}
size_t string_len = value_entry_len - string_start;
char *new_string_val = realloc(string_val, 4 * string_len + 1);
if (new_string_val == NULL) {
retval = READSTAT_ERROR_MALLOC;
lbp1 = value_start + value_offset[i];

if (&lbp1[30] - value_start > value_labels_len) {
retval = READSTAT_ERROR_PARSE;
goto cleanup;
}
string_val = new_string_val;
retval = readstat_convert(string_val, 4 * string_len + 1,
&lbp1[string_start], string_len, ctx->converter);
if (retval != READSTAT_OK)
goto cleanup;
value.type = is_string ? READSTAT_TYPE_STRING : READSTAT_TYPE_DOUBLE;
if (is_string) {
size_t value_entry_len = 6 + sas_read2(&lbp1[2], ctx->bswap);
size_t string_start = 22 + string_offset;
if (string_start > value_entry_len ||
&lbp1[value_entry_len] - value_start > value_labels_len) {
retval = READSTAT_ERROR_PARSE;
goto cleanup;
}
size_t string_len = value_entry_len - string_start;
char *new_string_val = realloc(string_val, 4 * string_len + 1);
if (new_string_val == NULL) {
retval = READSTAT_ERROR_MALLOC;
goto cleanup;
}
string_val = new_string_val;
retval = readstat_convert(string_val, 4 * string_len + 1,
&lbp1[string_start], string_len, ctx->converter);
if (retval != READSTAT_OK)
goto cleanup;

value.v.string_value = string_val;
} else {
uint64_t val = sas_read8(&lbp1[22], bswap_doubles);
double dval = NAN;
if ((val | 0xFF0000000000) == 0xFFFFFFFFFFFF) {
sas_assign_tag(&value, (val >> 40));
value.v.string_value = string_val;
} else {
memcpy(&dval, &val, 8);
if (dval > 0.0) {
val = ~val;
memcpy(&dval, &val, 8);
uint64_t val = sas_read8(&lbp1[22], bswap_doubles);
double dval = NAN;
if ((val | 0xFF0000000000) == 0xFFFFFFFFFFFF) {
sas_assign_tag(&value, (val >> 40));
} else {
dval *= -1.0;
memcpy(&dval, &val, 8);
if (dval > 0.0) {
val = ~val;
memcpy(&dval, &val, 8);
} else {
dval *= -1.0;
}
}
}

value.v.double_value = dval;
value.v.double_value = dval;
}
}
size_t label_len = sas_read2(&lbp2[8], ctx->bswap);
if (&lbp2[10] > value_start + value_labels_len) {
retval = READSTAT_ERROR_PARSE;
goto cleanup;
}
size_t label_len = sas_read2(&lbp2[8], ctx->bswap);
/* Some labels seem to overflow the reported block length, truncate it */
/* (Observed with formats.sasbcat from GSS2021, produced with 9.0401M6X64_SR12R2 */
if (label_len > value_start + value_labels_len - &lbp2[10]) {
label_len = value_start + value_labels_len - &lbp2[10];
}
if (ctx->value_label_handler) {
if (ctx->value_label_handler && !is_default_label) {
label = realloc(label, 4 * label_len + 1);
retval = readstat_convert(label, 4 * label_len + 1,
&lbp2[10], label_len, ctx->converter);
Expand Down Expand Up @@ -170,6 +191,7 @@ static readstat_error_t sas7bcat_parse_block(const char *data, size_t data_size,
size_t pad = 0;
uint64_t label_count_capacity = 0;
uint64_t label_count_used = 0;
uint64_t default_label_pos = 0;
int string_offset = 0;
int payload_offset = 106;
uint16_t flags = 0;
Expand All @@ -185,6 +207,7 @@ static readstat_error_t sas7bcat_parse_block(const char *data, size_t data_size,
goto cleanup;
label_count_capacity = sas_read8(&data[42+pad], ctx->bswap);
label_count_used = sas_read8(&data[50+pad], ctx->bswap);
default_label_pos = sas_read8(&data[74+pad], ctx->bswap);
/* Nonzero when the format's values are strings longer than 16 bytes,
* which shifts each value one byte to the right and NUL-pads it */
string_offset = sas_read4(&data[124+pad], ctx->bswap);
Expand All @@ -195,10 +218,11 @@ static readstat_error_t sas7bcat_parse_block(const char *data, size_t data_size,
goto cleanup;
label_count_capacity = sas_read4(&data[38+pad], ctx->bswap);
label_count_used = sas_read4(&data[42+pad], ctx->bswap);
default_label_pos = sas_read4(&data[66+pad], ctx->bswap);
string_offset = sas_read2(&data[104+pad], ctx->bswap);
}

if (string_offset < 0) {
if (string_offset < 0 || default_label_pos > label_count_used) {
retval = READSTAT_ERROR_PARSE;
goto cleanup;
}
Expand Down Expand Up @@ -227,7 +251,7 @@ static readstat_error_t sas7bcat_parse_block(const char *data, size_t data_size,
goto cleanup;

if ((retval = sas7bcat_parse_value_labels(&data[payload_offset+pad], data_size - payload_offset - pad,
label_count_used, label_count_capacity, string_offset, name, ctx)) != READSTAT_OK)
label_count_used, label_count_capacity, default_label_pos, string_offset, name, ctx)) != READSTAT_OK)
goto cleanup;

cleanup:
Expand Down Expand Up @@ -508,8 +532,9 @@ readstat_error_t readstat_parse_sas7bcat(readstat_parser_t *parser, const char *
retval = READSTAT_ERROR_READ;
goto cleanup;
}
if (memcmp(&page[16], "XLSR", sizeof("XLSR")-1) == 0) {
retval = sas7bcat_augment_index(&page[16], ctx->page_size - 16, ctx);
int offset = ctx->u64 ? 32 : 16;
if (memcmp(&page[offset], "XLSR", sizeof("XLSR")-1) == 0) {
retval = sas7bcat_augment_index(&page[offset], ctx->page_size - offset, ctx);
if (retval != READSTAT_OK)
goto cleanup;
}
Expand Down
Loading