reader_tests.h

Go to the documentation of this file.

/* -*- C -*-
 * Serene programming language
 * Copyright (C) 2019-2026 Sameer Rahmani <lxsameer@lxsameer.com>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 
#pragma once
 
#include <string.h>
 
#include <serene/ir/reader.h>
#include <serene/rt/context.h>
#include <serene/runtime.h>
 
#include "base.h"
#include "serene/rt/fiber.h"
 
#define READER_TESTS(X)                                                                            \
  X("ir::reader::read_list_from_buffer", test_reader_read_list_from_buffer),                       \
    X("ir::reader::malformed_utf8_in_string", test_reader_malformed_utf8_in_string),               \
    X("ir::reader::truncated_utf8_at_end", test_reader_truncated_utf8_at_end),                     \
    X("ir::reader::trailing_whitespace_and_comment", test_reader_trailing_whitespace_and_comment), \
    X("ir::reader::tree_has_no_error_on_success", test_reader_tree_has_no_error_on_success),       \
    X("ir::reader::location_len_over_64k", test_reader_location_len_over_64k)
 
// Runs the reader over `src` on its own fiber and returns the fiber result,
// which is nullptr on success and the parse error otherwise.
static srn_error_t *read_source(srn_context_t *ctx, const char *src, size_t len) {
  srn_sexp_parser_t parser = {
    .ctx             = ctx,
    .reading_context = nullptr,
    .buffer_id       = 0,
    .buffer          = (const unsigned char *)src,
    .len             = len,
    .pos             = 0,
  };
  reader_fiber_arg_t fiber_arg = {.parser = &parser, .tree = nullptr};
 
  auto f = srn_fiber_spawn(ctx, read_from_buffer, &fiber_arg);
  srn_sched_run(ctx->engine->scheduler, 1);
  return (srn_error_t *)f->result;
}
 
// A stray continuation byte inside a string literal decodes to a negative
// codepoint that re-encodes as four bytes, more than the one byte the output
// buffer budgets for it. The reader must reject the buffer up front instead of
// letting the string copier run past its allocation.
static void test_reader_malformed_utf8_in_string() {
  MAKE_ENGINE(mm, engine);
  MAKE_CONTEXT(engine, ctx);
 
  const char src[] = "\"\x80\"";
  srn_error_t *err = read_source(ctx, src, sizeof(src) - 1);
 
  ASSERT_NOT_NULL(err);
  TEST_CHECK(err->tag == IR_PARSE_ERROR);
  TEST_CHECK(err->as.metadata->loc->offset == 1);
 
  RELEASE_CONTEXT(ctx);
  SHUTDOWN_ENGINE(mm, engine);
}
 
// A lead byte with no continuation bytes before the terminator must not make
// the reader decode past the end of the buffer.
static void test_reader_truncated_utf8_at_end() {
  MAKE_ENGINE(mm, engine);
  MAKE_CONTEXT(engine, ctx);
 
  const char src[] = "foo \xE2";
  srn_error_t *err = read_source(ctx, src, sizeof(src) - 1);
 
  ASSERT_NOT_NULL(err);
  TEST_CHECK(err->tag == IR_PARSE_ERROR);
  TEST_CHECK(err->as.metadata->loc->offset == 4);
 
  RELEASE_CONTEXT(ctx);
  SHUTDOWN_ENGINE(mm, engine);
}
 
static void test_reader_trailing_whitespace_and_comment() {
  const char *cases[] = {"foo\n", "(a b)\n\n", "foo ; trailing comment", "   ", ""};
 
  // A scheduler runs once per engine, so every case gets its own.
  for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
    MAKE_ENGINE(mm, engine);
    MAKE_CONTEXT(engine, ctx);
 
    srn_error_t *err = read_source(ctx, cases[i], strlen(cases[i]));
    TEST_CHECK_(err == nullptr, "case %zu: '%s'", i, cases[i]);
 
    RELEASE_CONTEXT(ctx);
    SHUTDOWN_ENGINE(mm, engine);
  }
}
 
// The tree handed back after a successful read reports no error. Arena memory
// is not zeroed, so a few contexts are filled with junk and released first,
// which puts a recycled block under the tree instead of a fresh page.
static void test_reader_tree_has_no_error_on_success() {
  MAKE_ENGINE(mm, engine);
 
  for (int round = 0; round < 3; round++) {
    srn_context_t *junk = srn_context_make(engine);
    const size_t n      = 64 * 1024;
    char *p             = ALLOCN_OR_PANIC(junk, char, n);
    memset(p, 0xFF, n);
    srn_context_release(junk);
  }
 
  MAKE_CONTEXT(engine, ctx);
 
  const char *src          = "(foo)";
  srn_sexp_parser_t parser = {
    .ctx             = ctx,
    .reading_context = nullptr,
    .buffer_id       = 0,
    .buffer          = (const unsigned char *)src,
    .len             = strlen(src),
    .pos             = 0,
  };
  reader_fiber_arg_t fiber_arg = {.parser = &parser, .tree = nullptr};
 
  auto f = srn_fiber_spawn(ctx, read_from_buffer, &fiber_arg);
  srn_sched_run(engine->scheduler, 1);
  ASSERT_NULL(f->result);
 
  srn_sexp_tree_t *tree = fiber_arg.tree;
  ASSERT_NOT_NULL(tree);
  TEST_CHECK(!HAS_ERROR(*tree));
 
  RELEASE_CONTEXT(ctx);
  SHUTDOWN_ENGINE(mm, engine);
}
 
// A form longer than 64 KiB records its full length in the location.
static void test_reader_location_len_over_64k() {
  MAKE_ENGINE(mm, engine);
  MAKE_CONTEXT(engine, ctx);
 
  const size_t body = 70000;
  char *src         = ALLOCN_OR_PANIC(ctx, char, body + 3);
  src[0]            = '"';
  memset(src + 1, 'a', body);
  src[body + 1] = '"';
  src[body + 2] = '\0';
 
  srn_sexp_parser_t parser = {
    .ctx             = ctx,
    .reading_context = nullptr,
    .buffer_id       = 0,
    .buffer          = (const unsigned char *)src,
    .len             = body + 2,
    .pos             = 0,
  };
  reader_fiber_arg_t fiber_arg = {.parser = &parser, .tree = nullptr};
 
  auto f = srn_fiber_spawn(ctx, read_from_buffer, &fiber_arg);
  srn_sched_run(engine->scheduler, 1);
  ASSERT_NULL(f->result);
 
  srn_sexp_node_t *node = array_list_get(&fiber_arg.tree->exprs, 0).data;
  ASSERT_NOT_NULL(node);
  TEST_CHECK(node->metadata->loc->offset == 0);
  TEST_CHECK(node->metadata->loc->len == body + 2);
 
  RELEASE_CONTEXT(ctx);
  SHUTDOWN_ENGINE(mm, engine);
}
 
// read_from_buffer parses a top level list into a proper, nullptr terminated
// cons chain (not the circular chain the naive prepend-based construction
// used to produce), preserving element order.
static void test_reader_read_list_from_buffer() {
  MAKE_ENGINE(mm, engine);
  MAKE_CONTEXT(engine, ctx);
  auto sched = engine->scheduler;
 
  const char *src = "(foo bar baz)";
 
  srn_sexp_parser_t parser = {
    .ctx             = ctx,
    .reading_context = nullptr,
    .buffer_id       = 0,
    .buffer          = (const unsigned char *)src,
    .len             = strlen(src),
    .pos             = 0,
  };
 
  reader_fiber_arg_t fiber_arg = {
    .parser = &parser,
    .tree   = nullptr,
  };
 
  auto f = srn_fiber_spawn(ctx, read_from_buffer, &fiber_arg);
  srn_sched_run(sched, 1);
 
  srn_fiber_result_t result = f->result;
  ASSERT_NULL(result);
 
  srn_sexp_tree_t *tree = fiber_arg.tree;
  ASSERT_NOT_NULL(tree);
  TEST_CHECK(tree->exprs.len == 1);
 
  srn_sexp_node_t *root = array_list_get(&tree->exprs, 0).data;
  ASSERT_NOT_NULL(root);
  TEST_CHECK(root->type == SRN_SEXP_LIST);
  TEST_CHECK(root->as.list->len == 3);
 
  srn_sexp_node_t *cons1 = root->as.list->head;
  ASSERT_NOT_NULL(cons1);
  TEST_CHECK(cons1->type == SRN_SEXP_CONS);
  TEST_CHECK(strcmp(cons1->as.cons->head->as.atom->name, "foo") == 0);
 
  srn_sexp_node_t *cons2 = cons1->as.cons->tail;
  ASSERT_NOT_NULL(cons2);
  TEST_CHECK(strcmp(cons2->as.cons->head->as.atom->name, "bar") == 0);
 
  srn_sexp_node_t *cons3 = cons2->as.cons->tail;
  ASSERT_NOT_NULL(cons3);
  TEST_CHECK(strcmp(cons3->as.cons->head->as.atom->name, "baz") == 0);
 
  // The chain must terminate, not loop back to cons1.
  TEST_CHECK(cons3->as.cons->tail == nullptr);
 
  RELEASE_CONTEXT(ctx);
  SHUTDOWN_ENGINE(mm, engine);
}