diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 59974295..8ff9d06c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -313,45 +313,6 @@ jobs: echo or upstream/master depending on your setup fi - reporting: - name: "Pull Request Report" - # if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'brwarner/inkcpp' }} - if: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.ref == 'master' }} - runs-on: ubuntu-latest - needs: [compilation, clang-format] - permissions: - pull-requests: write - steps: - # Download Ink Proof Results - - uses: actions/download-artifact@v4 - with: - pattern: result-* - path: "results" - merge-multiple: true - - # Create comment text - - name: Create Comment Text File - shell: bash - run: | - echo "### Ink Proof Results" >> comment.txt - echo "" >> comment.txt - echo "These results are obtained by running the [Ink-Proof Testing Suite](https://github.com/chromy/ink-proof) on the compiled binaries in this pull request." >> comment.txt - echo "" >> comment.txt - echo "| System | Results |" >> comment.txt - echo "| --- | --- |" >> comment.txt - FILES="results/*.txt" - for f in $FILES - do - echo "Reading results from $f" - cat "$f" >> comment.txt - done - - # Post Comment - - uses: marocchino/sticky-pull-request-comment@v2.9.0 - with: - recreate: true - path: comment.txt - pages: permissions: contents: write diff --git a/.github/workflows/pr-report.yml b/.github/workflows/pr-report.yml new file mode 100644 index 00000000..e9d2928b --- /dev/null +++ b/.github/workflows/pr-report.yml @@ -0,0 +1,69 @@ +name: PR Report +on: + workflow_run: + workflows: ["build"] + types: [completed] + +jobs: + reporting: + name: "Pull Request Report" + # Only run for pull requests targeting master (not push/dispatch runs) + if: > + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.head_branch != 'master' + runs-on: ubuntu-latest + permissions: + pull-requests: write + actions: read + + steps: + # Download Ink Proof Results from the triggering workflow run + - uses: actions/download-artifact@v4 + with: + pattern: result-* + path: "results" + merge-multiple: true + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + # Retrieve the PR number from the workflow run + - name: Get PR number + id: get-pr + uses: actions/github-script@v7 + with: + script: | + const prs = await github.rest.pulls.list({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + head: `${context.payload.workflow_run.head_repository.owner.login}:${context.payload.workflow_run.head_branch}` + }); + if (prs.data.length === 0) { + core.setFailed('No matching open PR found'); + return; + } + core.setOutput('pr_number', prs.data[0].number); + + # Create comment text + - name: Create Comment Text File + shell: bash + run: | + echo "### Ink Proof Results" >> comment.txt + echo "" >> comment.txt + echo "These results are obtained by running the [Ink-Proof Testing Suite](https://github.com/chromy/ink-proof) on the compiled binaries in this pull request." >> comment.txt + echo "" >> comment.txt + echo "| System | Results |" >> comment.txt + echo "| --- | --- |" >> comment.txt + FILES="results/*.txt" + for f in $FILES + do + echo "Reading results from $f" + cat "$f" >> comment.txt + done + + # Post Comment + - uses: marocchino/sticky-pull-request-comment@v2.9.0 + with: + recreate: true + number: ${{ steps.get-pr.outputs.pr_number }} + path: comment.txt diff --git a/inkcpp/container_operations.cpp b/inkcpp/container_operations.cpp index a724baa2..6b7aa6d5 100644 --- a/inkcpp/container_operations.cpp +++ b/inkcpp/container_operations.cpp @@ -21,8 +21,7 @@ void operation::operator()( ) { container_t id; - bool success - = _story.get_container_id(_story.instructions() + vals[0].get(), id); + bool success = _story.find_container_id(vals[0].get(), id); inkAssert(success, "failed to find container to read visit count!"); stack.push(value{}.set(static_cast(_visit_counts.visits(id)))); } @@ -32,14 +31,14 @@ void operation::operator()( ) { container_t id; - bool success - = _story.get_container_id(_story.instructions() + vals[0].get(), id); + bool success = _story.find_container_id(vals[0].get(), id); inkAssert(success, "failed to find container to read turn count!"); stack.push(value{}.set(static_cast(_visit_counts.turns(id)))); } -void operation< - Command::CHOICE_COUNT, value_type::none, void>::operator()(basic_eval_stack& stack, value*) +void operation::operator()( + basic_eval_stack& stack, value* vals +) { stack.push(value{}.set(static_cast(_runner.num_choices()))); } diff --git a/inkcpp/globals_impl.cpp b/inkcpp/globals_impl.cpp index 57cadc5a..45efa0aa 100644 --- a/inkcpp/globals_impl.cpp +++ b/inkcpp/globals_impl.cpp @@ -20,7 +20,7 @@ globals_impl::globals_impl(const story_impl* story) , _visit_counts(visit_count(), visit_count_null_value) , _owner(story) , _runners_start(nullptr) - , _lists(story->list_meta(), story->get_header()) + , _lists(story->list_meta()) , _globals_initialized(false) { _visit_counts.resize(_num_containers); @@ -50,12 +50,9 @@ void globals_impl::init_static_list_flags() } } -void globals_impl::visit(uint32_t container_id, bool entering_at_start) +void globals_impl::visit(uint32_t container_id) { - if ((! (_owner->container_flag(container_id) & CommandFlag::CONTAINER_MARKER_ONLY_FIRST)) - || entering_at_start) { - _visit_counts.set(container_id, {_visit_counts[container_id].visits + 1, 0}); - } + _visit_counts.set(container_id, {_visit_counts[container_id].visits + 1, 0}); } uint32_t globals_impl::visits(uint32_t container_id) const @@ -276,7 +273,7 @@ size_t globals_impl::snap(unsigned char* data, const snapper& snapper) const ptr = snap_write(ptr, _turn_cnt, data != nullptr); ptr += _visit_counts.snap(data ? ptr : nullptr, snapper); for (unsigned i = 0; i < _visit_counts.capacity(); ++i) { - ptr = snap_write(ptr, _owner->container_hash(i), data != nullptr); + ptr = snap_write(ptr, _owner->container_data(i)._hash, data != nullptr); } ptr += _strings.snap(data ? ptr : nullptr, snapper); ptr += _lists.snap(data ? ptr : nullptr, snapper); @@ -307,7 +304,11 @@ const unsigned char* globals_impl::snap_load(const unsigned char* ptr, const loa hash_t path; ptr = snap_read(ptr, path); container_t c_id; - bool found = _owner->get_container_id(_owner->find_offset_for(path), c_id); + ip_t container_ip = _owner->find_offset_for(path); + bool found = container_ip != nullptr + && _owner->find_container_id( + static_cast(container_ip - _owner->instructions()), c_id + ); if (! loader.migratable) { inkAssert(found, "Invalid container id reference."); inkAssert(c_id == i, "tracked containere are not allowed to move, expect we migrate"); @@ -332,8 +333,8 @@ const unsigned char* globals_impl::snap_load(const unsigned char* ptr, const loa bool globals_impl::migrate_new_globals(globals_impl& new_globals, const char* list_metadata) { - bool success = _variables.migrate(new_globals._variables) - && ((! _lists) || _lists.migrate(list_metadata, _owner->get_header())); + bool success + = _variables.migrate(new_globals._variables) && ((! _lists) || _lists.migrate(list_metadata)); if (! success) { return false; } diff --git a/inkcpp/globals_impl.h b/inkcpp/globals_impl.h index 6b87f76d..d4dc1fe6 100644 --- a/inkcpp/globals_impl.h +++ b/inkcpp/globals_impl.h @@ -65,8 +65,7 @@ class globals_impl final public: // Records a visit to a container - /// @param start_cmd iff the visit was initiatet through a MARKER_START_CONTAINER - void visit(uint32_t container_id, bool entering_at_start); + void visit(uint32_t container_id); // Checks the number of visits to a container uint32_t visits(uint32_t container_id) const; diff --git a/inkcpp/header.cpp b/inkcpp/header.cpp index 96d1e836..e92ee2a1 100644 --- a/inkcpp/header.cpp +++ b/inkcpp/header.cpp @@ -10,32 +10,24 @@ namespace ink::internal { -header header::parse_header(const char* data) +bool header::verify() const { - header res; - const char* ptr = data; - res.endien = *reinterpret_cast(ptr); - ptr += sizeof(header::endian_types); - - using v_t = decltype(header::ink_version_number); - using vcpp_t = decltype(header::ink_bin_version_number); - - if (res.endien == header::endian_types::same) { - res.ink_version_number = *reinterpret_cast(ptr); - ptr += sizeof(v_t); - res.ink_bin_version_number = *reinterpret_cast(ptr); + if (endian() == endian_types::none) { + inkFail("Header magic number was wrong!"); + return false; + } - } else if (res.endien == header::endian_types::differ) { - res.ink_version_number = swap_bytes(*reinterpret_cast(ptr)); - ptr += sizeof(v_t); - res.ink_bin_version_number = swap_bytes(*reinterpret_cast(ptr)); - } else { - inkFail("Failed to parse endian encoding! %#04x", res.endien); + if (endian() == endian_types::differ) { + inkFail("Can't load content with different endian-ness!"); + return false; } - if (res.ink_bin_version_number != InkBinVersion) { + if (ink_bin_version_number != InkBinVersion) { inkFail("InkCpp-version mismatch: file was compiled with different InkCpp-version!"); + return false; } - return res; + + return true; } + } // namespace ink::internal diff --git a/inkcpp/list_table.cpp b/inkcpp/list_table.cpp index b329c1f5..e32dea84 100644 --- a/inkcpp/list_table.cpp +++ b/inkcpp/list_table.cpp @@ -34,7 +34,7 @@ void list_table::copy_lists(const data_t* src, data_t* dst) } } -list_table::list_table(const char* data, const ink::internal::header& header) +list_table::list_table(const char* data) : _valid{false} { if (data == nullptr) { @@ -43,7 +43,7 @@ list_table::list_table(const char* data, const ink::internal::header& header) list_flag flag; const char* ptr = data; int start = 0; - while ((flag = header.read_list_flag(ptr)) != null_flag) { + while ((flag = read_list_flag(ptr)) != null_flag) { // start of new list if (static_cast(_list_end.size()) == flag.list_id) { start = _list_end.size() == 0 ? 0 : _list_end.back(); @@ -1013,9 +1013,9 @@ float* cost_matrix(const MatchListValues& lh, const MatchListValues& rh, float d return matrix; } -bool list_table::migrate(const char* old_list_metadata, const ink::internal::header& header) +bool list_table::migrate(const char* old_list_metadata) { - list_table old_ref_table(old_list_metadata, header); + list_table old_ref_table(old_list_metadata); for (const auto& x : _data) { old_ref_table._data.push() = x; } diff --git a/inkcpp/list_table.h b/inkcpp/list_table.h index 7edbbb38..a30a82ab 100644 --- a/inkcpp/list_table.h +++ b/inkcpp/list_table.h @@ -113,9 +113,9 @@ class list_table : public snapshot_interface list create_permament(); list& add_inplace(list& lh, list_flag rh); - list_table(const char* data, const ink::internal::header&); + list_table(const char* data); // binary list metadata of currently loaded list - bool migrate(const char* old_list_metadata, const ink::internal::header& header); + bool migrate(const char* old_list_metadata); explicit list_table() : _entrySize{0} diff --git a/inkcpp/runner_impl.cpp b/inkcpp/runner_impl.cpp index b39f91f5..9a33fc4c 100644 --- a/inkcpp/runner_impl.cpp +++ b/inkcpp/runner_impl.cpp @@ -46,7 +46,7 @@ namespace ink::runtime::internal hash_t runner_impl::get_current_knot() const { - return _current_knot_id == ~0U ? 0 : _story->container_hash(_current_knot_id); + return _current_knot_id == ~0U ? 0 : _story->container_data(_current_knot_id)._hash; } template<> @@ -176,9 +176,6 @@ inline T runner_impl::read(optional pos) // Read memory T val = *( const T* ) ptr; - if (_story->get_header().endien == header::endian_types::differ) { - val = header::swap_bytes(val); - } // Advance ip if (! pos.has_value()) { @@ -326,96 +323,88 @@ void runner_impl::jump(ip_t dest, bool record_visits, bool track_knot_visit) // _should be_ able to safely assume that there is nothing to do here. A falling // divert should only be taking us from a container to that same container's end point // without entering any other containers - // OR IF is target is same position do nothing - // could happened if jumping to and of an unnamed container + // OR IF if target is same position do nothing + // could happend if jumping to and of an unnamed container if (dest == _ptr) { - _ptr = dest; return; } - const uint32_t* iter = nullptr; - container_t id; - ip_t offset = nullptr; - bool reversed = _ptr > dest; - // number of container which were already on the stack at current position - size_t comm_end = _container.size(); - - iter = nullptr; - while (_story->iterate_containers(iter, id, offset)) { - if (offset >= _ptr) { - break; - } - } - if (! reversed) { - _story->iterate_containers(iter, id, offset, true); + // Discard old stack, preserving save region. + while (! _container.empty()) { + _container.pop(); } - optional last_pop = nullopt; - while (_story->iterate_containers(iter, id, offset, reversed)) { - if ((! reversed && offset >= dest) || (reversed && offset < dest)) { - break; + + // Record location and jump. + const uint32_t current_offset + = _ptr != nullptr ? static_cast(_ptr - _story->instructions()) : ~0U; + _ptr = dest; + + // Find the container at or before dest, which will become the top of the post-jump stack. + const uint32_t dest_offset = dest - _story->instructions(); + const container_t dest_id = _story->find_container_for(dest_offset); + + // If there's no destination container, stop. + if (dest_id == ~0) + return; + + // Are we entering the new container at its start? + using container_data_t = ink::internal::container_data_t; + const container_data_t& dest_container = _story->container_data(dest_id); + if (dest_offset == dest_container._start_offset) { + // Record direct jump to non-knot if requested. (Knots handled below.) + if (record_visits && ! dest_container.knot()) { + _globals->visit(dest_id); } - if (_container.empty() || _container.top().id != id) { - const uint32_t* iter2 = nullptr; - container_t id2; - ip_t offset2; - while (_story->iterate_containers(iter2, id2, offset2) && id2 != id) {} - _container.push({id, offset2 - _story->instructions()}); - } else { - if (_container.size() == comm_end) { - last_pop = _container.pop(); - comm_end -= 1; - } else { - _container.pop(); + + // Consume instruction so we don't process it again during normal flow. (We need to do this here + // to know if it should be tracked or not.) + _ptr += 6; + } + + // If we're tracking knots, we only want the first one. + bool first_knot = track_knot_visit; + + // Assemble temp stack in reverse order by traversing container tree. + container_t stack[abs(config::limitContainerDepth)]; + uint32_t depth = 0; + for (container_t id = dest_id; id != ~0U; /* advance in body */) { + // Append to stack. + inkAssert(depth < abs(config::limitContainerDepth), "Container depth limit exceeded in jump!"); + stack[depth++] = id; + + // Find container for this ID. + const container_data_t& container = _story->container_data(id); + + // Is this a new knot? + if (container.knot() && ! container.contains(current_offset)) { + // Named knots/stitches need special handling - their visit counts are updated wherever the + // story enters them, + // and we generally need to know which knot we're in, for tagging, unless we're jumping to a + // tunnel or similar + // which suppresses knot tracking. + // + // Ink has a rule about incrementing visit counts when you jump to the top of a knot, which + // seems to need to override inkcpp's knot_visit flag. + if (track_knot_visit || container._start_offset == dest_offset) { + _globals->visit(id); } - } - } - iter = nullptr; - while (_story->iterate_containers(iter, id, offset)) { - if (offset >= dest) { - break; - } - } - // if we jump directly to a named container start, go inside, if it's a ONLY_FIRST container - // it will get visited in the next step - // todo: check if a while is needed - if (offset == dest && static_cast(offset[0]) == Command::START_CONTAINER_MARKER) { - if (track_knot_visit - && static_cast(offset[1]) & CommandFlag::CONTAINER_MARKER_IS_KNOT) { - _current_knot_id = id; - _entered_knot = true; - } - dest += 6; - _container.push({id, offset - _story->instructions()}); - // if we entered a knot we just left, do not recount enter - if (reversed && comm_end == _container.size() - 1 && last_pop.has_value() - && last_pop.value().id == id) { - comm_end += 1; + // If tracking, update with the first knot we encounter, which is the one closest to the top + // of the new stack. + if (first_knot) { + _current_knot_id = id; + _entered_knot = true; + first_knot = false; + } } + + // Next one. + id = container._parent; } - _ptr = dest; - // iff all container (until now) are entered at first position - bool allEnteredAtStart = true; - ip_t child_position = dest; - if (record_visits) { - const ContainerData* iData = nullptr; - size_t level = _container.size(); - while (_container.iter(iData)) { - if (level > comm_end - || _story->container_flag(iData->offset + _story->instructions()) - & CommandFlag::CONTAINER_MARKER_ONLY_FIRST) { - auto parrent_offset = _story->instructions() + iData->offset; - inkAssert(child_position >= parrent_offset, "Container stack order is broken"); - // 6 == len of START_CONTAINER_SIGNAL, if its 6 bytes behind the container it is a - // unnnamed subcontainers first child check if child_positino is the first child of - // current container - allEnteredAtStart = allEnteredAtStart && ((child_position - parrent_offset) <= 6); - child_position = parrent_offset; - _globals->visit(iData->id, allEnteredAtStart); - } - level -= 1; - } + // Reverse order onto final stack. + for (uint32_t d = 0; d < depth; ++d) { + _container.push(stack[depth - 1 - d]); } } @@ -495,7 +484,7 @@ runner_impl::runner_impl(const story_impl* data, globals global) , _evaluation_mode{false} , _choices() , _tags_begin(0, ~0) - , _container(ContainerData{}) + , _container(~0) #ifdef INK_ENABLE_CSTD , _rng(static_cast(time(NULL))) #else @@ -668,7 +657,11 @@ bool runner_impl::can_be_migrated() const if (_entered_knot) { return false; } - hash_t c_hash = _story->container_hash(_ptr - 6); + container_t container_id + = _ptr != nullptr && _ptr >= _story->instructions() + 6 + ? _story->find_container_for(static_cast(_ptr - _story->instructions() - 6)) + : ~0U; + hash_t c_hash = (container_id != ~0U) ? _story->container_data(container_id)._hash : 0; if (c_hash == 0) { return false; } @@ -682,19 +675,27 @@ size_t runner_impl::snap(unsigned char* data, snapper& snapper) const unsigned char* ptr = data; bool should_write = data != nullptr; std::uintptr_t offset = _ptr != nullptr ? _ptr - _story->instructions() : 0; - // TODO: remove - ptr = snap_write(ptr, _story->container_hash(_ptr - 6), should_write); - ptr = snap_write(ptr, offset, should_write); - offset = _backup - _story->instructions(); - ptr = snap_write(ptr, offset, should_write); - offset = _done - _story->instructions(); - ptr = snap_write(ptr, offset, should_write); - ptr = snap_write(ptr, _rng.get_state(), should_write); - ptr = snap_write(ptr, _evaluation_mode, should_write); - ptr = snap_write(ptr, _string_mode, should_write); - ptr = snap_write(ptr, _saved_evaluation_mode, should_write); - ptr = snap_write(ptr, _saved, should_write); - ptr = snap_write(ptr, _is_falling, should_write); + // This first field stores the hash of the container at the current position, + // used by migration (story_impl::new_runner_from_snapshot) to navigate to the correct location. + { + container_t container_id + = (_ptr != nullptr && _ptr >= _story->instructions() + 6) + ? _story->find_container_for(static_cast(_ptr - _story->instructions() - 6)) + : ~0U; + hash_t container_hash = (container_id != ~0U) ? _story->container_data(container_id)._hash : 0; + ptr = snap_write(ptr, container_hash, should_write); + } + ptr = snap_write(ptr, offset, should_write); + offset = _backup != nullptr ? _backup - _story->instructions() : 0; + ptr = snap_write(ptr, offset, should_write); + offset = _done != nullptr ? _done - _story->instructions() : 0; + ptr = snap_write(ptr, offset, should_write); + ptr = snap_write(ptr, _rng.get_state(), should_write); + ptr = snap_write(ptr, _evaluation_mode, should_write); + ptr = snap_write(ptr, _string_mode, should_write); + ptr = snap_write(ptr, _saved_evaluation_mode, should_write); + ptr = snap_write(ptr, _saved, should_write); + ptr = snap_write(ptr, _is_falling, should_write); ptr += _output.snap(data ? ptr : nullptr, snapper); ptr += _stack.snap(data ? ptr : nullptr, snapper); ptr += _ref_stack.snap(data ? ptr : nullptr, snapper); @@ -706,7 +707,7 @@ size_t runner_impl::snap(unsigned char* data, snapper& snapper) const ptr = snap_write(ptr, _entered_knot, should_write); ptr = snap_write(ptr, get_current_knot(), should_write); if (_current_knot_id_backup != ~0U) { - ptr = snap_write(ptr, _story->container_hash(_current_knot_id_backup), should_write); + ptr = snap_write(ptr, _story->container_data(_current_knot_id_backup)._hash, should_write); } else { hash_t none = 0; ptr = snap_write(ptr, none, should_write); @@ -731,9 +732,9 @@ const unsigned char* runner_impl::snap_load(const unsigned char* data, loader& l ptr = snap_read(ptr, offset); _ptr = offset == 0 ? nullptr : _story->instructions() + offset; ptr = snap_read(ptr, offset); - _backup = _story->instructions() + offset; + _backup = offset == 0 ? nullptr : _story->instructions() + offset; ptr = snap_read(ptr, offset); - _done = _story->instructions() + offset; + _done = offset == 0 ? nullptr : _story->instructions() + offset; int32_t seed; ptr = snap_read(ptr, seed); _rng.srand(seed); @@ -754,16 +755,22 @@ const unsigned char* runner_impl::snap_load(const unsigned char* data, loader& l _current_knot_id = ~0U; ptr = snap_read(ptr, current_knot_name); if (current_knot_name) { - bool found - = _story->get_container_id(_story->find_offset_for(current_knot_name), _current_knot_id); + ip_t knot_ip = _story->find_offset_for(current_knot_name); + bool found = knot_ip != nullptr + && _story->find_container_id( + static_cast(knot_ip - _story->instructions()), _current_knot_id + ); inkAssert(found, "Unable to find current knot in migrated story."); } _current_knot_id_backup = ~0U; ptr = snap_read(ptr, current_knot_name); if (current_knot_name) { - bool found = _story->get_container_id( - _story->find_offset_for(current_knot_name), _current_knot_id_backup - ); + ip_t knot_ip_backup = _story->find_offset_for(current_knot_name); + bool found + = knot_ip_backup != nullptr + && _story->find_container_id( + static_cast(knot_ip_backup - _story->instructions()), _current_knot_id_backup + ); inkAssert(found, "Unable to find current knot backup in migration %u", current_knot_name); } ptr = _container.snap_load(ptr, loader); @@ -817,7 +824,7 @@ bool runner_impl::migrate_to(hash_t path) fetch_tags(_story->instructions()); assign_tags({tags_level::GLOBAL}); if (_current_knot_id != ~0U) { - ip_t start_of_knot = _story->find_offset_for(_story->container_hash(_current_knot_id)); + ip_t start_of_knot = _story->find_offset_for(_story->container_data(_current_knot_id)._hash); fetch_tags(start_of_knot); assign_tags({tags_level::KNOT}); if (start_of_knot != destination) { @@ -1148,7 +1155,12 @@ void runner_impl::step() // Record the position of the instruction pointer at the first fallthrough. // We'll use this if we run out of content and hit an implied "done" to restore // our position when a choice is chosen. See ::choose - set_done_ptr(_ptr); + // Only update if not already set: after choices are presented, subsequent + // fallthrough diverts at the root level (between named knots) must not + // overwrite the valid done pointer we already recorded. + if (_done == nullptr) { + set_done_ptr(_ptr); + } _is_falling = true; } @@ -1467,7 +1479,7 @@ void runner_impl::step() if (flag & CommandFlag::CHOICE_IS_ONCE_ONLY) { // Need to convert offset to container index container_t destination = ~0U; - if (_story->get_container_id(_story->instructions() + path, destination)) { + if (_story->find_container_id(path, destination)) { // Ignore the choice if we've visited the destination before if (_globals->visits(destination) > 0) { break; @@ -1530,12 +1542,13 @@ void runner_impl::step() // Keep track of current container auto index = read(); // offset points to command, command has size 6 - _container.push({index, _ptr - _story->instructions() - 6}); + _container.push(index); // Increment visit count - if (flag & CommandFlag::CONTAINER_MARKER_TRACK_VISITS - || flag & CommandFlag::CONTAINER_MARKER_TRACK_TURNS) { - _globals->visit(_container.top().id, true); + if (uint8_t(flag) + & (uint8_t(CommandFlag::CONTAINER_MARKER_TRACK_VISITS) + | uint8_t(CommandFlag::CONTAINER_MARKER_TRACK_TURNS))) { + _globals->visit(index); } if (flag & CommandFlag::CONTAINER_MARKER_IS_KNOT) { _current_knot_id = index; @@ -1545,8 +1558,7 @@ void runner_impl::step() } break; case Command::END_CONTAINER_MARKER: { container_t index = read(); - - inkAssert(_container.top().id == index, "Leaving container we are not in!"); + inkAssert(_container.top() == index, "Leaving container we are not in!"); // Move up out of the current container _container.pop(); @@ -1582,7 +1594,7 @@ void runner_impl::step() // Push the visit count for the current container to the top // is 0-indexed for some reason. idk why but this is what ink expects _eval.push(value{}.set( - static_cast(_globals->visits(_container.top().id) - 1) + static_cast(_globals->visits(_container.top()) - 1) )); } break; case Command::TURN: { diff --git a/inkcpp/runner_impl.h b/inkcpp/runner_impl.h index c35e1d57..74ca4e9c 100644 --- a/inkcpp/runner_impl.h +++ b/inkcpp/runner_impl.h @@ -350,17 +350,8 @@ class runner_impl // TODO: Move to story? Both? functions _functions; - // Container set - struct ContainerData { - container_t id = ~0u; - ptrdiff_t offset = 0; - - bool operator==(const ContainerData& oth) const { return oth.id == id && oth.offset == offset; } - - bool operator!=(const ContainerData& oth) const { return ! (*this == oth); } - }; - - internal::managed_restorable_stack < ContainerData, + // Container stack + internal::managed_restorable_stack < container_t, config::limitContainerDepth<0, abs(config::limitContainerDepth)> _container; bool _is_falling = false; diff --git a/inkcpp/story_impl.cpp b/inkcpp/story_impl.cpp index 8d284deb..32f8b3f2 100644 --- a/inkcpp/story_impl.cpp +++ b/inkcpp/story_impl.cpp @@ -99,53 +99,65 @@ story_impl::~story_impl() const char* story_impl::string(uint32_t index) const { return _string_table + index; } -bool story_impl::iterate_containers( - const uint32_t*& iterator, container_t& index, ip_t& offset, bool reverse -) const +bool story_impl::find_container_id(uint32_t offset, container_t& container_id) const { - if (iterator == nullptr) { - // Empty check - if (_container_list_size == 0) { - return false; - } + // Find inmost container. + container_id = find_container_for(offset); - // Start - iterator = reverse ? _container_list + (_container_list_size - 1) * 2 : _container_list; - } else { - // Range check - inkAssert( - iterator >= _container_list && iterator <= _container_list + _container_list_size * 2, - "Container fail" - ); - - // Advance - iterator += reverse ? -2 : 2; - - // End? - if (iterator >= _container_list + _container_list_size * 2 || iterator < _container_list) { - iterator = nullptr; - index = 0; - offset = nullptr; - return false; - } + // Exact match? + if (container_id == ~0U) + return false; + return container_data(container_id)._start_offset == offset; +} + +// Search sorted looking for the target or the largest value smaller than target. +// Returns nullptr if key is smaller than all entries. +template +static const entry* upper_bound(const entry* sorted, uint32_t count, uint32_t key) +{ + if (count == 0 || sorted[0].key() > key) + return nullptr; + + uint32_t begin = 0; + uint32_t end = count - 1; + + while (begin < end) { + const uint32_t mid = begin + (end - begin + 1) / 2; + const uint32_t mid_key = sorted[mid].key(); + + if (mid_key > key) + // Look below + end = mid - 1; + else + // Look above + begin = mid; } - // Get metadata - index = *(iterator + 1); - offset = *iterator + instructions(); - return true; + return sorted + begin; } -bool story_impl::get_container_id(ip_t offset, container_t& container_id) const +container_t story_impl::find_container_for(uint32_t offset) const { - const uint32_t* iter = nullptr; - ip_t iter_offset = nullptr; - while (iterate_containers(iter, container_id, iter_offset)) { - if (iter_offset == offset) - return true; + // Container map contains offsets in even slots, container ids in odd. + const container_map_t* entry = upper_bound(_container_map, _container_map_size, offset); + + // The last container command before the offset could be either the start of a container + // (in which case the offset is contained within) or the end of a container, in which case + // the offset is inside that container's parent. + + // If we're not inside the container, walk out to find the actual parent. Normally we'd + // know that the parent contained the child, but the containers are sparse so we might + // not have anything. + container_t id = entry ? entry->_id : ~0; + while (id != ~0) { + const container_data_t& data = container_data(id); + if (data._start_offset <= offset && data._end_offset >= offset) + return id; + + id = data._parent; } - return false; + return id; } CommandFlag story_impl::container_flag(ip_t offset) const @@ -158,67 +170,12 @@ CommandFlag story_impl::container_flag(ip_t offset) const return static_cast(offset[1]); } -CommandFlag story_impl::container_flag(container_t id) const -{ - const uint32_t* iter = nullptr; - ip_t offset; - container_t c_id; - while (iterate_containers(iter, c_id, offset)) { - if (c_id == id) { - inkAssert( - static_cast(offset[0]) == Command::START_CONTAINER_MARKER, - "Container list pointer is invalid!" - ); - return static_cast(offset[1]); - } - } - inkFail("Container not found -> can't fetch flag"); - return CommandFlag::NO_FLAGS; -} - -hash_t story_impl::container_hash(container_t id) const -{ - const uint32_t* iter = nullptr; - ip_t offset; - container_t c_id; - bool hit = false; - while (iterate_containers(iter, c_id, offset)) { - if (c_id == id) { - hit = true; - break; - } - } - inkAssert(hit, "Unable to find container for id!"); - hash_t hash = container_hash(offset); - inkAssert(hash, "Did not find hash entry for container! (1)"); - return hash; -} - -hash_t story_impl::container_hash(ip_t offset) const -{ - hash_t* h_iter = _container_hash_start; - while (h_iter != _container_hash_end) { - if (instructions() + *( offset_t* ) (h_iter + 1) == offset) { - return *h_iter; - } - h_iter += 2; - } - return 0; -} - ip_t story_impl::find_offset_for(hash_t path) const { - hash_t* iter = _container_hash_start; + // Hash map contains hashes in even slots, offsets in odd. + const container_hash_t* entry = upper_bound(_container_hash, _container_hash_size, path); - while (iter != _container_hash_end) { - if (*iter == path) { - return instructions() + *( offset_t* ) (iter + 1); - } - - iter += 2; - } - - return nullptr; + return entry && entry->_hash == path ? _instruction_data + entry->_offset : nullptr; } globals story_impl::new_globals() @@ -290,110 +247,56 @@ runner story_impl::new_runner_from_snapshot(const snapshot& data, globals store, void story_impl::setup_pointers() { - using header = ink::internal::header; - _header = header::parse_header(reinterpret_cast(_file)); - - // String table is after the header - _string_table = ( char* ) _file + header::Size; - - // Pass over strings - const char* ptr = _string_table; - if (*ptr == 0) // SPECIAL: No strings - { - ptr++; - } else - while (true) { - // Read until null terminator - while (*ptr != 0) - ptr++; - - // Check next character - ptr++; - - // Second null. Strings are done. - if (*ptr == 0) { - ptr++; - break; - } - } - - // check if lists are defined - _list_meta = ptr; - if (list_flag flag = _header.read_list_flag(ptr); flag != null_flag) { - // skip list definitions - auto list_id = flag.list_id; - while (*ptr != 0) { - ++ptr; - } - ++ptr; // skip list name - do { - if (flag.list_id != list_id) { - list_id = flag.list_id; - while (*ptr != 0) { - ++ptr; - } - ++ptr; // skip list name - } - while (*ptr != 0) { - ++ptr; - } - ++ptr; // skip flag name - } while ((flag = _header.read_list_flag(ptr)) != null_flag); - - _lists = reinterpret_cast(ptr); - // skip predefined lists - while (_header.read_list_flag(ptr) != null_flag) { - while (_header.read_list_flag(ptr) != null_flag) - ; - } - _list_meta_size = static_cast(ptr - _list_meta); - } else { - _list_meta = nullptr; - _list_meta_size = 0; - _lists = nullptr; + const ink::internal::header& header = *reinterpret_cast(_file); + if (! header.verify()) { + return; } - inkAssert( - _header.ink_bin_version_number == ink::InkBinVersion, - "invalid InkBinVerison! currently: %i you used %i", ink::InkBinVersion, - _header.ink_bin_version_number - ); - inkAssert( - _header.endien == header::endian_types::same, "different endien support not yet implemented" - ); + // Locate sections + if (header._strings._bytes) + _string_table = reinterpret_cast(_file + header._strings._start); - _num_containers = *( uint32_t* ) (ptr); - ptr += sizeof(uint32_t); - - // Pass over the container data - _container_list_size = 0; - _container_list = ( uint32_t* ) (ptr); - while (true) { - uint32_t val = *( uint32_t* ) ptr; - if (val == ~0U) { - ptr += sizeof(uint32_t); - break; - } else { - ptr += sizeof(uint32_t) * 2; - _container_list_size++; - } + // Address list sections if they exist + if (header._list_meta._bytes) { + _list_meta = reinterpret_cast(_file + header._list_meta._start); + _list_meta_size = header._list_meta._bytes; + + // Lists require metadata + if (header._lists._bytes) + _lists = reinterpret_cast(_file + header._lists._start); } - // Next is the container hash map - _container_hash_start = ( hash_t* ) (ptr); - while (true) { - uint32_t val = *( uint32_t* ) ptr; - if (val == ~0U) { - _container_hash_end = ( hash_t* ) (ptr); - ptr += sizeof(uint32_t); - break; - } + // Address containers section if it exists + if (header._containers._bytes) { + _num_containers = header._containers._bytes / sizeof(container_data_t); + _container_data = reinterpret_cast(_file + header._containers._start); + } - ptr += sizeof(uint32_t) * 2; + // Address container map if it exists + if (header._container_map._bytes) { + _container_map_size = header._container_map._bytes / sizeof(container_map_t); + _container_map = reinterpret_cast(_file + header._container_map._start); } - // After strings comes instruction data - _instruction_data = ( ip_t ) ptr; + // Address container hash if it exists + if (header._container_hash._bytes) { + _container_hash_size = header._container_hash._bytes / sizeof(container_hash_t); + _container_hash + = reinterpret_cast(_file + header._container_hash._start); + } + + // Address instructions, which we hope exist! + if (header._instructions._bytes) + _instruction_data = _file + header._instructions._start; + + // Shrink file length to fit exact length of instructions section. + inkAssert( + end() >= _instruction_data + header._instructions._bytes, + "Story file size mismatch: file ends at %u but instructions end at %u", + ( uint32_t ) (end() - _file), + ( uint32_t ) (_instruction_data - _file) + header._instructions._bytes + ); + _length = _instruction_data + header._instructions._bytes - _file; // Debugging info /*{ diff --git a/inkcpp/story_impl.h b/inkcpp/story_impl.h index ae2a07e5..b4c6acfe 100644 --- a/inkcpp/story_impl.h +++ b/inkcpp/story_impl.h @@ -42,15 +42,36 @@ class story_impl : public story size_t list_meta_size() const { return _list_meta_size; } - bool iterate_containers( - const uint32_t*& iterator, container_t& index, ip_t& offset, bool reverse = false - ) const; - bool get_container_id(ip_t offset, container_t& container_id) const; - /// Get container flag from container offset (either start or end) + // Find the innermost container containing offset. If offset is the start of a container, return + // that container. + container_t find_container_for(uint32_t offset) const; + + // Find the container which starts exactly at offset. Return false if this isn't the start of a + // container. + bool find_container_id(uint32_t offset, container_t& container_id) const; + + using container_data_t = ink::internal::container_data_t; + using container_hash_t = ink::internal::container_hash_t; + using container_map_t = ink::internal::container_map_t; + + // Look up the details of the given container + const container_data_t& container_data(container_t id) const + { + inkAssert( + id < _num_containers, "Container ID %u out of range (num_containers=%u)", ( unsigned ) id, + ( unsigned ) _num_containers + ); + return _container_data[id]; + } + + // Look up the instruction pointer for the start of the given container + ip_t container_offset(container_t id) const + { + return _instruction_data + container_data(id)._start_offset; + } + + // Get container flag from container offset (either start or end) CommandFlag container_flag(ip_t offset) const; - CommandFlag container_flag(container_t id) const; - hash_t container_hash(container_t id) const; - hash_t container_hash(ip_t offset) const; ip_t find_offset_for(hash_t path) const; @@ -61,8 +82,6 @@ class story_impl : public story virtual runner new_runner_from_snapshot(const snapshot&, globals store = nullptr, unsigned idx = 0) override; - const ink::internal::header& get_header() const { return _header; } - hash_t hash() const override { return hash_data(_file, _length); } private: @@ -73,26 +92,27 @@ class story_impl : public story const unsigned char* _file; size_t _length; - ink::internal::header _header; - // string table - const char* _string_table; + const char* _string_table = nullptr; + + const char* _list_meta = nullptr; + size_t _list_meta_size = 0; + const list_flag* _lists = nullptr; - const char* _list_meta; - size_t _list_meta_size; - const list_flag* _lists; + // Information about containers. + const container_data_t* _container_data = nullptr; + uint32_t _num_containers = 0; - // container info - uint32_t* _container_list; - uint32_t _container_list_size; - uint32_t _num_containers; + // How to find containers from instruction offsets. + const container_map_t* _container_map = nullptr; + uint32_t _container_map_size = 0; - // container hashes - hash_t* _container_hash_start; - hash_t* _container_hash_end; + // How to find containers from string hashes. + const container_hash_t* _container_hash = nullptr; + uint32_t _container_hash_size = 0; // instruction info - ip_t _instruction_data; + ip_t _instruction_data = nullptr; // story block used to create various weak pointers ref_block* _block; diff --git a/inkcpp/string_operations.h b/inkcpp/string_operations.h index 2617871a..a392dd13 100644 --- a/inkcpp/string_operations.h +++ b/inkcpp/string_operations.h @@ -10,7 +10,6 @@ namespace ink::runtime::internal { - namespace casting { // define valid castings diff --git a/inkcpp_c/inkcpp.cpp b/inkcpp_c/inkcpp.cpp index 8c425301..b7008e2c 100644 --- a/inkcpp_c/inkcpp.cpp +++ b/inkcpp_c/inkcpp.cpp @@ -72,6 +72,10 @@ extern "C" { HInkStory* ink_story_from_file(const char* filename) { FILE* file = fopen(filename, "rb"); + if (file == NULL) { + fprintf(stderr, "Failed to open file: %s\n", filename); + return NULL; + } fseek(file, 0, SEEK_END); long file_length = ftell(file); if (file_length < 0) { diff --git a/inkcpp_compiler/binary_emitter.cpp b/inkcpp_compiler/binary_emitter.cpp index 3425201c..1f9e3e1c 100644 --- a/inkcpp_compiler/binary_emitter.cpp +++ b/inkcpp_compiler/binary_emitter.cpp @@ -12,6 +12,8 @@ #include #include +#include +#include #ifndef _MSC_VER # include @@ -100,7 +102,7 @@ uint32_t binary_emitter::start_container(int index_in_parent, const std::string& container->parent = _current; // Set offset to the current position - container->offset = _containers.pos(); + container->offset = _instructions.pos(); // Add to parents lists if (_current != nullptr) { @@ -116,17 +118,17 @@ uint32_t binary_emitter::start_container(int index_in_parent, const std::string& _current = container; // Return current position - return _containers.pos(); + return _instructions.pos(); } uint32_t binary_emitter::end_container() { // Move up the chain - _current->end_offset = _containers.pos(); + _current->end_offset = _instructions.pos(); _current = _current->parent; // Return offset - return _containers.pos(); + return _instructions.pos(); } int binary_emitter::function_container_arguments(const std::string& name) @@ -140,11 +142,11 @@ int binary_emitter::function_container_arguments(const std::string& name) } size_t offset = fn->second->offset; - byte_t cmd = _containers.get(offset); + byte_t cmd = _instructions.get(offset); int arity = 0; while (static_cast(cmd) == Command::DEFINE_TEMP) { offset += 6; // command(1) + flag(1) + variable_name_hash(4) - cmd = _containers.get(offset); + cmd = _instructions.get(offset); ++arity; } return arity; @@ -154,14 +156,14 @@ void binary_emitter::write_raw( Command command, CommandFlag flag, const char* payload, ink::size_t payload_size ) { - _containers.write(command); - _containers.write(flag); + _instructions.write(command); + _instructions.write(flag); constexpr size_t MAX_PAYLOAD_SIZE = 4; ink_assert(payload_size <= MAX_PAYLOAD_SIZE, "enforce constant instruction size"); if (payload_size > 0) - _containers.write(( const byte_t* ) payload, payload_size); + _instructions.write(( const byte_t* ) payload, payload_size); constexpr const byte_t empty[MAX_PAYLOAD_SIZE] = {}; - _containers.write(empty, MAX_PAYLOAD_SIZE - payload_size); + _instructions.write(empty, MAX_PAYLOAD_SIZE - payload_size); } void binary_emitter::write_path( @@ -172,7 +174,7 @@ void binary_emitter::write_path( write(command, ( uint32_t ) 0, flag); // Note the position of this later so we can resolve the paths at the end - size_t param_position = _containers.pos() - sizeof(uint32_t); + size_t param_position = _instructions.pos() - sizeof(uint32_t); bool op = flag & CommandFlag::FALLBACK_FUNCTION; _paths.push_back(std::make_tuple(param_position, path, op, _current, useCountIndex)); } @@ -215,43 +217,93 @@ void binary_emitter::write_list( void binary_emitter::handle_nop(int index_in_parent) { - _current->noop_offsets.insert({index_in_parent, _containers.pos()}); + _current->noop_offsets.insert({index_in_parent, _instructions.pos()}); +} + +template +void binary_emitter::emit_section(std::ostream& stream, const std::vector& data) const +{ + stream.write(reinterpret_cast(data.data()), data.size() * sizeof(type)); + close_section(stream); +} + +void binary_emitter::emit_section(std::ostream& stream, const binary_stream& data) const +{ + inkAssert((stream.tellp() & (ink::internal::header::Alignment - 1)) == 0); + data.write_to(stream); + close_section(stream); +} + +void binary_emitter::close_section(std::ostream& stream) const +{ + // Write zeroes until aligned. + while (! stream.fail() && (stream.tellp() % ink::internal::header::Alignment)) + stream.put('\0'); } void binary_emitter::output(std::ostream& out) { - // Write the ink version - // TODO: define this order in header? - using header = ink::internal::header; - header::endian_types same = header::endian_types::same; - out.write(( const char* ) &same, sizeof(decltype(same))); - out.write(( const char* ) &_ink_version, sizeof(decltype(_ink_version))); - out.write(( const char* ) &ink::InkBinVersion, sizeof(decltype(ink::InkBinVersion))); + // Create container data + std::vector container_data; + container_data.resize(_max_container_index); + build_container_data(container_data, ~0, _root); + + // Create container hash (and write the hashes into the data as well) + std::vector container_hash; + container_hash.reserve(_max_container_index); + build_container_hash_map(container_hash, container_data, "", _root); + + // Sort map on ascending hash code. + std::sort(container_hash.begin(), container_hash.end()); + + // If there's list meta data... + if (_list_meta.pos() > 0) { + // If there are any lists, terminate the data correctly. Otherwise leave an empty section. + if (_lists.pos() > 0) + _lists.write(null_flag); + } else + // No meta data -> no lists. + _lists.reset(); + + // Fill in header + ink::internal::header header; + header.ink_version_number = _ink_version; + header.ink_bin_version_number = ink::InkBinVersion; + + // Fill in sections + uint32_t offset = sizeof(header); + header._strings.setup(offset, _strings.pos()); + header._list_meta.setup(offset, _list_meta.pos()); + header._lists.setup(offset, _lists.pos()); + header._containers.setup(offset, container_data.size() * sizeof(container_data_t)); + header._container_map.setup(offset, _container_map.size() * sizeof(container_map_t)); + header._container_hash.setup(offset, container_hash.size() * sizeof(container_hash_t)); + header._instructions.setup(offset, _instructions.pos()); + + // Write the header + out.write(reinterpret_cast(&header), sizeof(header)); + close_section(out); // Write the string table - _strings.write_to(out); + emit_section(out, _strings); - // Write a separator - out << ( char ) 0; + // Write lists meta data and defined lists + emit_section(out, _list_meta); // Write lists meta data and defined lists - _lists.write_to(out); - // Write a seperator - out.write(reinterpret_cast(&null_flag), sizeof(null_flag)); + emit_section(out, _lists); - // Write out container map - write_container_map(out, _container_map, _max_container_index); + // Write out container information + emit_section(out, container_data); - // Write a separator - uint32_t END_MARKER = ~0U; - out.write(( const char* ) &END_MARKER, sizeof(uint32_t)); + // Write out container map + emit_section(out, _container_map); // Write container hash list - write_container_hash_map(out); - out.write(( const char* ) &END_MARKER, sizeof(uint32_t)); + emit_section(out, container_hash); - // Write the container data - _containers.write_to(out); + // Write the container contents (instruction stream) + emit_section(out, _instructions); // Flush the file out.flush(); @@ -262,8 +314,9 @@ void binary_emitter::initialize() // Reset binary data stores _strings.reset(); _list_count = 0; + _list_meta.reset(); _lists.reset(); - _containers.reset(); + _instructions.reset(); // clear other data _paths.clear(); @@ -289,13 +342,13 @@ uint32_t binary_emitter::fallthrough_divert() write(Command::DIVERT, ( uint32_t ) 0, CommandFlag::DIVERT_IS_FALLTHROUGH); // Return the location of the divert offset - return _containers.pos() - sizeof(uint32_t); + return _instructions.pos() - sizeof(uint32_t); } void binary_emitter::patch_fallthroughs(uint32_t position) { // Patch - _containers.set(position, _containers.pos()); + _instructions.set(position, _instructions.pos()); } void binary_emitter::process_paths() @@ -366,62 +419,78 @@ void binary_emitter::process_paths() if (noop_offset != ~0U) { inkAssert(! useCountIndex, "Can't count visits to a noop!"); - _containers.set(position, noop_offset); + _instructions.set(position, noop_offset); } else { // If we want the count index, write that out if (useCountIndex) { inkAssert(container->counter_index != ~0U, "No count index available for this container!"); - _containers.set(position, container->counter_index); + _instructions.set(position, container->counter_index); } else { // Otherwise, write container address if (container == nullptr) { - _containers.set(position, 0); + _instructions.set(position, 0); inkAssert(optional, "Was not able to resolve a not optional path! '%hs'", path.c_str()); } else { - _containers.set(position, container->offset); + _instructions.set(position, container->offset); } } } } } -void binary_emitter::write_container_map( - std::ostream& out, const container_map& map, container_t num -) +void binary_emitter::build_container_data( + std::vector& data, container_t parent, const container_data* context +) const { - // Write out container count - out.write(reinterpret_cast(&num), sizeof(container_t)); - - // Write out entries - for (const auto& pair : map) { - out.write(( const char* ) &pair.first, sizeof(uint32_t)); - out.write(( const char* ) &pair.second, sizeof(uint32_t)); + // Build data for this container + if (context->counter_index != ~0) { + container_data_t& d = data[context->counter_index]; + d._parent = parent; + d._start_offset = context->offset; + d._end_offset = context->end_offset; + const uint8_t flags = _instructions.get(context->offset + 1); + inkAssert(flags < 16); + d._flags = flags; + + // Since we might be skipping tree levels, we need to be explicit about the parent. + parent = context->counter_index; } -} -void binary_emitter::write_container_hash_map(std::ostream& out) -{ - write_container_hash_map(out, "", _root); + // Recurse + for (auto child : context->children) + build_container_data(data, parent, child); } -void binary_emitter::write_container_hash_map( - std::ostream& out, const std::string& name, const container_data* context -) +void binary_emitter::build_container_hash_map( + std::vector& hash_map, std::vector& data, + const std::string& name, const container_data* context +) const { + // Search named children first. for (auto child : context->named_children) { // Get the child's name in the hierarchy std::string child_name = name.empty() ? child.first : (name + "." + child.first); - hash_t name_hash = hash_string(child_name.c_str()); - // Write out name hash and offset - out.write(( const char* ) &name_hash, sizeof(hash_t)); - out.write(( const char* ) &child.second->offset, sizeof(uint32_t)); + + // Hash name. We only do this at the named child level. In theory we could support indexed + // children as well. The root is anonymous so the fact that it's skipped is not an issue. + const hash_t child_name_hash = hash_string(child_name.c_str()); + + // Store hash in the data. + if (child.second->counter_index != ~0) { + data[child.second->counter_index]._hash = child_name_hash; + } + + // Append the name hash and offset + hash_map.push_back({child_name_hash, child.second->offset}); // Recurse - write_container_hash_map(out, child_name, child.second); + build_container_hash_map(hash_map, data, child_name, child.second); } + // Search indexed children (which duplicates named childen...) + // TODO: Merge duplicate child arrays, very error-prone. for (auto child : context->indexed_children) { - write_container_hash_map(out, name, child.second); + build_container_hash_map(hash_map, data, name, child.second); } } @@ -435,21 +504,21 @@ void binary_emitter::set_list_meta(const list_data& list_defs) auto list_names = list_defs.get_list_names().begin(); int list_id = -1; for (const auto& flag : flags) { - _lists.write(flag.flag); + _list_meta.write(flag.flag); if (flag.flag.list_id != list_id) { list_id = flag.flag.list_id; - _lists.write( + _list_meta.write( reinterpret_cast(list_names->data()), static_cast(list_names->size()) ); ++list_names; - _lists.write('\0'); + _list_meta.write('\0'); } - _lists.write( + _list_meta.write( reinterpret_cast(flag.name->c_str()), static_cast(flag.name->size()) + 1 ); } - _lists.write(null_flag); + _list_meta.write(null_flag); } } // namespace ink::compiler::internal diff --git a/inkcpp_compiler/binary_emitter.h b/inkcpp_compiler/binary_emitter.h index a03137af..36dd4fa1 100644 --- a/inkcpp_compiler/binary_emitter.h +++ b/inkcpp_compiler/binary_emitter.h @@ -8,63 +8,86 @@ #include "emitter.h" #include "binary_stream.h" +#include "header.h" namespace ink::compiler::internal { - struct container_data; - class list_data; +struct container_data; +class list_data; - // binary emitter - class binary_emitter : public emitter - { - public: - binary_emitter(); - virtual ~binary_emitter(); +// binary emitter +class binary_emitter : public emitter +{ +public: + binary_emitter(); + virtual ~binary_emitter(); + + // Begin emitter + virtual uint32_t start_container(int index_in_parent, const std::string& name) override; + virtual uint32_t end_container() override; + virtual int function_container_arguments(const std::string& name) override; + virtual void write_raw( + Command command, CommandFlag flag = CommandFlag::NO_FLAGS, const char* payload = nullptr, + ink::size_t payload_size = 0 + ) override; + virtual void write_path( + Command command, CommandFlag flag, const std::string& path, bool useCountIndex = false + ) override; + virtual void write_variable(Command command, CommandFlag flag, const std::string& name) override; + virtual void write_string(Command command, CommandFlag flag, const std::string& string) override; + virtual void handle_nop(int index_in_parent) override; + virtual uint32_t fallthrough_divert() override; + virtual void patch_fallthroughs(uint32_t position) override; + virtual void set_list_meta(const list_data& list_defs) override; + virtual void + write_list(Command command, CommandFlag flag, const std::vector& entries) override; + // End emitter + + // write out the emitters data + virtual void output(std::ostream&); + +protected: + virtual void initialize() override; + virtual void finalize() override; + virtual void setContainerIndex(container_t index) override; + +private: + void process_paths(); - // Begin emitter - virtual uint32_t start_container(int index_in_parent, const std::string& name) override; - virtual uint32_t end_container() override; - virtual int function_container_arguments(const std::string& name) override; - virtual void write_raw(Command command, CommandFlag flag = CommandFlag::NO_FLAGS, const char* payload = nullptr, ink::size_t payload_size = 0) override; - virtual void write_path(Command command, CommandFlag flag, const std::string& path, bool useCountIndex = false) override; - virtual void write_variable(Command command, CommandFlag flag, const std::string& name) override; - virtual void write_string(Command command, CommandFlag flag, const std::string& string) override; - virtual void handle_nop(int index_in_parent) override; - virtual uint32_t fallthrough_divert() override; - virtual void patch_fallthroughs(uint32_t position) override; - virtual void set_list_meta(const list_data& list_defs) override; - virtual void write_list(Command command, CommandFlag flag, const std::vector& entries) override; - // End emitter + template + void emit_section(std::ostream& out, const std::vector& data) const; + void emit_section(std::ostream& out, const binary_stream& stream) const; + void close_section(std::ostream& out) const; - // write out the emitters data - virtual void output(std::ostream&); + using container_data_t = ink::internal::container_data_t; + using container_map_t = ink::internal::container_map_t; + using container_hash_t = ink::internal::container_hash_t; - protected: - virtual void initialize() override; - virtual void finalize() override; - virtual void setContainerIndex(container_t index) override; + void build_container_data( + std::vector& data, container_t parent, const container_data* context + ) const; - private: - void process_paths(); - void write_container_map(std::ostream&, const container_map&, container_t); - void write_container_hash_map(std::ostream&); - void write_container_hash_map(std::ostream&, const std::string&, const container_data*); + void build_container_hash_map( + std::vector& hash, std::vector& data, const std::string&, + const container_data* context + ) const; - private: - container_data* _root; - container_data* _current; - compilation_results* _results; +private: + container_data* _root; + container_data* _current; + compilation_results* _results; - binary_stream _strings; - uint32_t _list_count = 0; - binary_stream _lists; - binary_stream _containers; + binary_stream _strings; + uint32_t _list_count = 0; + binary_stream _list_meta; + binary_stream _lists; + binary_stream _instructions; - // positon to write address - // path as string - // if path may not exists (used for function fallbackes) - // container data - // use count index? - std::vector> _paths; - }; -} + // positon to write address + // path as string + // if path may not exists (used for function fallbackes) + // container data + // use count index? + std::vector> _paths; +}; +} // namespace ink::compiler::internal diff --git a/inkcpp_compiler/emitter.cpp b/inkcpp_compiler/emitter.cpp index 6c7d99a0..39e27fd3 100644 --- a/inkcpp_compiler/emitter.cpp +++ b/inkcpp_compiler/emitter.cpp @@ -8,55 +8,51 @@ namespace ink::compiler::internal { - void emitter::start(int ink_version, compilation_results* results) - { - // store - _ink_version = ink_version; - set_results(results); - - // reset - _container_map.clear(); - _max_container_index = 0; - - // initialize - initialize(); - } +void emitter::start(int ink_version, compilation_results* results) +{ + // store + _ink_version = ink_version; + set_results(results); - void emitter::finish(container_t max_container_index) - { - // store max index - _max_container_index = max_container_index; + // reset + _container_map.clear(); + _max_container_index = 0; - // finalize - finalize(); - } + // initialize + initialize(); +} - void emitter::add_start_to_container_map(uint32_t offset, container_t index) - { - if (_container_map.rbegin() != _container_map.rend()) - { - if (_container_map.rbegin()->first > offset) - { - warn() << "Container map written out of order. Wrote container at offset " - << offset << " after container with offset " << _container_map.rbegin()->first << std::flush; - } - } +void emitter::finish(container_t max_container_index) +{ + // store max index + _max_container_index = max_container_index; - _container_map.push_back(std::make_pair(offset, index)); - setContainerIndex(index); - } + // finalize + finalize(); +} - void emitter::add_end_to_container_map(uint32_t offset, container_t index) - { - if (_container_map.rbegin() != _container_map.rend()) - { - if (_container_map.rbegin()->first > offset) - { - warn() << "Container map written out of order. Wrote container at offset " - << offset << " after container with offset " << _container_map.rbegin()->first << std::flush; - } +void emitter::add_start_to_container_map(uint32_t offset, container_t index) +{ + if (_container_map.rbegin() != _container_map.rend()) { + if (_container_map.rbegin()->_offset > offset) { + warn() << "Container map written out of order. Wrote container at offset " << offset + << " after container with offset " << _container_map.rbegin()->_offset << std::flush; } + } + + _container_map.push_back({offset, index}); + setContainerIndex(index); +} - _container_map.push_back(std::make_pair(offset, index)); +void emitter::add_end_to_container_map(uint32_t offset, container_t index) +{ + if (_container_map.rbegin() != _container_map.rend()) { + if (_container_map.rbegin()->_offset > offset) { + warn() << "Container map written out of order. Wrote container at offset " << offset + << " after container with offset " << _container_map.rbegin()->_offset << std::flush; + } } + + _container_map.push_back({offset, index}); } +} // namespace ink::compiler::internal diff --git a/inkcpp_compiler/emitter.h b/inkcpp_compiler/emitter.h index ae1458d4..fa3eec21 100644 --- a/inkcpp_compiler/emitter.h +++ b/inkcpp_compiler/emitter.h @@ -9,96 +9,104 @@ #include "command.h" #include "system.h" #include "reporter.h" +#include "header.h" #include #include namespace ink::compiler::internal { - class list_data; +class list_data; - // Abstract base class for emitters which write ink commands to a file - class emitter : public reporter - { - public: - virtual ~emitter() { } +// Abstract base class for emitters which write ink commands to a file +class emitter : public reporter +{ +public: + virtual ~emitter() {} - // starts up the emitter (and calls initialize) - void start(int ink_version, compilation_results* results = nullptr); + // starts up the emitter (and calls initialize) + void start(int ink_version, compilation_results* results = nullptr); - // tells the emitter compilation is done (and calls finalize) - void finish(container_t max_container_index); + // tells the emitter compilation is done (and calls finalize) + void finish(container_t max_container_index); - // start a container - virtual uint32_t start_container(int index_in_parent, const std::string& name) = 0; + // start a container + virtual uint32_t start_container(int index_in_parent, const std::string& name) = 0; - // ends a container - virtual uint32_t end_container() = 0; + // ends a container + virtual uint32_t end_container() = 0; - // checks if _root contains a container named name to check - // if name is in valid internal function name - // @return number of arguments functions takes (arity) - // @retval -1 if the function was not found - virtual int function_container_arguments(const std::string& name) = 0; + // checks if _root contains a container named name to check + // if name is in valid internal function name + // @return number of arguments functions takes (arity) + // @retval -1 if the function was not found + virtual int function_container_arguments(const std::string& name) = 0; - // Writes a command with an optional payload - virtual void write_raw(Command command, CommandFlag flag = CommandFlag::NO_FLAGS, const char* payload = nullptr, ink::size_t payload_size = 0) = 0; + // Writes a command with an optional payload + virtual void write_raw( + Command command, CommandFlag flag = CommandFlag::NO_FLAGS, const char* payload = nullptr, + ink::size_t payload_size = 0 + ) = 0; - // Writes a command with a path as the payload - virtual void write_path(Command command, CommandFlag flag, const std::string& path, bool useCountIndex = false) = 0; + // Writes a command with a path as the payload + virtual void write_path( + Command command, CommandFlag flag, const std::string& path, bool useCountIndex = false + ) = 0; - // Writes a command with a variable as the payload - virtual void write_variable(Command command, CommandFlag flag, const std::string& name) = 0; + // Writes a command with a variable as the payload + virtual void write_variable(Command command, CommandFlag flag, const std::string& name) = 0; - // Writes a command with a string payload - virtual void write_string(Command command, CommandFlag flag, const std::string& string) = 0; + // Writes a command with a string payload + virtual void write_string(Command command, CommandFlag flag, const std::string& string) = 0; - // write a command with a list payload - virtual void write_list(Command commmand, CommandFlag flag, const std::vector& list) = 0; + // write a command with a list payload + virtual void write_list(Command commmand, CommandFlag flag, const std::vector& list) + = 0; - // Callback for nop commands - virtual void handle_nop(int index_in_parent) = 0; + // Callback for nop commands + virtual void handle_nop(int index_in_parent) = 0; - // adds a fallthrough divert - virtual uint32_t fallthrough_divert() = 0; + // adds a fallthrough divert + virtual uint32_t fallthrough_divert() = 0; - // Patches a fallthrough divert at the given position to divert to the current position - virtual void patch_fallthroughs(uint32_t position) = 0; + // Patches a fallthrough divert at the given position to divert to the current position + virtual void patch_fallthroughs(uint32_t position) = 0; - // Adds a container start marker to the container map - void add_start_to_container_map(uint32_t offset, container_t index); + // Adds a container start marker to the container map + void add_start_to_container_map(uint32_t offset, container_t index); - // Adds a container end marker to the container map - void add_end_to_container_map(uint32_t offset, container_t index); + // Adds a container end marker to the container map + void add_end_to_container_map(uint32_t offset, container_t index); - // add list definitions - virtual void set_list_meta(const list_data& lists_defs) = 0; + // add list definitions + virtual void set_list_meta(const list_data& lists_defs) = 0; - // Helpers - template - void write(Command command, const T& param, CommandFlag flag = CommandFlag::NO_FLAGS) - { - static_assert(sizeof(T) == 4, "Parameters must be 4 bytes long"); - write_raw(command, flag, (const char*)(¶m), sizeof(T)); - } + // Helpers + template + void write(Command command, const T& param, CommandFlag flag = CommandFlag::NO_FLAGS) + { + static_assert(sizeof(T) == 4, "Parameters must be 4 bytes long"); + write_raw(command, flag, ( const char* ) (¶m), sizeof(T)); + } - protected: - // Initialize (clear state, get ready for a new file) - virtual void initialize() = 0; +protected: + // Initialize (clear state, get ready for a new file) + virtual void initialize() = 0; - // Finalize (do any post processing necessary) - virtual void finalize() = 0; + // Finalize (do any post processing necessary) + virtual void finalize() = 0; - // Set container index for visit tracking - virtual void setContainerIndex(container_t index) = 0; + // Set container index for visit tracking + virtual void setContainerIndex(container_t index) = 0; - protected: - typedef std::vector> container_map; +protected: + using container_map_t = ink::internal::container_map_t; + typedef std::vector container_map; - // container map - container_map _container_map; - container_t _max_container_index; + // container map + container_map _container_map; + container_t _max_container_index; - // ink version - int _ink_version; - }; -} + // ink version + int _ink_version; +}; +} // namespace ink::compiler::internal diff --git a/inkcpp_test/Migration.cpp b/inkcpp_test/Migration.cpp index ab0f42ac..74549599 100644 --- a/inkcpp_test/Migration.cpp +++ b/inkcpp_test/Migration.cpp @@ -11,7 +11,7 @@ using namespace ink::runtime; -SCENARIO("Simple isolated migration tests.") +SCENARIO("Simple isolated migration tests.", "[migration]") { std::unique_ptr base_story{story::from_file(INK_TEST_RESOURCE_DIR "MigrationBase.bin")}; globals base_globals = base_story->new_globals(); @@ -31,6 +31,8 @@ SCENARIO("Simple isolated migration tests.") REQUIRE(base_thread->get_global_tag(1) == std::string("flavor:base")); REQUIRE(base_thread->num_knot_tags() == 1); REQUIRE(base_thread->get_knot_tag(0) == std::string("knot:Main")); + base_thread->choose(0); + REQUIRE(base_thread->getall() == "A\ncatch\n5 3\n1 -1 1\nOh.\n"); } GIVEN("Simple story with changes in globals.") { @@ -48,6 +50,8 @@ SCENARIO("Simple isolated migration tests.") CHECK(new_globals->get("do_migrate").value_or(0) == 10); CHECK(new_globals->get("new_var").value_or(0) == 20); } + new_thread->choose(0); + REQUIRE(new_thread->getall() == "A\ncatch\n1 -1 1\nOh.\n"); } WHEN("Run base story and load in new_story") { @@ -68,7 +72,7 @@ SCENARIO("Simple isolated migration tests.") THEN("expect story to continue normally") { content = new_thread->getall(); - REQUIRE(content == "A\ncatch\n1 -1 0\nOh.\n"); + REQUIRE(content == "A\ncatch\n1 -1 1\nOh.\n"); } } } @@ -85,7 +89,7 @@ SCENARIO("Simple isolated migration tests.") REQUIRE(content == "B\n-1 0 0\nThis is a simple story.\n"); new_thread->choose(0); content = new_thread->getall(); - REQUIRE(content == "A\ncatch\n-1 1 0\nOh.\n"); + REQUIRE(content == "A\ncatch\n-1 1 1\nOh.\n"); } WHEN("Run base story and load new story.") { @@ -100,7 +104,7 @@ SCENARIO("Simple isolated migration tests.") THEN("Migrated visit counts. Unreachable node has visit, new node has no") { content = new_thread->getall(); - REQUIRE(content == "A\ncatch\n1 1 0\nOh.\n"); + REQUIRE(content == "A\ncatch\n1 1 1\nOh.\n"); } } } @@ -119,7 +123,7 @@ SCENARIO("Simple isolated migration tests.") REQUIRE(new_thread->get_current_knot() == 0x25e83b84); content = new_thread->getall(); REQUIRE(new_thread->get_current_knot() == 0x25e83b84); - REQUIRE(content == "A\ncatch\n2 - 3\n1 -1 0\nOh.\n"); + REQUIRE(content == "A\ncatch\n2 - 3\n1 -1 1\nOh.\n"); } WHEN("Run base story and load new story.") { @@ -137,7 +141,7 @@ SCENARIO("Simple isolated migration tests.") { REQUIRE(new_thread->get_current_knot() == 0x25e83b84); content = new_thread->getall(); - REQUIRE(content == "A\ncatch\n5 - 6\n1 -1 0\nOh.\n"); + REQUIRE(content == "A\ncatch\n5 - 6\n1 -1 1\nOh.\n"); } } } @@ -175,18 +179,21 @@ SCENARIO("Simple isolated migration tests.") runner new_thread = new_story->new_runner_from_snapshot(*snap, new_globals); THEN("Got new global/knot tags") { - REQUIRE(content == "A\n0 -1 0\nThis is a simple story.\n"); REQUIRE(new_thread->num_global_tags() == 2); REQUIRE(new_thread->get_global_tag(0) == std::string("test:migration")); REQUIRE(new_thread->get_global_tag(1) == std::string("flavor:changed")); REQUIRE(new_thread->num_knot_tags() == 1); REQUIRE(new_thread->get_knot_tag(0) == std::string("knot:different")); } + THEN("continue the story normally") + { + REQUIRE(new_thread->getall() == "A\ncatch\n5 3\n1 -1 1\nOh.\n"); + } } } } -SCENARIO("Migration Test for small story") +SCENARIO("Migration Test for small story", "[migration]") { std::unique_ptr before{story::from_file(INK_TEST_RESOURCE_DIR "MigrationBefore.bin")}; std::unique_ptr after{story::from_file(INK_TEST_RESOURCE_DIR "MigrationAfter.bin")}; diff --git a/inkcpp_test/UTF8.cpp b/inkcpp_test/UTF8.cpp index 38ccff9d..4b2c7beb 100644 --- a/inkcpp_test/UTF8.cpp +++ b/inkcpp_test/UTF8.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -18,7 +19,7 @@ SCENARIO("a story supports UTF-8", "[utf-8]") std::unique_ptr ink{story::from_file("UTF8Story.bin")}; runner thread = ink->new_runner(); - std::ifstream demoFile("ink/UTF-8-demo.txt"); + std::ifstream demoFile(INK_TEST_RESOURCE_DIR "UTF-8-demo.txt"); if (! demoFile.is_open()) { throw std::runtime_error("cannot open UTF-8 demo file"); } diff --git a/shared/private/header.h b/shared/private/header.h index 81e151d1..43383f15 100644 --- a/shared/private/header.h +++ b/shared/private/header.h @@ -7,44 +7,113 @@ #pragma once #include "system.h" +#include "command.h" namespace ink::internal { struct header { - static header parse_header(const char* data); - template - static T swap_bytes(const T& value) + static constexpr uint32_t InkBinMagic = ('I' << 24) | ('N' << 16) | ('K' << 8) | 'B'; + static constexpr uint32_t InkBinMagic_Differ = ('B' << 24) | ('K' << 16) | ('N' << 8) | 'I'; + static constexpr uint32_t Alignment = 16; + + uint32_t ink_bin_magic = InkBinMagic; + uint16_t ink_version_number = 0; + uint16_t ink_bin_version_number = 0; + + enum class endian_types : uint8_t { + none, + same, + differ + }; + + constexpr endian_types endian() const { - char data[sizeof(T)]; - for (size_t i = 0; i < sizeof(T); ++i) { - data[i] = reinterpret_cast(&value)[sizeof(T) - 1 - i]; + switch (ink_bin_magic) { + case InkBinMagic: return endian_types::same; + case InkBinMagic_Differ: return endian_types::differ; + default: return endian_types::none; } - return *reinterpret_cast(data); } - list_flag read_list_flag(const char*& ptr) const - { - list_flag result = *reinterpret_cast(ptr); - ptr += sizeof(list_flag); - if (endien == ink::internal::header::endian_types::differ) { - result.flag = swap_bytes(result.flag); - result.list_id = swap_bytes(result.list_id); + bool verify() const; + + struct section_t { + uint32_t _start = 0; + uint32_t _bytes = 0; + + void setup(uint32_t& offset, uint32_t bytes) + { + _start = (offset + Alignment - 1) & ~(Alignment - 1); + _bytes = bytes; + offset = _start + _bytes; } - return result; - } + }; + + // File section sizes. Each section is aligned to Alignment + section_t _strings; + section_t _list_meta; + section_t _lists; + section_t _containers; + section_t _container_map; + section_t _container_hash; + section_t _instructions; +}; + +// One entry in the container hash. Used to translate paths into story locations. +struct container_hash_t { + // Hash of the container's path string. + hash_t _hash; + + // Offset to the start of this container. + uint32_t _offset; + + uint32_t key() const { return _hash; } + + bool operator<(const container_hash_t& other) const { return _hash < other._hash; } +}; + +// One entry in the container map. Used to work out which container a story location is in. +struct container_map_t { + // Offset to the start of this container's instructions. + uint32_t _offset; + + // Container index. + container_t _id; + + uint32_t key() const { return _offset; } + + bool operator<(const container_map_t& other) const { return _offset < other._offset; } +}; + +// One entry in the container data. Describes containers. +struct container_data_t { + /// Parent container, or ~0 if this is the root. + // TODO: Pack into 28 with explicit invalid container_t, since we expect fewer containers than + // instructions. + container_t _parent; + + /// Container flags (saves looking up via instruction data) + uint32_t _flags : 4; + + /// Instruction offset to the start instruction (enter marker) of this container. + uint32_t _start_offset : 28; + + /// Instruction offset to the end instruction (leave marker) of this container + uint32_t _end_offset; + + /// Container hash. + uint32_t _hash; + + /// Check to see if the instruction offset is part of the instructions for this container. Note + /// that this is inclusive not exclusive. + bool contains(uint32_t offset) const { return offset >= _start_offset && offset <= _end_offset; } + + /// Check to see if this is a knot container. + bool knot() const { return _flags & uint8_t(CommandFlag::CONTAINER_MARKER_IS_KNOT); } - enum class endian_types : uint16_t { - none = 0, - same = 0x0001, - differ = 0x0100 - } endien = endian_types::none; - uint32_t ink_version_number = 0; - uint32_t ink_bin_version_number = 0; - static constexpr size_t Size = ///< actual data size of Header, - /// because padding of struct may - /// differ between platforms - sizeof(uint16_t) + 2 * sizeof(uint32_t); + /// Check to see if this is a container which tracks visits. + bool visit() const { return _flags & uint8_t(CommandFlag::CONTAINER_MARKER_TRACK_VISITS); } }; } // namespace ink::internal diff --git a/shared/public/system.h b/shared/public/system.h index ecc25092..82259cbd 100644 --- a/shared/public/system.h +++ b/shared/public/system.h @@ -115,6 +115,13 @@ struct list_flag { bool operator!=(const list_flag& o) const { return ! (*this == o); } }; +inline list_flag read_list_flag(const char*& ptr) +{ + list_flag result = *reinterpret_cast(ptr); + ptr += sizeof(list_flag); + return result; +} + /** value of an unset list_flag */ constexpr list_flag null_flag{-1, -1}; /** value representing an empty list */ diff --git a/shared/public/version.h b/shared/public/version.h index db81dc29..62ea2faa 100644 --- a/shared/public/version.h +++ b/shared/public/version.h @@ -9,6 +9,6 @@ #include "system.h" namespace ink { -constexpr uint32_t InkBinVersion = 1; ///< Supportet version of ink.bin files +constexpr uint32_t InkBinVersion = 2; ///< Supportet version of ink.bin files constexpr uint32_t InkVersion = 21; ///< Supported version of ink.json files };